fix(api): stop signing sessions with a public default secret

.changeset/express-session-secret-no-default.md

@@ -0,0 +1,12 @@
+---
+'@hyperdx/api': minor
+---
+
+The API no longer falls back to the public session secret `hyperdx is cool 👋`.
+If `EXPRESS_SESSION_SECRET` is unset, the API generates a random secret and
+reports it once at startup when authentication is enabled. Authenticated
+deployments that explicitly set the old public value also get a random secret
+and the same warning; the public value remains available only in the no-auth
+local demo. Set it to a random string (`openssl rand -hex 32`) in any deployment
+with authentication enabled — without it, restarting the API signs every user
+out, and replicas do not share sessions.

.env

@@ -34,6 +34,10 @@ WEBHOOK_HOSTNAME_ALLOWLIST=
 # Generate with `openssl rand -base64 32`. Unset means plain text.
 TOKEN_ENCRYPTION_KEY=
 
+# Generate once with openssl rand -hex 32; reuse across restarts and replicas.
+# Configure in .env.local with docker compose --env-file .env --env-file .env.local up -d.
+# EXPRESS_SESSION_SECRET=
+
 # Docker service ports (overridden by scripts/dev-env.sh for isolation)
 HDX_DEV_MONGO_PORT=27017
 HDX_DEV_CH_HTTP_PORT=8123

DEPLOY.md

@@ -27,8 +27,11 @@ You'll need to set the `MONGO_URI` environment variable to the URI of your
 MongoDB instance. Afterwards, you'll want to visit http://localhost:8080 to set
 up your connection with ClickHouse.
 
-Before deploying into production, you'll want to set the
-`EXPRESS_SESSION_SECRET` environment variable to a random string.
+Before deploying into production, set the `EXPRESS_SESSION_SECRET` environment
+variable to a random string (e.g. `openssl rand -hex 32`). Without it the API
+generates one per process, which signs users out on every restart and prevents
+replicas from sharing sessions. The old public value `hyperdx is cool 👋` is
+also replaced with a per-process secret when authentication is enabled.
 
 To customize the frontend URL, set the `FRONTEND_URL` environment variable to
 the URL your HyperDX instance is hosted on.
@@ -55,8 +58,18 @@ such as ufw. See the
 [Docker docs](https://docs.docker.com/engine/network/packet-filtering-firewalls/#docker-and-ufw)
 for more information.
 
-Additionally, you'll want to set the `EXPRESS_SESSION_SECRET` environment
-variable to a random string.
+Generate a secret once with `openssl rand -hex 32`, then set
+`EXPRESS_SESSION_SECRET=<generated value>` in `.env.local`, which Git ignores.
+Do not put the secret in the tracked root `.env`. Compose forwards this value to
+the app container when you load `.env.local` after `.env`:
+
+```bash
+docker compose --env-file .env --env-file .env.local up -d
+```
+
+Reuse the same secret across restarts and all replicas. Without it the API
+generates one per process, which signs users out on every restart and prevents
+replicas from sharing sessions. The old public value is not a production secret.
 
 ## Local Development Mode
 

agent_docs/development.md

@@ -323,7 +323,7 @@ serves both the app and the API.
 | -------------------------------------------- | ----------------------------------------------------------------- |
 | `HDX_PREVIEW_INLINE_API`                     | `true` — turns on the inline path                                 |
 | `MONGO_URI`                                  | Preview MongoDB connection string                                 |
-| `EXPRESS_SESSION_SECRET`                     | Random 32+ char string                                            |
+| `EXPRESS_SESSION_SECRET`                     | Random 32+ char string shared by all instances; unset generates one per process, invalidating sessions on restart and between instances |
 | `DISABLED_AUTH_METHODS`                      | `google,saml` (avoids per-preview OAuth callback URL config)      |
 | `AI_PROVIDER`, `AI_API_KEY`, `AI_MODEL_NAME` | Optional, only if AI features should work                         |
 | `FRONTEND_URL`                               | Optional. Leave unset to use host-only cookies on the preview URL |

docker-compose.yml

@@ -82,6 +82,7 @@ services:
       # 32-byte base64 or hex key. Setting it encrypts stored Slack and OAuth
       # tokens; unset means they are stored in plain text.
       TOKEN_ENCRYPTION_KEY: ${TOKEN_ENCRYPTION_KEY:-}
+      EXPRESS_SESSION_SECRET: ${EXPRESS_SESSION_SECRET:-}
       # Uncomment the next two lines to enable PromQL (Prometheus-compatible metrics)
       # ENABLE_PROMQL: 'true'
       # NEXT_PUBLIC_ENABLE_PROMQL: 'true'

docker/hyperdx/entry.local.base.sh

@@ -22,7 +22,10 @@ export CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT="${CLICKHOUSE_PROMETHEUS_METRICS_E
 # (IS_LOCAL_IMAGE / IS_ALL_IN_ONE_IMAGE). The auth variant must report
 # "all-in-one-auth" so that the INGESTION_API_KEY bootstrap path is enabled.
 export HYPERDX_IMAGE=$([[ "${IS_LOCAL_APP_MODE}" == "DANGEROUSLY_is_local_app_mode💀" ]] && echo "all-in-one-noauth" || echo "all-in-one-auth")
-export EXPRESS_SESSION_SECRET="hyperdx is cool 👋"
+# Keep the public demo default only when authentication is disabled.
+if [[ "${IS_LOCAL_APP_MODE}" == "DANGEROUSLY_is_local_app_mode💀" ]]; then
+  export EXPRESS_SESSION_SECRET="${EXPRESS_SESSION_SECRET:-hyperdx is cool 👋}"
+fi
 # IS_LOCAL_APP_MODE should be set by the calling script
 # Default to dangerous mode if not set
 export IS_LOCAL_APP_MODE="${IS_LOCAL_APP_MODE}"

packages/api/.env.development

@@ -1,6 +1,7 @@
 # Ports come from scripts/dev-env.sh (worktree isolation) or root .env (defaults)
 HYPERDX_LOG_LEVEL=debug
-EXPRESS_SESSION_SECRET="hyperdx is cool 👋"
+# Development only: keep sessions valid across nodemon restarts.
+EXPRESS_SESSION_SECRET="local-development-only"
 FRONTEND_URL="http://localhost:${HYPERDX_APP_PORT}"
 HDX_NODE_ADVANCED_NETWORK_CAPTURE=1
 HDX_NODE_BETA_MODE=1

packages/api/.env.test

@@ -6,7 +6,6 @@ CLICKHOUSE_HOST=http://localhost:${HDX_CI_CH_PORT:-18123}
 CLICKHOUSE_PASSWORD=api
 CLICKHOUSE_USER=api
 RUN_SCHEDULED_TASKS_EXTERNALLY=true
-EXPRESS_SESSION_SECRET="hyperdx is cool 👋"
 FRONTEND_URL=http://app:8080
 MONGO_URI=mongodb://localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test
 NODE_ENV=test
@@ -16,4 +15,4 @@ OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320}
 # Note: the logger module is mocked in jest.setup.ts to suppress expected
 # operational noise (validation errors, MCP tool errors, etc.) during tests.
 HYPERDX_LOG_LEVEL=error
-ENABLE_PROMQL=true
+ENABLE_PROMQL=true

packages/api/src/__tests__/config.test.ts

@@ -123,4 +123,85 @@ describe('config', () => {
       },
     );
   });
+
+  describe('EXPRESS_SESSION_SECRET', () => {
+    const ORIGINAL_SECRET = process.env.EXPRESS_SESSION_SECRET;
+    const ORIGINAL_LOCAL_APP_MODE = process.env.IS_LOCAL_APP_MODE;
+
+    beforeEach(() => {
+      delete process.env.IS_LOCAL_APP_MODE;
+    });
+
+    afterEach(() => {
+      if (ORIGINAL_SECRET === undefined) {
+        delete process.env.EXPRESS_SESSION_SECRET;
+      } else {
+        process.env.EXPRESS_SESSION_SECRET = ORIGINAL_SECRET;
+      }
+      if (ORIGINAL_LOCAL_APP_MODE === undefined) {
+        delete process.env.IS_LOCAL_APP_MODE;
+      } else {
+        process.env.IS_LOCAL_APP_MODE = ORIGINAL_LOCAL_APP_MODE;
+      }
+      jest.resetModules();
+    });
+
+    // The `jest` object's isolateModules returns `jest` itself rather than the
+    // callback's value, so the module has to be captured from inside it.
+    const loadConfig = () => {
+      let loaded!: typeof import('@/config');
+      jest.isolateModules(() => {
+        loaded = jest.requireActual<typeof import('@/config')>('@/config');
+      });
+      return loaded;
+    };
+
+    it('uses a configured secret verbatim', () => {
+      process.env.EXPRESS_SESSION_SECRET = 'a-deployment-specific-secret';
+
+      const config = loadConfig();
+      expect(config.EXPRESS_SESSION_SECRET).toBe(
+        'a-deployment-specific-secret',
+      );
+      expect(config.IS_EXPRESS_SESSION_SECRET_GENERATED).toBe(false);
+    });
+
+    it('preserves a configured secret verbatim', () => {
+      process.env.EXPRESS_SESSION_SECRET = '  verbatim secret 👋  ';
+      const config = loadConfig();
+      expect(config.EXPRESS_SESSION_SECRET).toBe('  verbatim secret 👋  ');
+      expect(config.IS_EXPRESS_SESSION_SECRET_GENERATED).toBe(false);
+    });
+
+    it('generates a secret when unset, empty, or publicly known', () => {
+      for (const value of [undefined, '', 'hyperdx is cool 👋']) {
+        if (value === undefined) {
+          delete process.env.EXPRESS_SESSION_SECRET;
+        } else {
+          process.env.EXPRESS_SESSION_SECRET = value;
+        }
+
+        const config = loadConfig();
+        expect(config.EXPRESS_SESSION_SECRET).toMatch(/^[0-9a-f]{64}$/);
+        expect(config.IS_EXPRESS_SESSION_SECRET_GENERATED).toBe(true);
+      }
+    });
+
+    it('keeps the public demo secret only without authentication', () => {
+      process.env.IS_LOCAL_APP_MODE = 'DANGEROUSLY_is_local_app_mode💀';
+      process.env.EXPRESS_SESSION_SECRET = 'hyperdx is cool 👋';
+
+      const config = loadConfig();
+      expect(config.EXPRESS_SESSION_SECRET).toBe('hyperdx is cool 👋');
+      expect(config.IS_EXPRESS_SESSION_SECRET_GENERATED).toBe(false);
+    });
+
+    it('generates a different secret on each load', () => {
+      delete process.env.EXPRESS_SESSION_SECRET;
+
+      expect(loadConfig().EXPRESS_SESSION_SECRET).not.toBe(
+        loadConfig().EXPRESS_SESSION_SECRET,
+      );
+    });
+  });
 });

packages/api/src/__tests__/sessionSecretStartup.test.ts

@@ -0,0 +1,92 @@
+import { IncomingMessage, ServerResponse } from 'http';
+import { Socket } from 'net';
+
+jest.mock('@/api-app', () => ({ __esModule: true, default: jest.fn() }));
+jest.mock('@/controllers/team', () => ({
+  LOCAL_APP_TEAM: { _id: { toString: () => 'local-team' } },
+}));
+jest.mock('@/migrations', () => ({ runStartupMigrations: jest.fn() }));
+jest.mock('@/models', () => ({
+  connectDB: jest.fn().mockResolvedValue(undefined),
+  connectDBWithRetry: jest.fn().mockResolvedValue(undefined),
+}));
+jest.mock('@/opamp/app', () => ({ __esModule: true, default: jest.fn() }));
+jest.mock('@/setupDefaults', () => ({ setupTeamDefaults: jest.fn() }));
+jest.mock('@/utils/tokenEncryption', () => ({
+  verifyTokenEncryption: jest.fn(),
+}));
+jest.mock('@/utils/logger', () => ({
+  __esModule: true,
+  default: { error: jest.fn(), info: jest.fn() },
+}));
+jest.mock('@/utils/instrumentation', () => ({
+  getCounter: jest.fn(() => ({ add: jest.fn() })),
+}));
+jest.mock('http-graceful-shutdown', () => jest.fn());
+jest.mock('http', () => ({
+  ...jest.requireActual('http'),
+  createServer: jest.fn(() => ({ listen: jest.fn() })),
+}));
+
+describe.each(['server', 'serverless'])(
+  'session secret startup diagnostics (%s first)',
+  first => {
+    const originalEnv = { ...process.env };
+
+    afterEach(() => {
+      process.env = { ...originalEnv };
+      jest.resetModules();
+    });
+
+    it.each([
+      [false, undefined, 1],
+      [false, '', 1],
+      [false, 'hyperdx is cool 👋', 1],
+      [false, 'startup-secret-sentinel', 0],
+      [true, undefined, 0],
+      [true, 'hyperdx is cool 👋', 0],
+      [true, 'startup-secret-sentinel', 0],
+    ])(
+      'local=%s, secret=%s logs %s times across both entrypoints',
+      async (local, secret, count) => {
+        process.env.APP_TYPE = 'api';
+        if (local) {
+          process.env.IS_LOCAL_APP_MODE = 'DANGEROUSLY_is_local_app_mode💀';
+        } else {
+          delete process.env.IS_LOCAL_APP_MODE;
+        }
+        if (typeof secret === 'undefined')
+          delete process.env.EXPRESS_SESSION_SECRET;
+        else process.env.EXPRESS_SESSION_SECRET = secret;
+
+        const { default: logger } =
+          jest.requireMock<typeof import('@/utils/logger')>('@/utils/logger');
+        const { getCounter } = jest.requireMock<
+          typeof import('@/utils/instrumentation')
+        >('@/utils/instrumentation');
+        const { default: Server } =
+          jest.requireActual<typeof import('@/server')>('@/server');
+        const { serverlessHandler } =
+          jest.requireActual<typeof import('@/serverless')>('@/serverless');
+        const config =
+          jest.requireActual<typeof import('@/config')>('@/config');
+        expect(logger.error).not.toHaveBeenCalled();
+
+        const req = new IncomingMessage(new Socket());
+        req.url = '/api/health';
+        const res = new ServerResponse(req);
+        if (first === 'server') await new Server().start();
+        await serverlessHandler(req, res);
+        await new Server().start();
+        await serverlessHandler(req, res);
+
+        expect(logger.error).toHaveBeenCalledTimes(count);
+        const counter = jest.mocked(getCounter).mock.results[0].value;
+        expect(counter.add).toHaveBeenCalledTimes(count);
+        expect(
+          JSON.stringify(jest.mocked(logger.error).mock.calls),
+        ).not.toContain(config.EXPRESS_SESSION_SECRET);
+      },
+    );
+  },
+);

packages/api/src/config.ts

@@ -1,8 +1,9 @@
+import crypto from 'crypto';
+
 const env = process.env;
 
 // DEFAULTS
 const DEFAULT_APP_TYPE = 'api';
-const DEFAULT_EXPRESS_SESSION = 'hyperdx is cool 👋';
 const DEFAULT_FRONTEND_URL = env.HYPERDX_APP_PORT
   ? `http://localhost:${env.HYPERDX_APP_PORT}`
   : '';
@@ -13,8 +14,20 @@ export const APP_TYPE = (env.APP_TYPE || DEFAULT_APP_TYPE) as
   | 'api'
   | 'scheduled-task';
 export const CODE_VERSION = env.CODE_VERSION ?? '';
-export const EXPRESS_SESSION_SECRET = (env.EXPRESS_SESSION_SECRET ||
-  DEFAULT_EXPRESS_SESSION) as string;
+// Only for single container local deployments, disable authentication
+export const IS_LOCAL_APP_MODE =
+  env.IS_LOCAL_APP_MODE === 'DANGEROUSLY_is_local_app_mode💀';
+const LEGACY_PUBLIC_SESSION_SECRET = 'hyperdx is cool 👋';
+// Authenticated deployments never use the public demo secret. Without a safe
+// configured value, generate one per process (sessions end on restart).
+export const IS_EXPRESS_SESSION_SECRET_GENERATED =
+  !env.EXPRESS_SESSION_SECRET ||
+  (!IS_LOCAL_APP_MODE &&
+    env.EXPRESS_SESSION_SECRET === LEGACY_PUBLIC_SESSION_SECRET);
+export const EXPRESS_SESSION_SECRET =
+  !IS_EXPRESS_SESSION_SECRET_GENERATED && env.EXPRESS_SESSION_SECRET
+    ? env.EXPRESS_SESSION_SECRET
+    : crypto.randomBytes(32).toString('hex');
 export const FRONTEND_URL = (env.FRONTEND_URL ||
   DEFAULT_FRONTEND_URL) as string;
 const HYPERDX_IMAGE = env.HYPERDX_IMAGE;
@@ -52,10 +65,6 @@ export const RUN_SCHEDULED_TASKS_EXTERNALLY =
 // utils/tokenEncryption.ts.
 export const TOKEN_ENCRYPTION_KEY = env.TOKEN_ENCRYPTION_KEY;
 
-// Only for single container local deployments, disable authentication
-export const IS_LOCAL_APP_MODE =
-  env.IS_LOCAL_APP_MODE === 'DANGEROUSLY_is_local_app_mode💀';
-
 // Only used to bootstrap empty instances
 export const DEFAULT_CONNECTIONS = env.DEFAULT_CONNECTIONS;
 export const DEFAULT_SOURCES = env.DEFAULT_SOURCES;

packages/api/src/server.ts

@@ -10,6 +10,7 @@ import { connectDBWithRetry, mongooseConnection } from '@/models';
 import opampApp from '@/opamp/app';
 import { setupTeamDefaults } from '@/setupDefaults';
 import logger from '@/utils/logger';
+import { verifySessionSecret } from '@/utils/sessionSecret';
 import { verifyTokenEncryption } from '@/utils/tokenEncryption';
 
 export default class Server {
@@ -90,6 +91,7 @@ export default class Server {
     // Checked before Mongo so a bad encryption key or KMS policy is reported
     // even while the Mongo connect below is still retrying.
     await verifyTokenEncryption();
+    verifySessionSecret();
 
     // The HTTP servers above are already listening so that `/health`
     // (liveness) responds while we connect; `/ready` (readiness) stays 503

packages/api/src/serverless.ts

@@ -28,6 +28,7 @@ import app from './api-app';
 import * as config from './config';
 import { connectDB } from './models';
 import logger from './utils/logger';
+import { verifySessionSecret } from './utils/sessionSecret';
 
 // Guard against misconfigured Vercel previews. The serverless entrypoint only
 // supports the `api` app type — OPAMP runs as its own long-lived server and
@@ -76,6 +77,7 @@ export async function serverlessHandler(
   req: IncomingMessage,
   res: ServerResponse,
 ): Promise<void> {
+  verifySessionSecret();
   try {
     await ensureDb();
   } catch {

packages/api/src/utils/sessionSecret.ts

@@ -0,0 +1,23 @@
+import {
+  IS_EXPRESS_SESSION_SECRET_GENERATED,
+  IS_LOCAL_APP_MODE,
+} from '@/config';
+import { getCounter } from '@/utils/instrumentation';
+import logger from '@/utils/logger';
+
+const generatedSecretCounter = getCounter('hyperdx.session_secret.generated', {
+  description:
+    'API processes using a generated session secret with authentication enabled.',
+});
+
+let reported = false;
+
+export function verifySessionSecret(): void {
+  if (reported || IS_LOCAL_APP_MODE || !IS_EXPRESS_SESSION_SECRET_GENERATED)
+    return;
+  reported = true;
+  generatedSecretCounter.add(1);
+  logger.error(
+    'EXPRESS_SESSION_SECRET is missing or uses the public legacy value: generated a random secret for this process. Users are signed out whenever it restarts, and replicas do not share sessions. Configure a random secret shared by all replicas to keep sessions.',
+  );
+}