From 7439838e3e0ad46d55daf9922ff545f477139951 Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Thu, 7 May 2026 18:08:29 +0000 Subject: [PATCH 1/2] fix(api): tighten redactSecrets after deep-review on #2188 Deep-review on the merged #2188 surfaced three security/correctness gaps in the redactor; this PR addresses each. - `bearer` value class now includes `_`, so a JWT bearer token with underscores in its signature ("eyJ...AbC_DeF...") no longer terminates at the first underscore and leaks the trailing bytes past the [REDACTED] marker. base64url uses both "_" and "-"; the alphabet is now consistent with the `jwt` pattern. - `basic-auth-url` scheme allowlist now covers the database/queue connection strings most likely to land in observability payloads with embedded credentials: postgres(ql), mysql, mariadb, mongodb(+srv), redis(s), amqp(s), kafka, clickhouse. Previously these slipped through entirely while http(s)/ftp/ssh were redacted. - New `llm-vendor-key` pattern catches OpenAI ("sk-...") and Anthropic ("sk-ant-...") API keys. This redactor specifically fronts an LLM-provider call, so a vendor-shape key must not leak to the very provider that issued it. Floors at 20 chars after the prefix to avoid catching English fragments like "sk-ip" or "sk-line". Docstring now scopes the redactor explicitly to LLM input ("LLM-input only; not a general-purpose secret redactor"), per fleon's caveat on #2188 and the discovered gaps. Pattern coverage test, multi-secret integration test, and per-pattern regression tests cover each new shape plus the JWT-with-underscore regression. 53/53 unit tests pass. No OpenAPI / changeset surface change. --- .../redact-secrets-deep-review-followups.md | 23 +++++ .../src/utils/__tests__/redactSecrets.test.ts | 99 ++++++++++++++++++- packages/api/src/utils/redactSecrets.ts | 54 ++++++++-- 3 files changed, 165 insertions(+), 11 deletions(-) create mode 100644 .changeset/redact-secrets-deep-review-followups.md diff --git a/.changeset/redact-secrets-deep-review-followups.md b/.changeset/redact-secrets-deep-review-followups.md new file mode 100644 index 0000000000..939a9c66eb --- /dev/null +++ b/.changeset/redact-secrets-deep-review-followups.md @@ -0,0 +1,23 @@ +--- +'@hyperdx/api': patch +--- + +fix(api): tighten redactSecrets after deep-review on #2188 + +Three security/correctness gaps surfaced by the deep-review on the +original redactSecrets PR. + +- The `bearer` value class now includes `_`, so a JWT bearer token + with underscores in the signature ("eyJ...AbC_DeF") no longer + terminates at the first underscore and leaks the trailing bytes + past the [REDACTED] marker. +- The `basic-auth-url` scheme allowlist now covers + postgres / postgresql / mysql / mariadb / mongodb(+srv) / redis(s) + / amqp(s) / kafka / clickhouse, so embedded credentials in + database connection strings are redacted alongside http(s) / ftp / ssh. +- New `llm-vendor-key` pattern catches OpenAI ("sk-...") and + Anthropic ("sk-ant-...") API keys, so a vendor key cannot leak to + the very provider that issued it. + +Docstring now scopes the redactor explicitly to LLM input. Tests +cover each new shape and the JWT-with-underscore regression. diff --git a/packages/api/src/utils/__tests__/redactSecrets.test.ts b/packages/api/src/utils/__tests__/redactSecrets.test.ts index 97589b6448..0b8e3f292b 100644 --- a/packages/api/src/utils/__tests__/redactSecrets.test.ts +++ b/packages/api/src/utils/__tests__/redactSecrets.test.ts @@ -110,6 +110,19 @@ describe('redactSecrets', () => { ); }); + it('redacts a JWT-shaped Bearer with underscores in the signature', () => { + // base64url uses both "_" and "-"; a real JWT signature routinely + // contains underscores. Without "_" in the bearer alphabet the + // match terminated at the first underscore and the signature tail + // leaked past the [REDACTED] marker. + const line = + 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.AbC_DeF_GhIjKl'; + const out = redactSecrets(line); + expect(out).toContain('Bearer [REDACTED]'); + expect(out).not.toContain('AbC_DeF_GhIjKl'); + expect(out).not.toContain('GhIjKl'); + }); + it('redacts Basic values', () => { expect(redactSecrets('Authorization: Basic dXNlcjpwYXNz')).toContain( 'Basic [REDACTED]', @@ -241,6 +254,51 @@ describe('redactSecrets', () => { const line = 'fetch ssh://git@github.com/acme/repo'; expect(redactSecrets(line)).toBe(line); }); + + it('redacts user:pass in a postgres connection string', () => { + const out = redactSecrets( + 'DATABASE_URL=postgres://app:hunter2@db.internal:5432/orders', + ); + expect(out).toContain( + 'postgres://[REDACTED]:[REDACTED]@db.internal:5432/orders', + ); + expect(out).not.toContain('hunter2'); + }); + + it('redacts user:pass in a postgresql connection string', () => { + const out = redactSecrets( + 'jdbc:postgresql://app:hunter2@db.internal:5432/orders', + ); + expect(out).toContain( + 'postgresql://[REDACTED]:[REDACTED]@db.internal:5432/orders', + ); + expect(out).not.toContain('hunter2'); + }); + + it('redacts user:pass in a mongodb+srv connection string', () => { + const out = redactSecrets( + 'mongo: mongodb+srv://svc:hunter2@cluster.mongodb.net/db', + ); + expect(out).toContain( + 'mongodb+srv://[REDACTED]:[REDACTED]@cluster.mongodb.net/db', + ); + expect(out).not.toContain('hunter2'); + }); + + it('redacts user:pass in mysql, redis, amqp, kafka, clickhouse', () => { + const cases = [ + ['mysql://app:hunter2@db.local/orders', 'mysql'], + ['rediss://default:hunter2@cache.local:6379/0', 'rediss'], + ['amqps://app:hunter2@rabbit.local:5671/vh', 'amqps'], + ['kafka://app:hunter2@broker.local:9092/topic', 'kafka'], + ['clickhouse://default:hunter2@ch.local:9000/default', 'clickhouse'], + ]; + for (const [input, scheme] of cases) { + const out = redactSecrets(input); + expect(out.startsWith(`${scheme}://[REDACTED]:[REDACTED]@`)).toBe(true); + expect(out).not.toContain('hunter2'); + } + }); }); describe('AWS access keys', () => { @@ -311,14 +369,48 @@ describe('redactSecrets', () => { }); }); + describe('LLM vendor API keys', () => { + it('redacts an OpenAI sk- key', () => { + const key = 'sk-' + 'A'.repeat(48); + const out = redactSecrets(`OPENAI_API_KEY=${key}`); + expect(out).toContain('[REDACTED_LLM_KEY]'); + expect(out).not.toContain(key); + }); + + it('redacts an Anthropic sk-ant- key', () => { + const key = 'sk-ant-' + 'B'.repeat(48); + const out = redactSecrets(`ANTHROPIC_API_KEY=${key}`); + expect(out).toContain('[REDACTED_LLM_KEY]'); + expect(out).not.toContain(key); + }); + + it('redacts a free-floating sk- key in a log line', () => { + const key = 'sk-' + 'C'.repeat(40); + const out = redactSecrets(`request body contained ${key} apparently`); + expect(out).toContain('[REDACTED_LLM_KEY]'); + expect(out).not.toContain(key); + }); + + it('does not match short fragments', () => { + // "sk-" alone, or with very few trailing chars, is a common + // English fragment ("sk-ip", "sk-line"). The pattern requires + // at least 20 chars after the prefix. + const line = 'task: sk-ip-the-rest and sk-line are not tokens'; + expect(redactSecrets(line)).toBe(line); + }); + }); + describe('multi-secret payloads', () => { it('redacts every distinct secret in one pass', () => { + const llmKey = 'sk-' + 'L'.repeat(48); const input = [ 'conn url: https://alice:hunter2@db.example.com/app', - 'auth: Bearer eyJhbGciOiJIUzI1NiJ9.eyJ1IjoxfQ.sig', + 'pg url: postgres://svc:hunter2@db.internal/orders', + 'auth: Bearer eyJhbGciOiJIUzI1NiJ9.eyJ1IjoxfQ.AbC_DeF_GhIjKl', 'aws: AKIAIOSFODNN7EXAMPLE', 'gh: ghp_' + 'X'.repeat(36), 'slack: xoxb-1111-aaaaaaaaaa', + `openai: ${llmKey}`, 'env: password=hunter2 api_key=abc', ].join('\n'); @@ -327,11 +419,15 @@ describe('redactSecrets', () => { expect(out).not.toContain('hunter2'); expect(out).not.toContain('AKIAIOSFODNN7EXAMPLE'); expect(out).not.toContain('xoxb-1111-aaaaaaaaaa'); + expect(out).not.toContain(llmKey); + expect(out).not.toContain('GhIjKl'); expect(out).toContain('[REDACTED]:[REDACTED]@db.example.com'); + expect(out).toContain('postgres://[REDACTED]:[REDACTED]@db.internal'); expect(out).toContain('Bearer [REDACTED]'); expect(out).toContain('[REDACTED_AWS_KEY]'); expect(out).toContain('[REDACTED_GITHUB_TOKEN]'); expect(out).toContain('[REDACTED_SLACK_TOKEN]'); + expect(out).toContain('[REDACTED_LLM_KEY]'); expect(out).toContain('password=[REDACTED]'); expect(out).toContain('api_key=[REDACTED]'); }); @@ -349,6 +445,7 @@ describe('redactSecrets', () => { 'aws-access-key', 'slack-token', 'github-token', + 'llm-vendor-key', 'key-value', 'json-quoted', 'http-header', diff --git a/packages/api/src/utils/redactSecrets.ts b/packages/api/src/utils/redactSecrets.ts index f364089f65..263a593ca7 100644 --- a/packages/api/src/utils/redactSecrets.ts +++ b/packages/api/src/utils/redactSecrets.ts @@ -2,6 +2,12 @@ // originates from observability data (log bodies, span attributes, // pattern samples, alert payloads). // +// **LLM-input only.** Do not use as a general-purpose secret redactor: +// the patterns are tuned for the shapes that show up in observability +// payloads, not for export pipelines, audit logs, or anywhere a missed +// secret has compliance consequences. A redactor for those callers +// needs a different threat model and probably entropy-based scanning. +// // Design rule: any LLM input derived from observability data goes // through redactSecrets before leaving the API process. User-authored // prose (e.g. the chart-builder assistant where the user types their @@ -15,22 +21,30 @@ // // Patterns covered: // pem PEM key blocks (-----BEGIN ... PRIVATE KEY-----) -// basic-auth-url https://user:pass@host +// basic-auth-url scheme://user:pass@host. Schemes: http(s), ftp, +// ssh, postgres(ql), mysql, mariadb, mongodb(+srv), +// redis(s), amqp(s), kafka, clickhouse. // key-value password=secret, api_key=abc // json-quoted {"password":"secret"} and similar // http-header X-Api-Key: abc, Api-Key: abc -// bearer Authorization: Bearer xxx +// bearer Authorization: Bearer xxx (token alphabet now +// includes "_" so JWT signatures are fully covered). // basic Authorization: Basic xxx // jwt eyJ... three dot-separated base64 segments // aws-access-key AKIA[16 chars], ASIA[16 chars] // slack-token xox[a-z]-... shape // github-token ghp_, gho_, ghu_, ghs_, ghr_ prefixes +// llm-vendor-key sk-... (OpenAI), sk-ant-... (Anthropic). This +// redactor specifically fronts an LLM-provider +// call, so vendor-shaped keys must not leak to +// the very provider that issued them. // // Known gaps (extend when seen in production): -// URL-percent-encoded values, vendor-specific tokens (Stripe, Twilio, -// Datadog, etc.), generic high-entropy hex blobs (too many false -// positives without surrounding context), basic-auth URLs with raw -// "@" in the username (ambiguous to parse without percent-encoding). +// URL-percent-encoded values; non-LLM vendor tokens (Stripe, Twilio, +// Datadog, GCP / Google Maps, generic OAuth refresh shapes); generic +// high-entropy hex blobs (too many false positives without +// surrounding context); basic-auth URLs with raw "@" in the username +// (ambiguous to parse without percent-encoding). const SECRET_KEY_TOKENS = 'password|passwd|pwd|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|client[_-]?secret|authorization|auth'; @@ -57,16 +71,24 @@ const PATTERNS: RedactionPattern[] = [ }, // scheme://user:pass@host. Password may contain "@"; the engine // backtracks to the last "@" before the host. Host is captured and - // preserved in the replacement. + // preserved in the replacement. The scheme group covers HTTP-family + // protocols plus the database/queue connection strings most likely + // to land in observability payloads with embedded credentials: + // postgres / mysql / mongodb / redis / amqp / kafka / clickhouse. { name: 'basic-auth-url', - re: /\b(https?|ftp|ssh):\/\/([^/\s:@]+):([^/\s]+)@([^/\s@]+)/g, + re: /\b(https?|ftp|ssh|postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|rediss?|amqps?|kafka|clickhouse):\/\/([^/\s:@]+):([^/\s]+)@([^/\s@]+)/g, replace: '$1://[REDACTED]:[REDACTED]@$4', }, - // Authorization: Bearer xxx + // Authorization: Bearer xxx. The token alphabet covers both URL-safe + // base64 ("_-" plus alphanumerics) and the standard base64 alphabet + // ("+/" plus "=" padding) so a JWT bearer ("eyJ..._...sig") and a + // token with literal "+" / "=" both round-trip cleanly. Without "_" + // a JWT terminates at the first underscore inside the signature and + // the trailing bytes leak past the [REDACTED] marker. { name: 'bearer', - re: /Bearer\s+[A-Za-z0-9._~+/=-]+/gi, + re: /Bearer\s+[A-Za-z0-9._~+/=_-]+/gi, replace: 'Bearer [REDACTED]', }, // Authorization: Basic xxx (base64 user:pass) @@ -102,6 +124,18 @@ const PATTERNS: RedactionPattern[] = [ re: /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/g, replace: '[REDACTED_GITHUB_TOKEN]', }, + // LLM-vendor API keys: OpenAI ("sk-..."), Anthropic ("sk-ant-..."), + // and adjacent vendors that follow the "sk-" convention. This + // redactor specifically fronts an LLM call; a leaked vendor key + // would be exfiltrated to the very provider that issued it. + // Floor at 20 chars after the prefix to avoid catching English + // words like "sk-ip-line" or short test fixtures while still + // covering OpenAI's 48+ char and Anthropic's longer formats. + { + name: 'llm-vendor-key', + re: /\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, + replace: '[REDACTED_LLM_KEY]', + }, // key=value with secret-ish key. Three value forms: double-quoted, // single-quoted, or unquoted (stops at whitespace, comma, semicolon, // ampersand, quote so URL query-string boundaries are preserved). From 41816215506ec6b5b54e4661c59a449827e8ffb3 Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Fri, 8 May 2026 00:45:16 +0000 Subject: [PATCH 2/2] fix(api): close four more redactSecrets gaps from deep-review on #2235 Deep-review on the in-flight #2235 flagged two P0/P1 leak paths and two critical P2 bypass paths. This commit addresses each. - `llm-vendor-key`: add Google Gemini ("AIza..." with 35 trailing chars from [A-Za-z0-9_-]; full key is 39 chars). Without this branch a Gemini key in an observability payload reaches the very provider that issued it. - `basic-auth-url` scheme allowlist: add ws(s), sftp, mssql, sqlserver, snowflake, kafka+ssl, smtp(s), ldap(s), nats. Embedded credentials in those connection strings previously slipped through unredacted. Schemes now alphabetical for easier visual diffing. - `basic-auth-url` regex: add `i` flag. RFC 3986 declares schemes case-insensitive, so HTTPS://user:pass@host now redacts the same as the lowercase form. Previously an attacker could bypass redaction by upcasing the scheme. - `bearer` token alphabet: replace [A-Za-z0-9._~+/=_-]+ with \S+. RFC 6750's b64token alphabet is a strict subset of \S+, but real observability payloads carry plenty of opaque non-JWT bearers with ":", "%", or quote chars in them, and any narrower alphabet leaks the suffix past [REDACTED]. \S+ stops at whitespace, which is what actually terminates a bearer in practice. Tests cover each new shape: a Gemini key (positive + length-floor negative + bare-prefix negative), the 13-scheme parametric row for the extended allowlist (incidentally adding the missing mariadb case), uppercase + mixed-case scheme regressions, and three new opaque-bearer regressions (":", "%", and a "stops at whitespace" guard). Skipped from this commit and tracked elsewhere: the `/api/sk-...` path lookbehind false-positive (#2237) and the P3 nitpicks. 63/63 unit tests pass. No OpenAPI / API surface change. Tier 2. --- .../redact-secrets-deep-review-followups.md | 37 ++++-- .../src/utils/__tests__/redactSecrets.test.ts | 121 +++++++++++++++++- packages/api/src/utils/redactSecrets.ts | 67 ++++++---- 3 files changed, 180 insertions(+), 45 deletions(-) diff --git a/.changeset/redact-secrets-deep-review-followups.md b/.changeset/redact-secrets-deep-review-followups.md index 939a9c66eb..0a5f04ab64 100644 --- a/.changeset/redact-secrets-deep-review-followups.md +++ b/.changeset/redact-secrets-deep-review-followups.md @@ -4,20 +4,29 @@ fix(api): tighten redactSecrets after deep-review on #2188 -Three security/correctness gaps surfaced by the deep-review on the -original redactSecrets PR. +Several security/correctness gaps surfaced by deep-review across +two passes on the original redactSecrets PR. -- The `bearer` value class now includes `_`, so a JWT bearer token - with underscores in the signature ("eyJ...AbC_DeF") no longer - terminates at the first underscore and leaks the trailing bytes - past the [REDACTED] marker. +- The `bearer` value alphabet is now `\S+`. Real-world payloads + carry plenty of opaque non-JWT bearers with `:`, `%`, or quote + chars in them, and any alphabet narrower than `\S+` leaks the + suffix past `[REDACTED]`. RFC 6750's b64token alphabet is a + strict subset of `\S+`. (Same fix subsumes the earlier change + that added `_` to cover JWT signatures.) - The `basic-auth-url` scheme allowlist now covers - postgres / postgresql / mysql / mariadb / mongodb(+srv) / redis(s) - / amqp(s) / kafka / clickhouse, so embedded credentials in - database connection strings are redacted alongside http(s) / ftp / ssh. -- New `llm-vendor-key` pattern catches OpenAI ("sk-...") and - Anthropic ("sk-ant-...") API keys, so a vendor key cannot leak to - the very provider that issued it. + http(s) / ws(s) / ftp / sftp / ssh / postgres(ql) / mysql / + mariadb / mongodb(+srv) / mssql / sqlserver / snowflake / + redis(s) / amqp(s) / kafka(+ssl) / clickhouse / smtp(s) / + ldap(s) / nats. The match is also case-insensitive (RFC 3986 + declares schemes case-insensitive), so `HTTPS://user:pw@host` + no longer bypasses redaction. +- The `llm-vendor-key` pattern now catches OpenAI ("sk-..."), + Anthropic ("sk-ant-..."), and Google Gemini ("AIza..." with 35 + trailing chars). Without Gemini coverage, a Gemini API key in + an observability payload would be exfiltrated to the very + provider that issued it. -Docstring now scopes the redactor explicitly to LLM input. Tests -cover each new shape and the JWT-with-underscore regression. +Docstring scopes the redactor explicitly to LLM input. Tests +cover each new shape, the JWT-with-underscore regression, the +opaque-bearer-with-`:` / `%` regressions, the uppercase-scheme +bypass, and the Gemini key shape. diff --git a/packages/api/src/utils/__tests__/redactSecrets.test.ts b/packages/api/src/utils/__tests__/redactSecrets.test.ts index 0b8e3f292b..f829a7a05a 100644 --- a/packages/api/src/utils/__tests__/redactSecrets.test.ts +++ b/packages/api/src/utils/__tests__/redactSecrets.test.ts @@ -112,9 +112,9 @@ describe('redactSecrets', () => { it('redacts a JWT-shaped Bearer with underscores in the signature', () => { // base64url uses both "_" and "-"; a real JWT signature routinely - // contains underscores. Without "_" in the bearer alphabet the - // match terminated at the first underscore and the signature tail - // leaked past the [REDACTED] marker. + // contains underscores. \S+ trivially covers them, but historically + // the alphabet excluded "_" and the signature tail leaked past the + // [REDACTED] marker. const line = 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.AbC_DeF_GhIjKl'; const out = redactSecrets(line); @@ -123,6 +123,35 @@ describe('redactSecrets', () => { expect(out).not.toContain('GhIjKl'); }); + it('redacts an opaque non-JWT bearer that contains a colon', () => { + // Opaque tokens (non-RFC-6750) sometimes use ":" as an internal + // delimiter. An alphabet that excludes ":" terminates the match + // early and leaks the suffix past [REDACTED]. + const out = redactSecrets( + 'Authorization: Bearer abc:def_real-secret-tail', + ); + expect(out).toContain('Authorization: Bearer [REDACTED]'); + expect(out).not.toContain('def_real-secret-tail'); + expect(out).not.toContain('secret-tail'); + }); + + it('redacts a bearer that contains percent-encoded bytes', () => { + // "%" is outside RFC 6750's b64token alphabet but is a legal byte + // inside an opaque bearer. \S+ covers it. + const out = redactSecrets('Authorization: Bearer token%20with%20encoded'); + expect(out).toContain('Authorization: Bearer [REDACTED]'); + expect(out).not.toContain('token%20with%20encoded'); + expect(out).not.toContain('encoded'); + }); + + it('stops a bearer match at whitespace', () => { + // Two tokens separated by whitespace must not collapse into one + // greedy match. Only the first non-whitespace run is the bearer. + const out = redactSecrets('auth: Bearer abc123 trailing-noise'); + expect(out).toContain('Bearer [REDACTED]'); + expect(out).toContain('trailing-noise'); + }); + it('redacts Basic values', () => { expect(redactSecrets('Authorization: Basic dXNlcjpwYXNz')).toContain( 'Basic [REDACTED]', @@ -299,6 +328,55 @@ describe('redactSecrets', () => { expect(out).not.toContain('hunter2'); } }); + + it('redacts user:pass across the extended scheme allowlist', () => { + // Schemes added to keep up with connection strings that show + // up in observability payloads: SQL Server (both spellings), + // Snowflake, Kafka with TLS, WebSocket, mail, SFTP, LDAP, + // NATS, MariaDB. + const cases: Array<[string, string]> = [ + ['mssql://app:hunter2@db.local:1433/orders', 'mssql'], + ['sqlserver://app:hunter2@db.local:1433/orders', 'sqlserver'], + [ + 'snowflake://acct:hunter2@xy12345.snowflakecomputing.com/db', + 'snowflake', + ], + ['kafka+ssl://app:hunter2@broker.local:9093/topic', 'kafka+ssl'], + ['ws://svc:hunter2@ws.local:80/socket', 'ws'], + ['wss://svc:hunter2@ws.local:443/socket', 'wss'], + ['smtp://app:hunter2@mail.local:25', 'smtp'], + ['smtps://app:hunter2@mail.local:465', 'smtps'], + ['sftp://app:hunter2@host.local:22/dir', 'sftp'], + ['ldap://app:hunter2@dir.local:389', 'ldap'], + ['ldaps://app:hunter2@dir.local:636', 'ldaps'], + ['nats://app:hunter2@nats.local:4222', 'nats'], + ['mariadb://app:hunter2@db.local:3306/orders', 'mariadb'], + ]; + for (const [input, scheme] of cases) { + const out = redactSecrets(input); + expect(out.startsWith(`${scheme}://[REDACTED]:[REDACTED]@`)).toBe(true); + expect(out).not.toContain('hunter2'); + } + }); + + it('redacts user:pass when the scheme is uppercase', () => { + // RFC 3986 declares schemes case-insensitive. HTTPS:// must + // redact the same as https:// or an attacker can bypass the + // pattern by upcasing the scheme. + const out = redactSecrets('clone HTTPS://alice:hunter2@example.com/repo'); + expect(out).toContain('HTTPS://[REDACTED]:[REDACTED]@example.com/repo'); + expect(out).not.toContain('hunter2'); + }); + + it('redacts user:pass when the scheme is mixed case', () => { + const out = redactSecrets( + 'load Postgres://app:hunter2@db.internal/orders', + ); + expect(out).toContain( + 'Postgres://[REDACTED]:[REDACTED]@db.internal/orders', + ); + expect(out).not.toContain('hunter2'); + }); }); describe('AWS access keys', () => { @@ -398,6 +476,43 @@ describe('redactSecrets', () => { const line = 'task: sk-ip-the-rest and sk-line are not tokens'; expect(redactSecrets(line)).toBe(line); }); + + it('redacts a Google Gemini AIza- key', () => { + // Real Gemini keys are exactly 39 chars: the literal "AIza" + // prefix plus 35 chars from the [A-Za-z0-9_-] alphabet. Without + // a Gemini-specific pattern, a key in an observability payload + // would be exfiltrated to the very provider that issued it. + const key = 'AIza' + 'B'.repeat(35); + const out = redactSecrets(`GEMINI_API_KEY=${key}`); + expect(out).toContain('[REDACTED_LLM_KEY]'); + expect(out).not.toContain(key); + }); + + it('redacts a free-floating Gemini key with mixed alphabet chars', () => { + // 4-char "AIza" prefix + 35 chars from [A-Za-z0-9_-]; the full + // key is exactly 39 chars wide. The trailing alphabet here + // exercises every char class the pattern accepts: alphas of + // both cases, digits, "-", and "_". + const tail = 'SyA1b2C3d4-_E5f6G7h8I9j0K1l2M3n4O5p'; + expect(tail.length).toBe(35); + const key = `AIza${tail}`; + const out = redactSecrets(`note: leaked ${key} in pretrigger`); + expect(out).toContain('[REDACTED_LLM_KEY]'); + expect(out).not.toContain(key); + }); + + it('does not match an AIza prefix with fewer than 35 trailing chars', () => { + // The fixed length is what differentiates a real Gemini key from + // an arbitrary string starting with "AIza". 34 chars is too short + // and must not match. + const tooShort = 'AIza' + 'B'.repeat(34); + expect(redactSecrets(tooShort)).toBe(tooShort); + }); + + it('does not match the bare AIza prefix', () => { + const line = 'note about the AIza prefix used by Google'; + expect(redactSecrets(line)).toBe(line); + }); }); describe('multi-secret payloads', () => { diff --git a/packages/api/src/utils/redactSecrets.ts b/packages/api/src/utils/redactSecrets.ts index 263a593ca7..34b64cb8f9 100644 --- a/packages/api/src/utils/redactSecrets.ts +++ b/packages/api/src/utils/redactSecrets.ts @@ -21,23 +21,28 @@ // // Patterns covered: // pem PEM key blocks (-----BEGIN ... PRIVATE KEY-----) -// basic-auth-url scheme://user:pass@host. Schemes: http(s), ftp, -// ssh, postgres(ql), mysql, mariadb, mongodb(+srv), -// redis(s), amqp(s), kafka, clickhouse. +// basic-auth-url scheme://user:pass@host. Schemes: http(s), +// ws(s), ftp, sftp, ssh, postgres(ql), mysql, +// mariadb, mongodb(+srv), mssql, sqlserver, +// snowflake, redis(s), amqp(s), kafka(+ssl), +// clickhouse, smtp(s), ldap(s), nats. Match is +// case-insensitive (RFC 3986 schemes are). // key-value password=secret, api_key=abc // json-quoted {"password":"secret"} and similar // http-header X-Api-Key: abc, Api-Key: abc -// bearer Authorization: Bearer xxx (token alphabet now -// includes "_" so JWT signatures are fully covered). +// bearer Authorization: Bearer xxx. Token is \S+ so +// opaque non-JWT bearers (with ":", "%", or +// quote chars in them) round-trip cleanly. // basic Authorization: Basic xxx // jwt eyJ... three dot-separated base64 segments // aws-access-key AKIA[16 chars], ASIA[16 chars] // slack-token xox[a-z]-... shape // github-token ghp_, gho_, ghu_, ghs_, ghr_ prefixes -// llm-vendor-key sk-... (OpenAI), sk-ant-... (Anthropic). This -// redactor specifically fronts an LLM-provider -// call, so vendor-shaped keys must not leak to -// the very provider that issued them. +// llm-vendor-key sk-... (OpenAI), sk-ant-... (Anthropic), +// AIza... (Google Gemini). This redactor +// specifically fronts an LLM-provider call, so +// vendor-shaped keys must not leak to the very +// provider that issued them. // // Known gaps (extend when seen in production): // URL-percent-encoded values; non-LLM vendor tokens (Stripe, Twilio, @@ -71,24 +76,28 @@ const PATTERNS: RedactionPattern[] = [ }, // scheme://user:pass@host. Password may contain "@"; the engine // backtracks to the last "@" before the host. Host is captured and - // preserved in the replacement. The scheme group covers HTTP-family - // protocols plus the database/queue connection strings most likely - // to land in observability payloads with embedded credentials: - // postgres / mysql / mongodb / redis / amqp / kafka / clickhouse. + // preserved in the replacement. Match is case-insensitive (RFC 3986 + // declares schemes case-insensitive, so HTTPS://user:pw@host is the + // same authority as the lowercase form). The scheme group covers + // HTTP-family protocols plus the database / queue / messaging + // connection strings most likely to land in observability payloads + // with embedded credentials. Schemes listed alphabetically. { name: 'basic-auth-url', - re: /\b(https?|ftp|ssh|postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|rediss?|amqps?|kafka|clickhouse):\/\/([^/\s:@]+):([^/\s]+)@([^/\s@]+)/g, + re: /\b(amqps?|clickhouse|ftp|https?|kafka(?:\+ssl)?|ldaps?|mariadb|mongodb(?:\+srv)?|mssql|mysql|nats|postgres(?:ql)?|rediss?|sftp|smtps?|snowflake|sqlserver|ssh|wss?):\/\/([^/\s:@]+):([^/\s]+)@([^/\s@]+)/gi, replace: '$1://[REDACTED]:[REDACTED]@$4', }, - // Authorization: Bearer xxx. The token alphabet covers both URL-safe - // base64 ("_-" plus alphanumerics) and the standard base64 alphabet - // ("+/" plus "=" padding) so a JWT bearer ("eyJ..._...sig") and a - // token with literal "+" / "=" both round-trip cleanly. Without "_" - // a JWT terminates at the first underscore inside the signature and - // the trailing bytes leak past the [REDACTED] marker. + // Authorization: Bearer xxx. RFC 6750's b64token alphabet + // (alphanumerics, "-._~+/", optional "=" padding) is a strict + // subset of \S+, and observability payloads regularly carry + // opaque non-JWT bearers with ":", "%", or quote chars in them. + // Anything narrower than \S+ leaks the suffix past [REDACTED]; + // \S+ stops at whitespace, which is what actually terminates a + // bearer token in practice (header line break, log delimiter, + // JSON field boundary). { name: 'bearer', - re: /Bearer\s+[A-Za-z0-9._~+/=_-]+/gi, + re: /Bearer\s+\S+/gi, replace: 'Bearer [REDACTED]', }, // Authorization: Basic xxx (base64 user:pass) @@ -125,15 +134,17 @@ const PATTERNS: RedactionPattern[] = [ replace: '[REDACTED_GITHUB_TOKEN]', }, // LLM-vendor API keys: OpenAI ("sk-..."), Anthropic ("sk-ant-..."), - // and adjacent vendors that follow the "sk-" convention. This - // redactor specifically fronts an LLM call; a leaked vendor key - // would be exfiltrated to the very provider that issued it. - // Floor at 20 chars after the prefix to avoid catching English - // words like "sk-ip-line" or short test fixtures while still - // covering OpenAI's 48+ char and Anthropic's longer formats. + // and adjacent vendors that follow the "sk-" convention; plus + // Google Gemini ("AIza..." with 35 trailing chars, 39 total). + // This redactor specifically fronts an LLM call; a leaked vendor + // key would be exfiltrated to the very provider that issued it. + // The "sk-" branch floors at 20 chars after the prefix to avoid + // catching English words like "sk-ip-line" or short test fixtures + // while still covering OpenAI's 48+ char and Anthropic's longer + // formats. Gemini keys are exactly 39 chars wide. { name: 'llm-vendor-key', - re: /\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, + re: /\b(?:sk-(?:ant-)?[A-Za-z0-9_-]{20,}|AIza[A-Za-z0-9_-]{35})\b/g, replace: '[REDACTED_LLM_KEY]', }, // key=value with secret-ish key. Three value forms: double-quoted,