From 309223a8bca0a8b348ed6f11a9cf673b7fad75d3 Mon Sep 17 00:00:00 2001 From: Rodrigo Barona Date: Fri, 16 Oct 2026 14:52:40 +0100 Subject: [PATCH 1/7] Dato Agent: redact credentials from copied failure diagnostics Provider API keys, OAuth access/refresh tokens, PKCE verifiers and similar fields are replaced with [Redacted] (empty values are kept), and sk-* keys or Bearer tokens embedded in strings are scrubbed. Diagnostics are commonly pasted into issues and chats, so they must not carry live credentials. Co-authored-by: Cursor --- .../src/entrypoints/AgentFrame.test.tsx | 10 ++- dato-agent/src/lib/diagnostics.test.ts | 57 ++++++++++++++++- dato-agent/src/lib/diagnostics.ts | 61 +++++++++++++++++-- 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/dato-agent/src/entrypoints/AgentFrame.test.tsx b/dato-agent/src/entrypoints/AgentFrame.test.tsx index 03f8cf89..1aaffb91 100644 --- a/dato-agent/src/entrypoints/AgentFrame.test.tsx +++ b/dato-agent/src/entrypoints/AgentFrame.test.tsx @@ -6708,11 +6708,15 @@ describe('AgentFrame', () => { expect(writeText).toHaveBeenCalledOnce(); const diagnostics = JSON.parse(String(writeText.mock.calls[0]?.[0])); - expect(diagnostics.configuration.openAiApiKey).toBe('sk-diagnostic-secret'); - expect(diagnostics.configuration.activeApiKey).toBe('sk-diagnostic-secret'); - expect(diagnostics.oauthCredentials.token.accessToken).toBe( + expect(diagnostics.configuration.openAiApiKey).toBe('[Redacted]'); + expect(diagnostics.configuration.activeApiKey).toBe('[Redacted]'); + expect(diagnostics.oauthCredentials.token.accessToken).toBe('[Redacted]'); + expect(String(writeText.mock.calls[0]?.[0])).not.toContain( 'diagnostic-oauth-secret', ); + expect(String(writeText.mock.calls[0]?.[0])).not.toContain( + 'sk-diagnostic-secret', + ); expect(diagnostics.turn.message).toBe( 'Find the private record with all of its content', ); diff --git a/dato-agent/src/lib/diagnostics.test.ts b/dato-agent/src/lib/diagnostics.test.ts index 6617bcf6..a425b346 100644 --- a/dato-agent/src/lib/diagnostics.test.ts +++ b/dato-agent/src/lib/diagnostics.test.ts @@ -28,7 +28,7 @@ afterEach(() => { }); describe('serializeDiagnostics', () => { - it('produces deterministic pretty JSON while preserving raw diagnostic values', () => { + it('produces deterministic pretty JSON with credentials redacted', () => { const first = { token: 'dato_oauth_secret', schemaVersion: DIAGNOSTICS_SCHEMA_VERSION, @@ -46,15 +46,66 @@ describe('serializeDiagnostics', () => { expect(serializeDiagnostics(first)).toBe(`{ "arguments": { "a": false, - "apiKey": "sk-secret", + "apiKey": "[Redacted]", "z": true }, "prompt": "Change record 178178740", "schemaVersion": 1, - "token": "dato_oauth_secret" + "token": "[Redacted]" }`); }); + it('redacts provider keys and OAuth credentials while keeping non-secret settings', () => { + const serialized = serializeDiagnostics({ + configuration: { + anthropicApiKey: 'sk-ant-api03-abcdefghijklmnopqrstuvwxyz', + openaiApiKey: '', + activeApiKey: 'sk-ant-api03-abcdefghijklmnopqrstuvwxyz', + anthropicModelMaxOutputTokens: 128_000, + oauthCredentials: { + client: { clientId: 'client-123' }, + token: { + accessToken: 'dato-access-token-value', + refreshToken: 'dato-refresh-token-value', + expiresAt: 1_800_000_000, + }, + pending: { codeVerifier: 'pkce-verifier-value', state: 'abc' }, + }, + }, + turn: { mcpAccessToken: 'dato-access-token-value' }, + error: { + message: + 'Request failed: Authorization: Bearer dato-access-token-value with key sk-proj-abcdefghijklmnopqrstuvwxyz', + }, + }); + + expect(serialized).not.toContain('abcdefghijklmnopqrstuvwxyz'); + expect(serialized).not.toContain('dato-access-token-value'); + expect(serialized).not.toContain('dato-refresh-token-value'); + expect(serialized).not.toContain('pkce-verifier-value'); + expect(JSON.parse(serialized)).toMatchObject({ + configuration: { + anthropicApiKey: '[Redacted]', + openaiApiKey: '', + activeApiKey: '[Redacted]', + anthropicModelMaxOutputTokens: 128_000, + oauthCredentials: { + client: { clientId: 'client-123' }, + token: { + accessToken: '[Redacted]', + refreshToken: '[Redacted]', + expiresAt: 1_800_000_000, + }, + pending: { codeVerifier: '[Redacted]', state: 'abc' }, + }, + }, + error: { + message: + 'Request failed: Authorization: Bearer [Redacted] with key [Redacted]', + }, + }); + }); + it('handles errors, BigInt, Map, Set, circular references, and unsupported values', () => { const cause = new Error('Provider request failed'); const error = new Error('Turn failed', { cause }); diff --git a/dato-agent/src/lib/diagnostics.ts b/dato-agent/src/lib/diagnostics.ts index 6ec9610d..6d7b2313 100644 --- a/dato-agent/src/lib/diagnostics.ts +++ b/dato-agent/src/lib/diagnostics.ts @@ -50,6 +50,45 @@ function describeSymbol(value: symbol): string { } } +const REDACTED = '[Redacted]'; + +const SENSITIVE_EXACT_KEYS = new Set([ + 'authorization', + 'bearer', + 'codeverifier', + 'cookie', + 'idtoken', + 'token', +]); + +const SENSITIVE_KEY_SUFFIXES = [ + 'accesstoken', + 'apikey', + 'password', + 'refreshtoken', + 'secret', +]; + +const SENSITIVE_VALUE_PATTERNS: ReadonlyArray<[RegExp, string]> = [ + [/\bsk-[A-Za-z0-9_-]{16,}/g, REDACTED], + [/\b(Bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, `$1 ${REDACTED}`], +]; + +function isSensitiveKey(key: string): boolean { + const normalized = key.toLowerCase().replace(/[^a-z]/g, ''); + return ( + SENSITIVE_EXACT_KEYS.has(normalized) || + SENSITIVE_KEY_SUFFIXES.some((suffix) => normalized.endsWith(suffix)) + ); +} + +function redactSensitiveSubstrings(value: string): string { + return SENSITIVE_VALUE_PATTERNS.reduce( + (current, [pattern, replacement]) => current.replace(pattern, replacement), + value, + ); +} + function propertyPath(parentPath: string, key: string): string { return /^[A-Za-z_$][\w$]*$/.test(key) ? `${parentPath}.${key}` @@ -59,6 +98,17 @@ function propertyPath(parentPath: string, key: string): string { function normalizeDiagnostics(value: unknown): JsonSafeValue { const seen = new WeakMap(); + const normalizeProperty = ( + key: string, + propertyValue: unknown, + path: string, + ): JsonSafeValue => + typeof propertyValue === 'string' && + propertyValue !== '' && + isSensitiveKey(key) + ? REDACTED + : normalize(propertyValue, propertyPath(path, key)); + // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Diagnostics must safely normalize every JavaScript value shape in one recursive traversal. const normalize = (current: unknown, path: string): JsonSafeValue => { try { @@ -68,6 +118,7 @@ function normalizeDiagnostics(value: unknown): JsonSafeValue { switch (typeof current) { case 'string': + return redactSensitiveSubstrings(current); case 'boolean': return current; case 'number': @@ -142,9 +193,10 @@ function normalizeDiagnostics(value: unknown): JsonSafeValue { } try { - normalizedError[key] = normalize( + normalizedError[key] = normalizeProperty( + key, (current as unknown as Record)[key], - propertyPath(path, key), + path, ); } catch (error) { normalizedError[key] = `[Unserializable: ${errorMessage(error)}]`; @@ -182,9 +234,10 @@ function normalizeDiagnostics(value: unknown): JsonSafeValue { const normalizedObject: Record = {}; for (const key of Object.keys(current).sort(compareStrings)) { try { - normalizedObject[key] = normalize( + normalizedObject[key] = normalizeProperty( + key, (current as Record)[key], - propertyPath(path, key), + path, ); } catch (error) { normalizedObject[key] = `[Unserializable: ${errorMessage(error)}]`; From 22b0583d1ed69c4ee57e9dbdd520e99ed31f8d86 Mon Sep 17 00:00:00 2001 From: "Roger (DatoCMS)" Date: Tue, 3 Nov 2026 14:11:38 -0800 Subject: [PATCH 2/7] Dato Agent: also redact credentials in errors, maps, URLs and other key formats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copied diagnostics still leaked credentials through several paths: - Error message and stack went out raw. - Map entries skipped the key-based rule. - Query-string secrets (?access_token=, ?code=, ?key=) and api_key=… text were not matched. - Gemini (AIza…), Vercel AI Gateway (vck_…) and DatoCMS CMA tokens were not matched. Redact all of these, treat credentials/jwt/session/privateKey keys as sensitive, and use the plugin's existing "[redacted]" marker and 12-char sk- minimum. Harmless text such as "status code=401" is kept. Co-Authored-By: Claude Opus 5.5 --- .../src/entrypoints/AgentFrame.test.tsx | 6 +-- dato-agent/src/lib/diagnostics.test.ts | 52 ++++++++++++++++--- dato-agent/src/lib/diagnostics.ts | 34 ++++++++++-- 3 files changed, 76 insertions(+), 16 deletions(-) diff --git a/dato-agent/src/entrypoints/AgentFrame.test.tsx b/dato-agent/src/entrypoints/AgentFrame.test.tsx index 1aaffb91..a9df6870 100644 --- a/dato-agent/src/entrypoints/AgentFrame.test.tsx +++ b/dato-agent/src/entrypoints/AgentFrame.test.tsx @@ -6708,9 +6708,9 @@ describe('AgentFrame', () => { expect(writeText).toHaveBeenCalledOnce(); const diagnostics = JSON.parse(String(writeText.mock.calls[0]?.[0])); - expect(diagnostics.configuration.openAiApiKey).toBe('[Redacted]'); - expect(diagnostics.configuration.activeApiKey).toBe('[Redacted]'); - expect(diagnostics.oauthCredentials.token.accessToken).toBe('[Redacted]'); + expect(diagnostics.configuration.openAiApiKey).toBe('[redacted]'); + expect(diagnostics.configuration.activeApiKey).toBe('[redacted]'); + expect(diagnostics.oauthCredentials.token.accessToken).toBe('[redacted]'); expect(String(writeText.mock.calls[0]?.[0])).not.toContain( 'diagnostic-oauth-secret', ); diff --git a/dato-agent/src/lib/diagnostics.test.ts b/dato-agent/src/lib/diagnostics.test.ts index a425b346..c290b058 100644 --- a/dato-agent/src/lib/diagnostics.test.ts +++ b/dato-agent/src/lib/diagnostics.test.ts @@ -46,12 +46,12 @@ describe('serializeDiagnostics', () => { expect(serializeDiagnostics(first)).toBe(`{ "arguments": { "a": false, - "apiKey": "[Redacted]", + "apiKey": "[redacted]", "z": true }, "prompt": "Change record 178178740", "schemaVersion": 1, - "token": "[Redacted]" + "token": "[redacted]" }`); }); @@ -85,23 +85,23 @@ describe('serializeDiagnostics', () => { expect(serialized).not.toContain('pkce-verifier-value'); expect(JSON.parse(serialized)).toMatchObject({ configuration: { - anthropicApiKey: '[Redacted]', + anthropicApiKey: '[redacted]', openaiApiKey: '', - activeApiKey: '[Redacted]', + activeApiKey: '[redacted]', anthropicModelMaxOutputTokens: 128_000, oauthCredentials: { client: { clientId: 'client-123' }, token: { - accessToken: '[Redacted]', - refreshToken: '[Redacted]', + accessToken: '[redacted]', + refreshToken: '[redacted]', expiresAt: 1_800_000_000, }, - pending: { codeVerifier: '[Redacted]', state: 'abc' }, + pending: { codeVerifier: '[redacted]', state: 'abc' }, }, }, error: { message: - 'Request failed: Authorization: Bearer [Redacted] with key [Redacted]', + 'Request failed: Authorization: Bearer [redacted] with key [redacted]', }, }); }); @@ -155,6 +155,42 @@ describe('serializeDiagnostics', () => { }); }); + it('redacts credentials inside real errors, maps, URLs and other providers’ key formats', () => { + // Fake values shaped like real credentials. + const secrets = { + anthropic: 'sk-ant-api03-FAKEFAKEFAKEFAKE', + bearer: 'FAKEtokenFAKEtokenFAKE', + gemini: 'AIzaSyFAKEFAKEFAKEFAKEFAKEFAKE1234', + vercel: 'vck_FAKEFAKEFAKEFAKEFAKE', + cma: 'f00dfacef00dfacef00dfacef00dfa', + oauth: 'FAKEoauthFAKEoauth', + code: 'FAKEauthorizationCODE', + }; + const error = new Error(`401 for ${secrets.anthropic}`); + error.stack = `Error: boom\n Authorization: Bearer ${secrets.bearer}`; + + const output = serializeDiagnostics({ + thrownError: error, + context: new Map([['apiKey', secrets.cma]]), + request: { + url: `https://example.com/cb?code=${secrets.code}&access_token=${secrets.oauth}`, + }, + notes: [ + `invalid key ${secrets.gemini}`, + `gateway rejected ${secrets.vercel}`, + `api_key=${secrets.oauth}`, + ], + credentials: 'anything', + status: 'status code=401 kept', + }); + + for (const secret of Object.values(secrets)) + expect(output).not.toContain(secret); + expect(output).toContain('status code=401 kept'); + expect(output).toContain('?code=[redacted]&access_token=[redacted]'); + expect(output).toContain('"message": "401 for [redacted]"'); + }); + it('does not throw for values whose properties cannot be inspected', () => { const { proxy, revoke } = Proxy.revocable({}, {}); revoke(); diff --git a/dato-agent/src/lib/diagnostics.ts b/dato-agent/src/lib/diagnostics.ts index 6d7b2313..1b527aba 100644 --- a/dato-agent/src/lib/diagnostics.ts +++ b/dato-agent/src/lib/diagnostics.ts @@ -50,14 +50,19 @@ function describeSymbol(value: symbol): string { } } -const REDACTED = '[Redacted]'; +// Same marker as the plugin's other redactors (conversations, MCP client, script outcomes). +const REDACTED = '[redacted]'; const SENSITIVE_EXACT_KEYS = new Set([ 'authorization', 'bearer', 'codeverifier', 'cookie', + 'credentials', 'idtoken', + 'jwt', + 'session', + 'sessionid', 'token', ]); @@ -65,13 +70,29 @@ const SENSITIVE_KEY_SUFFIXES = [ 'accesstoken', 'apikey', 'password', + 'privatekey', 'refreshtoken', 'secret', ]; const SENSITIVE_VALUE_PATTERNS: ReadonlyArray<[RegExp, string]> = [ - [/\bsk-[A-Za-z0-9_-]{16,}/g, REDACTED], + // OpenAI and Anthropic keys; 12+ matches the plugin's other redactors. + [/\bsk-[A-Za-z0-9_-]{12,}/g, REDACTED], + [/\bAIza[0-9A-Za-z_-]{20,}/g, REDACTED], + [/\bvck_[A-Za-z0-9_-]{16,}/g, REDACTED], + // DatoCMS CMA tokens are 30 lowercase hex characters. + [/\b[a-f0-9]{30}\b/g, REDACTED], [/\b(Bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, `$1 ${REDACTED}`], + // Query-string secrets, e.g. ?access_token=…, ?code=… (OAuth) or ?key=… (Gemini). + [ + /([?&])((?:access|refresh|id)_token|token|api_?key|key|client_secret|code_verifier|code)=[^&#\s"']+/gi, + `$1$2=${REDACTED}`, + ], + // Unambiguous key=value secrets in free text, e.g. "api_key=…". + [ + /\b((?:access|refresh|id)_token|api_?key|client_secret)=[^&#\s"']+/gi, + `$1=${REDACTED}`, + ], ]; function isSensitiveKey(key: string): boolean { @@ -161,14 +182,15 @@ function normalizeDiagnostics(value: unknown): JsonSafeValue { } if (current instanceof Error) { + // Error messages and stacks can echo request URLs, headers or keys. const normalizedError: Record = { $type: 'Error', name: current.name, - message: current.message, + message: redactSensitiveSubstrings(current.message), }; if (current.stack !== undefined) { - normalizedError.stack = current.stack; + normalizedError.stack = redactSensitiveSubstrings(current.stack); } if ('cause' in current) { @@ -211,7 +233,9 @@ function normalizeDiagnostics(value: unknown): JsonSafeValue { $type: 'Map', entries: Array.from(current.entries(), ([key, entryValue], index) => [ normalize(key, `${path}.entries[${index}][0]`), - normalize(entryValue, `${path}.entries[${index}][1]`), + typeof key === 'string' + ? normalizeProperty(key, entryValue, `${path}.entries[${index}]`) + : normalize(entryValue, `${path}.entries[${index}][1]`), ]), }; } From 21ba433e9f584a7ad471d6c372ed73ec62d86e03 Mon Sep 17 00:00:00 2001 From: "Roger (DatoCMS)" Date: Tue, 3 Nov 2026 14:27:04 -0800 Subject: [PATCH 3/7] Dato Agent: build diagnostics from an allowlist and mask known secrets Replace the secret-shape regexes with two structural guarantees: - Allowlist: diagnostics list the non-secret configuration fields explicitly (diagnosticConfiguration) and report API keys only as hasOpenAiApiKey/hasAnthropicApiKey. OAuth state is reduced to version, client registration and token metadata (diagnosticOAuthCredentials); the access token is never included. New secret fields can't leak by default. - Known-value masking: serializeDiagnostics takes the frame's live secrets (both provider keys and the OAuth access token) and replaces every exact occurrence anywhere in the payload, including error messages, stacks, URLs and map values, whatever their format. The credential-key list stays as a structural backstop for nested objects such as headers.authorization. Co-Authored-By: Claude Opus 5.5 --- .../src/entrypoints/AgentFrame.test.tsx | 9 +- dato-agent/src/entrypoints/AgentFrame.tsx | 178 ++++++++++-------- dato-agent/src/lib/config.ts | 24 +++ dato-agent/src/lib/credentials.ts | 20 ++ dato-agent/src/lib/diagnostics.test.ts | 105 ++++++----- dato-agent/src/lib/diagnostics.ts | 82 ++++---- 6 files changed, 253 insertions(+), 165 deletions(-) diff --git a/dato-agent/src/entrypoints/AgentFrame.test.tsx b/dato-agent/src/entrypoints/AgentFrame.test.tsx index a9df6870..c5797f1d 100644 --- a/dato-agent/src/entrypoints/AgentFrame.test.tsx +++ b/dato-agent/src/entrypoints/AgentFrame.test.tsx @@ -6708,9 +6708,12 @@ describe('AgentFrame', () => { expect(writeText).toHaveBeenCalledOnce(); const diagnostics = JSON.parse(String(writeText.mock.calls[0]?.[0])); - expect(diagnostics.configuration.openAiApiKey).toBe('[redacted]'); - expect(diagnostics.configuration.activeApiKey).toBe('[redacted]'); - expect(diagnostics.oauthCredentials.token.accessToken).toBe('[redacted]'); + // Allowlisted payload: keys and tokens are absent, not just masked. + expect(diagnostics.configuration.openAiApiKey).toBeUndefined(); + expect(diagnostics.configuration.activeApiKey).toBeUndefined(); + expect(diagnostics.configuration.hasOpenAiApiKey).toBe(true); + expect(diagnostics.oauthCredentials.token.accessToken).toBeUndefined(); + expect(diagnostics.oauthCredentials.token.tokenType).toBe('Bearer'); expect(String(writeText.mock.calls[0]?.[0])).not.toContain( 'diagnostic-oauth-secret', ); diff --git a/dato-agent/src/entrypoints/AgentFrame.tsx b/dato-agent/src/entrypoints/AgentFrame.tsx index d739546b..fa2610dd 100644 --- a/dato-agent/src/entrypoints/AgentFrame.tsx +++ b/dato-agent/src/entrypoints/AgentFrame.tsx @@ -53,6 +53,7 @@ import { activeModel, activeModelMaxOutputTokens, activeReasoningEffort, + diagnosticConfiguration, providerLabel, } from '../lib/config'; import { @@ -71,6 +72,7 @@ import { type CredentialScope, createCredentialStore, createOAuthCredentials, + diagnosticOAuthCredentials, invalidateOAuthToken, isOAuthClientRegistrationReusable, type OAuthCredentials, @@ -3200,90 +3202,102 @@ export default function AgentFrame(props: AgentFrameProps) { const systemContext = runtimeSystemContext(); failureDiagnosticsRef.current.set( failureId, - serializeDiagnostics({ - schemaVersion: DIAGNOSTICS_SCHEMA_VERSION, - capturedAt, - plugin: { - name: 'Dato Agent (Beta)', - package: 'datocms-plugin-dato-agent', - version: '0.1.0', - url: window.location.href, - }, - browser: { - userAgent: navigator.userAgent, - language: navigator.language, - online: navigator.onLine, - }, - project: { - pluginId: props.pluginId, - siteId: props.siteId, - siteName: props.siteName, - environment: props.environment, - isEnvironmentPrimary: props.isEnvironmentPrimary, - currentUserId: props.currentUserId, - scope: stableScope, - currentRecord: props.currentRecord, - editorHasUnsavedChanges: editorDirtyRef.current, - }, - configuration: { - ...props.config, - activeApiKey: activeApiKey(props.config), - activeModel: activeModel(props.config), - activeModelMaxOutputTokens: activeModelMaxOutputTokens(props.config), - activeReasoningEffort: activeReasoningEffort(props.config), - autoApproveEnabled: autoApproveEnabledRef.current, - }, - oauthCredentials, - conversation: { - beforeTurn: turn.conversationBefore, - current: conversationRef.current, - providerHistory: turn.history, + serializeDiagnostics( + { + schemaVersion: DIAGNOSTICS_SCHEMA_VERSION, + capturedAt, + plugin: { + name: 'Dato Agent (Beta)', + package: 'datocms-plugin-dato-agent', + version: '0.1.0', + url: window.location.href, + }, + browser: { + userAgent: navigator.userAgent, + language: navigator.language, + online: navigator.onLine, + }, + project: { + pluginId: props.pluginId, + siteId: props.siteId, + siteName: props.siteName, + environment: props.environment, + isEnvironmentPrimary: props.isEnvironmentPrimary, + currentUserId: props.currentUserId, + scope: stableScope, + currentRecord: props.currentRecord, + editorHasUnsavedChanges: editorDirtyRef.current, + }, + configuration: { + ...diagnosticConfiguration(props.config), + activeModel: activeModel(props.config), + activeModelMaxOutputTokens: activeModelMaxOutputTokens( + props.config, + ), + activeReasoningEffort: activeReasoningEffort(props.config), + autoApproveEnabled: autoApproveEnabledRef.current, + }, + oauthCredentials: diagnosticOAuthCredentials(oauthCredentials), + conversation: { + beforeTurn: turn.conversationBefore, + current: conversationRef.current, + providerHistory: turn.history, + }, + turn: { + id: turn.id, + source, + message: turn.message, + userEntryId: turn.userEntryId, + assistantEntryId: turn.assistantEntryId, + activityEntryId: turn.activityEntryId, + provider: turn.provider, + model: turn.model, + startedAt: turn.startedAt, + finishedAt: capturedAt, + durationMs: Math.max( + 0, + Date.parse(capturedAt) - Date.parse(turn.startedAt), + ), + previousResponseId: turn.previousResponseId, + injectHostContext: turn.injectHostContext, + textDeltaCount: turn.textDeltaCount, + textDeltaCharacters: turn.textDeltaCharacters, + diagnosticEventOutputCharacters: + turn.diagnosticEventOutputCharacters, + diagnosticEventsDropped: turn.diagnosticEventsDropped, + hostContexts: turn.hostContexts, + userStopped: turn.userStopped, + unsafeOperationDispatched: turn.unsafeOperationDispatched, + dispatchedApprovalIds: turn.dispatchedApprovalIds, + cancelledBeforeDispatchApprovalIds: + turn.cancelledBeforeDispatchApprovalIds, + cancelledBeforeDispatchReason: turn.cancelledBeforeDispatchReason, + autoApprovalBundleCount: turn.autoApprovalBundleCount, + approvalSubmissions: turn.approvalSubmissions, + approvalOutcomes: turn.approvalOutcomes, + confirmedApprovalIds: turn.confirmedApprovalIds, + completionResult: result, + events: turn.events, + thrownError, + systemPrompt: buildSystemPrompt(systemContext, { + additionalInstructions: props.config.additionalInstructions, + readOnly: props.config.readOnly, + mcpAccessLevel: accessRef.current.snapshot.level, + }), + }, + transcript: entriesRef.current, + pendingApprovals: [...pendingApprovalsRef.current.values()], + pendingNavigation: pendingNavigationRef.current, }, - turn: { - id: turn.id, - source, - message: turn.message, - userEntryId: turn.userEntryId, - assistantEntryId: turn.assistantEntryId, - activityEntryId: turn.activityEntryId, - provider: turn.provider, - model: turn.model, - startedAt: turn.startedAt, - finishedAt: capturedAt, - durationMs: Math.max( - 0, - Date.parse(capturedAt) - Date.parse(turn.startedAt), - ), - previousResponseId: turn.previousResponseId, - injectHostContext: turn.injectHostContext, - textDeltaCount: turn.textDeltaCount, - textDeltaCharacters: turn.textDeltaCharacters, - diagnosticEventOutputCharacters: turn.diagnosticEventOutputCharacters, - diagnosticEventsDropped: turn.diagnosticEventsDropped, - hostContexts: turn.hostContexts, - userStopped: turn.userStopped, - unsafeOperationDispatched: turn.unsafeOperationDispatched, - dispatchedApprovalIds: turn.dispatchedApprovalIds, - cancelledBeforeDispatchApprovalIds: - turn.cancelledBeforeDispatchApprovalIds, - cancelledBeforeDispatchReason: turn.cancelledBeforeDispatchReason, - autoApprovalBundleCount: turn.autoApprovalBundleCount, - approvalSubmissions: turn.approvalSubmissions, - approvalOutcomes: turn.approvalOutcomes, - confirmedApprovalIds: turn.confirmedApprovalIds, - completionResult: result, - events: turn.events, - thrownError, - systemPrompt: buildSystemPrompt(systemContext, { - additionalInstructions: props.config.additionalInstructions, - readOnly: props.config.readOnly, - mcpAccessLevel: accessRef.current.snapshot.level, - }), + { + // Every live secret this frame holds; masked wherever it appears in the payload. + secrets: [ + props.config.openAiApiKey, + props.config.anthropicApiKey, + oauthCredentials?.token?.accessToken, + ], }, - transcript: entriesRef.current, - pendingApprovals: [...pendingApprovalsRef.current.values()], - pendingNavigation: pendingNavigationRef.current, - }), + ), ); retryCandidateRef.current = retryable diff --git a/dato-agent/src/lib/config.ts b/dato-agent/src/lib/config.ts index 2dbf21d9..73aa5c84 100644 --- a/dato-agent/src/lib/config.ts +++ b/dato-agent/src/lib/config.ts @@ -296,3 +296,27 @@ export function serializeConfig( ...normalizeConfig(next), }; } + +/** + * The configuration as it may appear in copied diagnostics: an explicit allowlist of non-secret + * settings. API keys are reported only as present or missing, so a new secret field never leaks + * by default. + */ +export function diagnosticConfiguration(config: AgentConfig) { + return { + provider: config.provider, + model: config.model, + reasoningEffort: config.reasoningEffort, + openAiFastMode: config.openAiFastMode, + anthropicModel: config.anthropicModel, + anthropicModelMaxOutputTokens: config.anthropicModelMaxOutputTokens, + anthropicReasoningEffort: config.anthropicReasoningEffort, + anthropicFastMode: config.anthropicFastMode, + additionalInstructions: config.additionalInstructions, + readOnly: config.readOnly, + enableRecordSidebar: config.enableRecordSidebar, + allowedRoleIds: config.allowedRoleIds, + hasOpenAiApiKey: config.openAiApiKey.trim() !== '', + hasAnthropicApiKey: config.anthropicApiKey.trim() !== '', + }; +} diff --git a/dato-agent/src/lib/credentials.ts b/dato-agent/src/lib/credentials.ts index 9497743f..5b9b73fb 100644 --- a/dato-agent/src/lib/credentials.ts +++ b/dato-agent/src/lib/credentials.ts @@ -363,3 +363,23 @@ function isCleanAbsoluteUrl(value: unknown): value is string { return false; } } + +/** + * OAuth state as it may appear in copied diagnostics: version, client registration and token + * metadata only. The access token itself is never included. + */ +export function diagnosticOAuthCredentials( + credentials: OAuthCredentials | null | undefined, +) { + if (!credentials) return undefined; + return { + version: credentials.version, + client: credentials.client, + token: credentials.token + ? { + tokenType: credentials.token.tokenType, + obtainedAt: credentials.token.obtainedAt, + } + : undefined, + }; +} diff --git a/dato-agent/src/lib/diagnostics.test.ts b/dato-agent/src/lib/diagnostics.test.ts index c290b058..f4508a9a 100644 --- a/dato-agent/src/lib/diagnostics.test.ts +++ b/dato-agent/src/lib/diagnostics.test.ts @@ -56,28 +56,37 @@ describe('serializeDiagnostics', () => { }); it('redacts provider keys and OAuth credentials while keeping non-secret settings', () => { - const serialized = serializeDiagnostics({ - configuration: { - anthropicApiKey: 'sk-ant-api03-abcdefghijklmnopqrstuvwxyz', - openaiApiKey: '', - activeApiKey: 'sk-ant-api03-abcdefghijklmnopqrstuvwxyz', - anthropicModelMaxOutputTokens: 128_000, - oauthCredentials: { - client: { clientId: 'client-123' }, - token: { - accessToken: 'dato-access-token-value', - refreshToken: 'dato-refresh-token-value', - expiresAt: 1_800_000_000, + const serialized = serializeDiagnostics( + { + configuration: { + anthropicApiKey: 'sk-ant-api03-abcdefghijklmnopqrstuvwxyz', + openaiApiKey: '', + activeApiKey: 'sk-ant-api03-abcdefghijklmnopqrstuvwxyz', + anthropicModelMaxOutputTokens: 128_000, + oauthCredentials: { + client: { clientId: 'client-123' }, + token: { + accessToken: 'dato-access-token-value', + refreshToken: 'dato-refresh-token-value', + expiresAt: 1_800_000_000, + }, + pending: { codeVerifier: 'pkce-verifier-value', state: 'abc' }, }, - pending: { codeVerifier: 'pkce-verifier-value', state: 'abc' }, + }, + turn: { mcpAccessToken: 'dato-access-token-value' }, + error: { + message: + 'Request failed: Authorization: Bearer dato-access-token-value with key sk-proj-abcdefghijklmnopqrstuvwxyz', }, }, - turn: { mcpAccessToken: 'dato-access-token-value' }, - error: { - message: - 'Request failed: Authorization: Bearer dato-access-token-value with key sk-proj-abcdefghijklmnopqrstuvwxyz', + { + secrets: [ + 'sk-ant-api03-abcdefghijklmnopqrstuvwxyz', + 'sk-proj-abcdefghijklmnopqrstuvwxyz', + 'dato-access-token-value', + ], }, - }); + ); expect(serialized).not.toContain('abcdefghijklmnopqrstuvwxyz'); expect(serialized).not.toContain('dato-access-token-value'); @@ -155,40 +164,40 @@ describe('serializeDiagnostics', () => { }); }); - it('redacts credentials inside real errors, maps, URLs and other providers’ key formats', () => { - // Fake values shaped like real credentials. - const secrets = { - anthropic: 'sk-ant-api03-FAKEFAKEFAKEFAKE', - bearer: 'FAKEtokenFAKEtokenFAKE', - gemini: 'AIzaSyFAKEFAKEFAKEFAKEFAKEFAKE1234', - vercel: 'vck_FAKEFAKEFAKEFAKEFAKE', - cma: 'f00dfacef00dfacef00dfacef00dfa', - oauth: 'FAKEoauthFAKEoauth', - code: 'FAKEauthorizationCODE', - }; - const error = new Error(`401 for ${secrets.anthropic}`); - error.stack = `Error: boom\n Authorization: Bearer ${secrets.bearer}`; + it('masks known secret values wherever they appear, whatever their format', () => { + // Deliberately formatless: masking must not depend on recognising a key shape. + const providerKey = 'not-a-known-shape-0123456789'; + const accessToken = 'opaque.token.value~42'; + const error = new Error(`401 for ${providerKey}`); + error.stack = `Error: boom\n Authorization: Bearer ${accessToken}`; - const output = serializeDiagnostics({ - thrownError: error, - context: new Map([['apiKey', secrets.cma]]), - request: { - url: `https://example.com/cb?code=${secrets.code}&access_token=${secrets.oauth}`, + const output = serializeDiagnostics( + { + thrownError: error, + context: new Map([['notASecretKeyName', providerKey]]), + request: { + url: `https://example.com/cb?state=x&access_token=${accessToken}`, + }, + notes: [`the user pasted ${providerKey} into chat`], + status: 'status code=401 kept', }, - notes: [ - `invalid key ${secrets.gemini}`, - `gateway rejected ${secrets.vercel}`, - `api_key=${secrets.oauth}`, - ], - credentials: 'anything', - status: 'status code=401 kept', - }); + { secrets: [providerKey, accessToken, '', undefined, 'short'] }, + ); - for (const secret of Object.values(secrets)) - expect(output).not.toContain(secret); - expect(output).toContain('status code=401 kept'); - expect(output).toContain('?code=[redacted]&access_token=[redacted]'); + expect(output).not.toContain(providerKey); + expect(output).not.toContain(accessToken); expect(output).toContain('"message": "401 for [redacted]"'); + expect(output).toContain('?state=x&access_token=[redacted]'); + expect(output).toContain('status code=401 kept'); + }); + + it('keeps short or empty secret values from masking ordinary text', () => { + expect( + serializeDiagnostics( + { note: 'a short word' }, + { secrets: ['short', ''] }, + ), + ).toContain('a short word'); }); it('does not throw for values whose properties cannot be inspected', () => { diff --git a/dato-agent/src/lib/diagnostics.ts b/dato-agent/src/lib/diagnostics.ts index 1b527aba..213218bb 100644 --- a/dato-agent/src/lib/diagnostics.ts +++ b/dato-agent/src/lib/diagnostics.ts @@ -75,26 +75,6 @@ const SENSITIVE_KEY_SUFFIXES = [ 'secret', ]; -const SENSITIVE_VALUE_PATTERNS: ReadonlyArray<[RegExp, string]> = [ - // OpenAI and Anthropic keys; 12+ matches the plugin's other redactors. - [/\bsk-[A-Za-z0-9_-]{12,}/g, REDACTED], - [/\bAIza[0-9A-Za-z_-]{20,}/g, REDACTED], - [/\bvck_[A-Za-z0-9_-]{16,}/g, REDACTED], - // DatoCMS CMA tokens are 30 lowercase hex characters. - [/\b[a-f0-9]{30}\b/g, REDACTED], - [/\b(Bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, `$1 ${REDACTED}`], - // Query-string secrets, e.g. ?access_token=…, ?code=… (OAuth) or ?key=… (Gemini). - [ - /([?&])((?:access|refresh|id)_token|token|api_?key|key|client_secret|code_verifier|code)=[^&#\s"']+/gi, - `$1$2=${REDACTED}`, - ], - // Unambiguous key=value secrets in free text, e.g. "api_key=…". - [ - /\b((?:access|refresh|id)_token|api_?key|client_secret)=[^&#\s"']+/gi, - `$1=${REDACTED}`, - ], -]; - function isSensitiveKey(key: string): boolean { const normalized = key.toLowerCase().replace(/[^a-z]/g, ''); return ( @@ -103,11 +83,30 @@ function isSensitiveKey(key: string): boolean { ); } -function redactSensitiveSubstrings(value: string): string { - return SENSITIVE_VALUE_PATTERNS.reduce( - (current, [pattern, replacement]) => current.replace(pattern, replacement), - value, - ); +/** Shorter values would mask ordinary words; real keys and tokens are far longer. */ +const MIN_SECRET_LENGTH = 8; + +/** + * Builds a masker for the plugin's own secret values (provider API keys, OAuth tokens). Replacing + * exact values catches them wherever they end up, e.g. echoed inside an error message, whatever + * their format. Longest first, so a secret that contains another is masked whole. + */ +function knownSecretMasker( + secrets: readonly (string | null | undefined)[], +): (text: string) => string { + const values = [ + ...new Set( + secrets.filter( + (secret): secret is string => + typeof secret === 'string' && secret.length >= MIN_SECRET_LENGTH, + ), + ), + ].sort((left, right) => right.length - left.length); + return (text) => + values.reduce( + (current, secret) => current.replaceAll(secret, REDACTED), + text, + ); } function propertyPath(parentPath: string, key: string): string { @@ -116,7 +115,10 @@ function propertyPath(parentPath: string, key: string): string { : `${parentPath}[${JSON.stringify(key)}]`; } -function normalizeDiagnostics(value: unknown): JsonSafeValue { +function normalizeDiagnostics( + value: unknown, + maskSecrets: (text: string) => string, +): JsonSafeValue { const seen = new WeakMap(); const normalizeProperty = ( @@ -139,7 +141,7 @@ function normalizeDiagnostics(value: unknown): JsonSafeValue { switch (typeof current) { case 'string': - return redactSensitiveSubstrings(current); + return maskSecrets(current); case 'boolean': return current; case 'number': @@ -182,15 +184,15 @@ function normalizeDiagnostics(value: unknown): JsonSafeValue { } if (current instanceof Error) { - // Error messages and stacks can echo request URLs, headers or keys. + // Error messages and stacks can echo request headers or keys. const normalizedError: Record = { $type: 'Error', name: current.name, - message: redactSensitiveSubstrings(current.message), + message: maskSecrets(current.message), }; if (current.stack !== undefined) { - normalizedError.stack = redactSensitiveSubstrings(current.stack); + normalizedError.stack = maskSecrets(current.stack); } if ('cause' in current) { @@ -292,8 +294,24 @@ function normalizeDiagnostics(value: unknown): JsonSafeValue { return normalize(value, '$'); } -export function serializeDiagnostics(value: unknown): string { - return JSON.stringify(normalizeDiagnostics(value), null, 2); +export type DiagnosticsOptions = { + /** The plugin's live secret values; every exact occurrence is replaced in the output. */ + secrets?: readonly (string | null | undefined)[]; +}; + +/** + * Serializes failure diagnostics for the clipboard. Callers pass an allowlisted payload plus the + * secrets they hold; values under credential-like keys are also replaced as a structural backstop. + */ +export function serializeDiagnostics( + value: unknown, + { secrets = [] }: DiagnosticsOptions = {}, +): string { + return JSON.stringify( + normalizeDiagnostics(value, knownSecretMasker(secrets)), + null, + 2, + ); } function restoreFocus(element: Element | null): void { From 3f03ee6e2be0b84b0b5f53c82d3c07782f8860d6 Mon Sep 17 00:00:00 2001 From: "Roger (DatoCMS)" Date: Tue, 3 Nov 2026 14:37:49 -0800 Subject: [PATCH 4/7] Dato Agent: ask before copying diagnostics that contain secret-looking values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Known secrets are already masked. For anything else that looks like a credential, such as a key pasted into chat, "Copy diagnostics" now lists the candidates (last 4 characters, length and payload path, never the value) and lets the user choose which to redact. All start selected. Detection is generic, with no provider patterns: tokens of 20+ characters that mix letters and digits and have high entropy. DatoCMS IDs (isValidId from @datocms/cma-client-browser), canonical UUIDs, URL escapes and values under id/…Id/…Ids keys are skipped. With no candidates, copying works as before. Co-Authored-By: Claude Opus 5.5 --- .../src/components/AgentSurface.module.css | 25 +++ .../src/components/AgentSurface.test.tsx | 71 +++++++- dato-agent/src/components/AgentSurface.tsx | 87 ++++++++- dato-agent/src/entrypoints/AgentFrame.tsx | 16 +- dato-agent/src/lib/secretCandidates.test.ts | 80 +++++++++ dato-agent/src/lib/secretCandidates.ts | 168 ++++++++++++++++++ 6 files changed, 441 insertions(+), 6 deletions(-) create mode 100644 dato-agent/src/lib/secretCandidates.test.ts create mode 100644 dato-agent/src/lib/secretCandidates.ts diff --git a/dato-agent/src/components/AgentSurface.module.css b/dato-agent/src/components/AgentSurface.module.css index 860a527a..d17c8a0f 100644 --- a/dato-agent/src/components/AgentSurface.module.css +++ b/dato-agent/src/components/AgentSurface.module.css @@ -644,6 +644,31 @@ line-height: 1.4; } +.diagnosticsReview { + display: flex; + flex-direction: column; + gap: var(--spacing-xs); + margin: var(--spacing-s) 0 0; + padding: var(--spacing-s); + border: 1px solid var(--color--border); + border-radius: var(--border-radius-m, 6px); + font-size: var(--font-size-xs); +} + +.diagnosticsReview legend { + padding: 0 var(--spacing-xs); + color: var(--color--ink); + font-weight: 600; +} + +.diagnosticsReviewItem { + display: flex; + align-items: baseline; + gap: var(--spacing-xs); + color: var(--color--ink-subtle); + word-break: break-all; +} + .activityDisclosure { color: var(--color--ink-subtle); font-size: var(--font-size-s); diff --git a/dato-agent/src/components/AgentSurface.test.tsx b/dato-agent/src/components/AgentSurface.test.tsx index 1f3c3fc2..2bb1930e 100644 --- a/dato-agent/src/components/AgentSurface.test.tsx +++ b/dato-agent/src/components/AgentSurface.test.tsx @@ -1384,6 +1384,72 @@ describe('AgentSurface', () => { ).not.toBeInTheDocument(); }); + it('asks which secret-looking values to redact before copying diagnostics', async () => { + const onCopyFailureDiagnostics = vi + .fn() + .mockResolvedValueOnce({ + status: 'review', + candidates: [ + { + value: 'first-secret-value-0001', + preview: '…0001 · 23 chars', + path: 'conversation[0].text', + occurrences: 1, + }, + { + value: 'second-secret-value-002', + preview: '…0002 · 23 chars', + path: 'turn.events[1].output', + occurrences: 2, + }, + ], + }) + .mockResolvedValueOnce({ status: 'copied' }); + const user = userEvent.setup(); + + render( + , + ); + + await user.click(screen.getByRole('button', { name: 'Copy diagnostics' })); + expect( + screen.getByText(/2 values in the diagnostics look like secrets/), + ).toBeInTheDocument(); + expect(screen.getByText('…0002 · 23 chars')).toBeInTheDocument(); + expect( + screen.queryByText('second-secret-value-002'), + ).not.toBeInTheDocument(); + + // Everything starts selected; untick one, then copy. + const checkboxes = screen.getAllByRole('checkbox'); + expect( + checkboxes.every((checkbox) => (checkbox as HTMLInputElement).checked), + ).toBe(true); + await user.click(checkboxes[1] as HTMLElement); + await user.click( + screen.getByRole('button', { name: 'Copy with 1 redacted' }), + ); + + expect(onCopyFailureDiagnostics).toHaveBeenLastCalledWith('failure-1', [ + 'first-secret-value-0001', + ]); + expect(screen.queryByText(/look like secrets/)).not.toBeInTheDocument(); + }); + it('offers compact recovery actions only for a terminal assistant failure', async () => { const onRetryFailedTurn = vi.fn(); const onCopyFailureDiagnostics = vi.fn().mockResolvedValue(undefined); @@ -1442,7 +1508,10 @@ describe('AgentSurface', () => { await user.click(copyButton); expect(onCopyFailureDiagnostics).toHaveBeenCalledOnce(); - expect(onCopyFailureDiagnostics).toHaveBeenCalledWith('failure-1'); + expect(onCopyFailureDiagnostics).toHaveBeenCalledWith( + 'failure-1', + undefined, + ); }); it('shows only diagnostics for a non-retryable terminal failure', () => { diff --git a/dato-agent/src/components/AgentSurface.tsx b/dato-agent/src/components/AgentSurface.tsx index 416398b4..d5f53c08 100644 --- a/dato-agent/src/components/AgentSurface.tsx +++ b/dato-agent/src/components/AgentSurface.tsx @@ -31,6 +31,7 @@ import type { Mention, } from '../lib/mentions'; import { fallbackAssetMention, fallbackRecordMention } from '../lib/mentions'; +import type { SecretCandidate } from '../lib/secretCandidates'; import { MentionDisplay } from '../recordComments/entrypoints/components/shared/MentionDisplay'; import commentStyles from '../recordComments/entrypoints/styles/comment.module.css'; import { @@ -313,7 +314,14 @@ export type AgentSurfaceProps = { onRejectUnsafeAction?: (approval: UnsafeApprovalViewModel) => void; onRepairUnsafeAction?: (approval: UnsafeApprovalViewModel) => void; onRetryFailedTurn?: (failureId: string) => void | Promise; - onCopyFailureDiagnostics?: (failureId: string) => Promise; + /** + * Copies a failure's diagnostics. Without `redact`, it may return `review` with values that look + * like secrets instead of copying; calling again with the values to redact (possibly none) copies. + */ + onCopyFailureDiagnostics?: ( + failureId: string, + redact?: readonly string[], + ) => Promise; autoApproveEnabled?: boolean; autoApproveChanging?: boolean; autoApproveError?: string; @@ -695,6 +703,10 @@ function EmptyState() { ); } +export type CopyDiagnosticsResult = + | { status: 'copied' } + | { status: 'review'; candidates: readonly SecretCandidate[] }; + function FailureActions({ failure, onRetry, @@ -708,6 +720,9 @@ function FailureActions({ const [copyState, setCopyState] = useState< 'idle' | 'copying' | 'copied' | 'error' >('idle'); + // Values that look like secrets, awaiting the user's choice; all start selected. + const [review, setReview] = useState(); + const [selected, setSelected] = useState>(new Set()); const copiedTimerRef = useRef>(undefined); useEffect( @@ -735,7 +750,7 @@ function FailureActions({ } }; - const copyDiagnostics = async () => { + const copyDiagnostics = async (redact?: readonly string[]) => { if (retryPending || copyState === 'copying' || !onCopyDiagnostics) { return; } @@ -748,7 +763,16 @@ function FailureActions({ setCopyState('copying'); try { - await onCopyDiagnostics(failure.id); + const result = await onCopyDiagnostics(failure.id, redact); + if (result?.status === 'review') { + setReview(result.candidates); + setSelected( + new Set(result.candidates.map((candidate) => candidate.value)), + ); + setCopyState('idle'); + return; + } + setReview(undefined); setCopyState('copied'); copiedTimerRef.current = setTimeout(() => { copiedTimerRef.current = undefined; @@ -797,6 +821,63 @@ function FailureActions({ Couldn’t copy diagnostics

)} + {review && ( +
+ + {review.length === 1 + ? 'This value in the diagnostics looks like a secret.' + : `${review.length} values in the diagnostics look like secrets.`}{' '} + Redact before copying? + + {review.map((candidate) => ( + + ))} +
+ + +
+
+ )} ); } diff --git a/dato-agent/src/entrypoints/AgentFrame.tsx b/dato-agent/src/entrypoints/AgentFrame.tsx index fa2610dd..ed95e95b 100644 --- a/dato-agent/src/entrypoints/AgentFrame.tsx +++ b/dato-agent/src/entrypoints/AgentFrame.tsx @@ -9,6 +9,7 @@ import { type AgentRecordResultViewModel, AgentSurface, type AgentTranscriptEntry, + type CopyDiagnosticsResult, type UnsafeApprovalViewModel, } from '../components/AgentSurface'; import { @@ -126,6 +127,7 @@ import { revokeToken, waitForOAuthCallback, } from '../lib/oauth'; +import { findSecretCandidates, redactValues } from '../lib/secretCandidates'; import { type AgentSurfaceKind, buildSystemPrompt } from '../lib/systemPrompt'; import { createUnsafeDispatchJournalStore, @@ -4450,13 +4452,23 @@ export default function AgentFrame(props: AgentFrameProps) { await submit(candidate.submission, candidate); }; - const copyFailureDiagnostics = async (failureId: string): Promise => { + const copyFailureDiagnostics = async ( + failureId: string, + redact?: readonly string[], + ): Promise => { const diagnostics = failureDiagnosticsRef.current.get(failureId); if (!diagnostics) { throw new Error('Diagnostics are no longer available.'); } - await copyTextToClipboard(diagnostics); + // Known secrets are already masked; ask about anything else that looks like one. + if (redact === undefined) { + const candidates = findSecretCandidates(diagnostics); + if (candidates.length > 0) return { status: 'review', candidates }; + } + + await copyTextToClipboard(redactValues(diagnostics, redact ?? [])); + return { status: 'copied' }; }; const stopActiveTurn = (): void => { diff --git a/dato-agent/src/lib/secretCandidates.test.ts b/dato-agent/src/lib/secretCandidates.test.ts new file mode 100644 index 00000000..23ca3da1 --- /dev/null +++ b/dato-agent/src/lib/secretCandidates.test.ts @@ -0,0 +1,80 @@ +import { generateId } from '@datocms/cma-client-browser'; +import { describe, expect, it } from 'vitest'; +import { findSecretCandidates, redactValues } from './secretCandidates'; + +// Fake, random-looking values; none is a real credential. +const PASTED_KEY = 'xK9mQ2vL7pR4tZ8wN3bY6cF1hJ5dS0gA'; +const OTHER_TOKEN = 'aB3dE5fG7hJ9kL1mN2pQ4rS6tU8vW0xY'; + +const serialize = (value: unknown) => JSON.stringify(value, null, 2); + +describe('findSecretCandidates', () => { + it('flags long random-looking tokens anywhere, with a safe preview and path', () => { + const candidates = findSecretCandidates( + serialize({ + conversation: [{ text: `here is my key ${PASTED_KEY} please use it` }], + turn: { + events: [ + { output: `token=${OTHER_TOKEN}` }, + { output: `again ${PASTED_KEY}` }, + ], + }, + }), + ); + + expect(candidates.map((candidate) => candidate.value).sort()).toEqual( + [PASTED_KEY, OTHER_TOKEN].sort(), + ); + const pasted = candidates.find( + (candidate) => candidate.value === PASTED_KEY, + ); + expect(pasted).toMatchObject({ + preview: `…${PASTED_KEY.slice(-4)} · ${PASTED_KEY.length} chars`, + path: 'conversation[0].text', + occurrences: 2, + }); + expect(pasted?.preview).not.toContain(PASTED_KEY.slice(0, 8)); + }); + + it('skips DatoCMS IDs, identifier keys, words, paths and URLs', () => { + const recordId = generateId(); + const candidates = findSecretCandidates( + serialize({ + mention: `see record ${recordId}`, + script: + 'script://dato-agent/238763/primary/conversation%3A2ae6507d-2d04-4751-a5f3-7b809c9cfe92/x.ts', + siteId: PASTED_KEY, + itemIds: [OTHER_TOKEN], + currentRecord: { id: PASTED_KEY }, + prose: 'Internationalization considerations notwithstanding everything', + stack: 'at runTurn (dato-agent/src/lib/agentRuntime.ts:4873:21)', + url: 'https://site-api.datocms.com/items?filter[type]=article&page[limit]=100', + timestamp: '2026-11-03T22:05:03.968Z', + }), + ); + + expect(candidates).toEqual([]); + }); + + it('still flags secrets under keys that merely end in "id"-like words', () => { + const candidates = findSecretCandidates( + serialize({ valid: PASTED_KEY, paid: OTHER_TOKEN }), + ); + expect(candidates).toHaveLength(2); + }); +}); + +describe('redactValues', () => { + it('replaces every occurrence of the chosen values only', () => { + const serialized = serialize({ + a: PASTED_KEY, + b: `x ${PASTED_KEY}`, + c: OTHER_TOKEN, + }); + const redacted = redactValues(serialized, [PASTED_KEY]); + + expect(redacted).not.toContain(PASTED_KEY); + expect(redacted).toContain(OTHER_TOKEN); + expect(redactValues(serialized, [])).toBe(serialized); + }); +}); diff --git a/dato-agent/src/lib/secretCandidates.ts b/dato-agent/src/lib/secretCandidates.ts new file mode 100644 index 00000000..7621c542 --- /dev/null +++ b/dato-agent/src/lib/secretCandidates.ts @@ -0,0 +1,168 @@ +import { isValidId } from '@datocms/cma-client-browser'; + +/** A value in the diagnostics that looks like a credential the plugin doesn't know about. */ +export type SecretCandidate = { + value: string; + /** Safe to show: the last 4 characters and the length, never the value itself. */ + preview: string; + /** Where it first appears in the payload, e.g. `turn.events[4].output`. */ + path: string; + occurrences: number; +}; + +const MIN_TOKEN_LENGTH = 20; +/** Bits per character. English words and file paths sit well below; keys and tokens above. */ +const MIN_ENTROPY = 3.5; +/** Characters that can belong to a token. Anything else (spaces, quotes, `/`, `.`, `:`) splits it. */ +const TOKEN_CHARACTERS = new Set( + 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_~+'.split( + '', + ), +); + +const HEX_DIGITS = new Set('0123456789abcdefABCDEF'.split('')); + +function tokens(text: string): string[] { + const found: string[] = []; + let current = ''; + for (let index = 0; index < text.length; index++) { + const character = text[index] ?? ''; + if (TOKEN_CHARACTERS.has(character)) { + current += character; + continue; + } + if (current.length >= MIN_TOKEN_LENGTH) found.push(current); + current = ''; + // A URL escape such as %3A is a separator too, so `id%3A` yields the UUID alone. + if ( + character === '%' && + HEX_DIGITS.has(text[index + 1] ?? '') && + HEX_DIGITS.has(text[index + 2] ?? '') + ) + index += 2; + } + if (current.length >= MIN_TOKEN_LENGTH) found.push(current); + return found; +} + +function shannonEntropy(text: string): number { + const counts = new Map(); + for (const character of text) + counts.set(character, (counts.get(character) ?? 0) + 1); + let entropy = 0; + for (const count of counts.values()) { + const probability = count / text.length; + entropy -= probability * Math.log2(probability); + } + return entropy; +} + +function isRandomLooking(token: string): boolean { + let hasLetter = false; + let hasDigit = false; + for (const character of token) { + if (character >= '0' && character <= '9') hasDigit = true; + else if ( + (character >= 'a' && character <= 'z') || + (character >= 'A' && character <= 'Z') + ) + hasLetter = true; + } + return hasLetter && hasDigit && shannonEntropy(token) >= MIN_ENTROPY; +} + +/** Canonical 8-4-4-4-12 hex UUIDs (conversation, request and response IDs), checked by position. */ +function isUuid(token: string): boolean { + if (token.length !== 36) return false; + for (let index = 0; index < token.length; index++) { + const isDashPosition = + index === 8 || index === 13 || index === 18 || index === 23; + if ( + isDashPosition + ? token[index] !== '-' + : !HEX_DIGITS.has(token[index] ?? '') + ) + return false; + } + return true; +} + +/** `id`, `siteId`, `itemIds`, `item_ids`, `ID`…: record and entity identifiers, not secrets. */ +function isIdentifierKey(key: string | undefined): boolean { + if (key === undefined) return false; + if (['id', 'ids', 'ID', 'IDs'].includes(key)) return true; + return ['Id', 'Ids', 'ID', 'IDs', '_id', '_ids'].some((suffix) => + key.endsWith(suffix), + ); +} + +function collectCandidates( + text: string, + path: string, + candidates: Map, +): void { + for (const token of tokens(text)) { + if (isValidId(token) || isUuid(token) || !isRandomLooking(token)) continue; + const existing = candidates.get(token); + if (existing) { + existing.occurrences += 1; + continue; + } + candidates.set(token, { + value: token, + preview: `…${token.slice(-4)} · ${token.length} chars`, + path, + occurrences: 1, + }); + } +} + +function visitValue( + value: unknown, + path: string, + key: string | undefined, + candidates: Map, +): void { + if (typeof value === 'string') { + if (!isIdentifierKey(key)) collectCandidates(value, path, candidates); + return; + } + if (Array.isArray(value)) { + for (const [index, entry] of value.entries()) + visitValue(entry, `${path}[${index}]`, key, candidates); + return; + } + if (value && typeof value === 'object') + for (const [childKey, child] of Object.entries(value)) + visitValue( + child, + path ? `${path}.${childKey}` : childKey, + childKey, + candidates, + ); +} + +/** + * Finds values in serialized diagnostics that look like secrets the plugin doesn't hold, such as + * a key a user pasted into chat, so the user can choose to redact them before copying. Detection is + * deliberately generic (long, mixed, high-entropy tokens) because the user reviews every hit. + * DatoCMS IDs (`isValidId`), UUIDs and values under identifier keys are skipped. + */ +export function findSecretCandidates(serialized: string): SecretCandidate[] { + const candidates = new Map(); + visitValue(JSON.parse(serialized), '', undefined, candidates); + return [...candidates.values()]; +} + +/** Replaces every occurrence of the chosen values with the diagnostics' redaction marker. */ +export function redactValues( + serialized: string, + values: readonly string[], +): string { + return [...values] + .sort((left, right) => right.length - left.length) + .reduce( + (current, value) => current.replaceAll(value, '[redacted]'), + serialized, + ); +} From e7c81df21049fc5d26f18b5f54135faebfeffbe1 Mon Sep 17 00:00:00 2001 From: "Roger (DatoCMS)" Date: Tue, 3 Nov 2026 14:49:54 -0800 Subject: [PATCH 5/7] Dato Agent: mask the OAuth token a turn actually used registerTerminalFailure is a per-render closure, so its oauthCredentials could predate a token rotation while the turn used the newer token from oauthCredentialsRef. An error message echoing that token reached the clipboard unmasked. Also mask the live ref and the turn's mcpAccessToken/repairOauthAccessToken, with a regression test. Bump DIAGNOSTICS_SCHEMA_VERSION to 2: configuration and OAuth state now have an allowlisted shape. Co-Authored-By: Claude Opus 5.5 --- .../src/entrypoints/AgentFrame.test.tsx | 94 +++++++++++++++++++ dato-agent/src/entrypoints/AgentFrame.tsx | 6 +- dato-agent/src/lib/diagnostics.test.ts | 2 +- dato-agent/src/lib/diagnostics.ts | 3 +- 4 files changed, 102 insertions(+), 3 deletions(-) diff --git a/dato-agent/src/entrypoints/AgentFrame.test.tsx b/dato-agent/src/entrypoints/AgentFrame.test.tsx index c5797f1d..95601e24 100644 --- a/dato-agent/src/entrypoints/AgentFrame.test.tsx +++ b/dato-agent/src/entrypoints/AgentFrame.test.tsx @@ -6759,6 +6759,100 @@ describe('AgentFrame', () => { ); }); + it('masks the OAuth token a turn actually used, not only the one its render closed over', async () => { + const frameProps = props({ + currentUserId: `diagnostics-rotated-user-${testUser}`, + }); + const credentialStore = createCredentialStore({ + siteId: frameProps.siteId, + currentUserId: frameProps.currentUserId, + }); + const saveToken = (accessToken: string) => + credentialStore.save( + createOAuthCredentials( + { + clientId: 'diagnostic-client', + clientIdIssuedAt: 1_700_000_000, + redirectUri: 'https://example.test/oauth/callback', + }, + { accessToken, tokenType: 'Bearer', obtainedAt: 1_700_000_000 }, + ), + { remember: true }, + ); + saveToken('stale-oauth-secret-value'); + const failure = { + code: 'api_error' as const, + message: + 'MCP rejected Bearer rotated-oauth-secret-value after stale-oauth-secret-value expired.', + retryable: true, + }; + mocks.runtime = { + runTurn: vi.fn( + async ( + _args: unknown, + onEvent?: (event: AgentRuntimeEvent) => void | Promise, + ) => { + const failed = completedResult({ + status: 'failed', + responseId: 'resp_rotated', + error: failure, + }); + await onEvent?.({ + type: 'error', + responseId: 'resp_rotated', + error: failure, + }); + await onEvent?.({ type: 'turn_completed', result: failed }); + return failed; + }, + ), + } as unknown as AgentRuntime; + const writeText = vi.fn().mockResolvedValue(undefined); + Object.defineProperty(navigator, 'clipboard', { + configurable: true, + value: { writeText }, + }); + + render(); + // A submit handler from before the token rotation: its render still holds the stale token. + const staleSubmit = mocks.surfaceProps?.onSubmit; + saveToken('rotated-oauth-secret-value'); + act(() => { + window.dispatchEvent( + new StorageEvent('storage', { + key: credentialStore.key, + newValue: localStorage.getItem(credentialStore.key), + storageArea: localStorage, + }), + ); + }); + act(() => { + staleSubmit?.('Describe the project'); + }); + await waitFor(() => expect(mocks.surfaceProps?.isRunning).toBe(false)); + + const failedEntry = mocks.surfaceProps?.entries.find( + (entry) => + entry.kind === 'message' && entry.role === 'assistant' && entry.failure, + ); + if (failedEntry?.kind !== 'message' || !failedEntry.failure) { + throw new Error('Expected terminal failure diagnostics.'); + } + await act(async () => { + await mocks.surfaceProps?.onCopyFailureDiagnostics?.( + failedEntry.failure?.id ?? '', + ); + }); + + expect(writeText).toHaveBeenCalledOnce(); + const serialized = String(writeText.mock.calls[0]?.[0]); + expect(serialized).not.toContain('rotated-oauth-secret-value'); + expect(serialized).not.toContain('stale-oauth-secret-value'); + expect(serialized).toContain( + 'MCP rejected Bearer [redacted] after [redacted] expired.', + ); + }); + it('offers diagnostics but never retry when the runtime throws without an explicit retryable result', async () => { mocks.runtime = { runTurn: vi diff --git a/dato-agent/src/entrypoints/AgentFrame.tsx b/dato-agent/src/entrypoints/AgentFrame.tsx index ed95e95b..9d648c8b 100644 --- a/dato-agent/src/entrypoints/AgentFrame.tsx +++ b/dato-agent/src/entrypoints/AgentFrame.tsx @@ -3292,11 +3292,15 @@ export default function AgentFrame(props: AgentFrameProps) { pendingNavigation: pendingNavigationRef.current, }, { - // Every live secret this frame holds; masked wherever it appears in the payload. + // Every live secret this frame holds; masked wherever it appears in the payload. This + // closure can predate a token rotation, so read the live ref and the tokens the turn used. secrets: [ props.config.openAiApiKey, props.config.anthropicApiKey, oauthCredentials?.token?.accessToken, + oauthCredentialsRef.current?.token?.accessToken, + turn.mcpAccessToken, + turn.repairOauthAccessToken, ], }, ), diff --git a/dato-agent/src/lib/diagnostics.test.ts b/dato-agent/src/lib/diagnostics.test.ts index f4508a9a..f54c99a1 100644 --- a/dato-agent/src/lib/diagnostics.test.ts +++ b/dato-agent/src/lib/diagnostics.test.ts @@ -50,7 +50,7 @@ describe('serializeDiagnostics', () => { "z": true }, "prompt": "Change record 178178740", - "schemaVersion": 1, + "schemaVersion": 2, "token": "[redacted]" }`); }); diff --git a/dato-agent/src/lib/diagnostics.ts b/dato-agent/src/lib/diagnostics.ts index 213218bb..a892a04b 100644 --- a/dato-agent/src/lib/diagnostics.ts +++ b/dato-agent/src/lib/diagnostics.ts @@ -1,4 +1,5 @@ -export const DIAGNOSTICS_SCHEMA_VERSION = 1; +/** 2: configuration and OAuth state are allowlisted; secrets are never included. */ +export const DIAGNOSTICS_SCHEMA_VERSION = 2; type JsonSafeValue = | null From d41d3187a19091023d68bd1e5e1c5948c0e46993 Mon Sep 17 00:00:00 2001 From: "Roger (DatoCMS)" Date: Tue, 3 Nov 2026 14:58:44 -0800 Subject: [PATCH 6/7] Dato Agent: fewer false positives in the diagnostics secret review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live review flagged the project name (claude-…-2026-11-03) and a host context fingerprint. Flag a token only if one of its -/_ segments mixes letters and digits, as keys do (sk-ant-api03-…) and slugs don't; treat fingerprint/hash/checksum/digest keys as identifiers. Keep each preview on one line. Co-Authored-By: Claude Opus 5.5 --- .../src/components/AgentSurface.module.css | 11 ++++- dato-agent/src/lib/secretCandidates.test.ts | 2 + dato-agent/src/lib/secretCandidates.ts | 45 +++++++++++++++---- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/dato-agent/src/components/AgentSurface.module.css b/dato-agent/src/components/AgentSurface.module.css index d17c8a0f..5f963454 100644 --- a/dato-agent/src/components/AgentSurface.module.css +++ b/dato-agent/src/components/AgentSurface.module.css @@ -666,7 +666,16 @@ align-items: baseline; gap: var(--spacing-xs); color: var(--color--ink-subtle); - word-break: break-all; +} + +.diagnosticsReviewItem code { + flex: none; + white-space: nowrap; +} + +.diagnosticsReviewItem span { + min-width: 0; + overflow-wrap: anywhere; } .activityDisclosure { diff --git a/dato-agent/src/lib/secretCandidates.test.ts b/dato-agent/src/lib/secretCandidates.test.ts index 23ca3da1..ba8be672 100644 --- a/dato-agent/src/lib/secretCandidates.test.ts +++ b/dato-agent/src/lib/secretCandidates.test.ts @@ -50,6 +50,8 @@ describe('findSecretCandidates', () => { stack: 'at runTurn (dato-agent/src/lib/agentRuntime.ts:4873:21)', url: 'https://site-api.datocms.com/items?filter[type]=article&page[limit]=100', timestamp: '2026-11-03T22:05:03.968Z', + siteName: 'claude-prtest-recordbin-2026-11-03', + hostContexts: [{ fingerprint: 'a1b2c3d4e5f6a7b8c9d0-12d' }], }), ); diff --git a/dato-agent/src/lib/secretCandidates.ts b/dato-agent/src/lib/secretCandidates.ts index 7621c542..f31712cf 100644 --- a/dato-agent/src/lib/secretCandidates.ts +++ b/dato-agent/src/lib/secretCandidates.ts @@ -57,10 +57,10 @@ function shannonEntropy(text: string): number { return entropy; } -function isRandomLooking(token: string): boolean { +function mixesLettersAndDigits(text: string): boolean { let hasLetter = false; let hasDigit = false; - for (const character of token) { + for (const character of text) { if (character >= '0' && character <= '9') hasDigit = true; else if ( (character >= 'a' && character <= 'z') || @@ -68,7 +68,18 @@ function isRandomLooking(token: string): boolean { ) hasLetter = true; } - return hasLetter && hasDigit && shannonEntropy(token) >= MIN_ENTROPY; + return hasLetter && hasDigit; +} + +/** + * Keys and tokens have at least one segment (between `-` / `_`) that mixes letters and digits, like + * `sk-ant-api03-…`. Slugs and names such as `my-site-2026-11-03` only join words and numbers. + */ +function isRandomLooking(token: string): boolean { + const segments = token.split('-').flatMap((part) => part.split('_')); + return ( + segments.some(mixesLettersAndDigits) && shannonEntropy(token) >= MIN_ENTROPY + ); } /** Canonical 8-4-4-4-12 hex UUIDs (conversation, request and response IDs), checked by position. */ @@ -87,13 +98,31 @@ function isUuid(token: string): boolean { return true; } -/** `id`, `siteId`, `itemIds`, `item_ids`, `ID`…: record and entity identifiers, not secrets. */ +/** Identifiers and digests (`id`, `siteId`, `item_ids`, `fingerprint`, `contentHash`…), not secrets. */ function isIdentifierKey(key: string | undefined): boolean { if (key === undefined) return false; - if (['id', 'ids', 'ID', 'IDs'].includes(key)) return true; - return ['Id', 'Ids', 'ID', 'IDs', '_id', '_ids'].some((suffix) => - key.endsWith(suffix), - ); + const exact = [ + 'id', + 'ids', + 'ID', + 'IDs', + 'fingerprint', + 'hash', + 'checksum', + 'digest', + ]; + if (exact.includes(key)) return true; + return [ + 'Id', + 'Ids', + 'ID', + 'IDs', + '_id', + '_ids', + 'Hash', + 'Fingerprint', + '_hash', + ].some((suffix) => key.endsWith(suffix)); } function collectCandidates( From 46614c0da52249267b01d4734d8a634f3de17914 Mon Sep 17 00:00:00 2001 From: "Roger (DatoCMS)" Date: Tue, 3 Nov 2026 15:09:32 -0800 Subject: [PATCH 7/7] Dato Agent: say what ticking a value in the diagnostics review does Each candidate now has a visible "Redact" label, and a hint under the heading says ticked values are replaced with [redacted] in the copy. Co-Authored-By: Claude Opus 5.5 --- dato-agent/src/components/AgentSurface.module.css | 11 +++++++++++ dato-agent/src/components/AgentSurface.test.tsx | 6 ++++++ dato-agent/src/components/AgentSurface.tsx | 7 +++++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/dato-agent/src/components/AgentSurface.module.css b/dato-agent/src/components/AgentSurface.module.css index 5f963454..f98dd120 100644 --- a/dato-agent/src/components/AgentSurface.module.css +++ b/dato-agent/src/components/AgentSurface.module.css @@ -668,6 +668,17 @@ color: var(--color--ink-subtle); } +.diagnosticsReviewHint { + margin: 0; + color: var(--color--ink-subtle); +} + +.diagnosticsReviewItem .diagnosticsReviewAction { + flex: none; + color: var(--color--ink); + font-weight: 600; +} + .diagnosticsReviewItem code { flex: none; white-space: nowrap; diff --git a/dato-agent/src/components/AgentSurface.test.tsx b/dato-agent/src/components/AgentSurface.test.tsx index 2bb1930e..fd0c5fc5 100644 --- a/dato-agent/src/components/AgentSurface.test.tsx +++ b/dato-agent/src/components/AgentSurface.test.tsx @@ -1430,6 +1430,12 @@ describe('AgentSurface', () => { screen.getByText(/2 values in the diagnostics look like secrets/), ).toBeInTheDocument(); expect(screen.getByText('…0002 · 23 chars')).toBeInTheDocument(); + expect( + screen.getByText( + 'Ticked values are replaced with [redacted] in the copy.', + ), + ).toBeInTheDocument(); + expect(screen.getAllByText('Redact')).toHaveLength(2); expect( screen.queryByText('second-secret-value-002'), ).not.toBeInTheDocument(); diff --git a/dato-agent/src/components/AgentSurface.tsx b/dato-agent/src/components/AgentSurface.tsx index d5f53c08..0ab660d9 100644 --- a/dato-agent/src/components/AgentSurface.tsx +++ b/dato-agent/src/components/AgentSurface.tsx @@ -826,9 +826,11 @@ function FailureActions({ {review.length === 1 ? 'This value in the diagnostics looks like a secret.' - : `${review.length} values in the diagnostics look like secrets.`}{' '} - Redact before copying? + : `${review.length} values in the diagnostics look like secrets.`} +

+ Ticked values are replaced with [redacted] in the copy. +

{review.map((candidate) => (