diff --git a/apps/sim/app/api/billing/update-cost/route.test.ts b/apps/sim/app/api/billing/update-cost/route.test.ts index 29f86b66aea..a0098ea6995 100644 --- a/apps/sim/app/api/billing/update-cost/route.test.ts +++ b/apps/sim/app/api/billing/update-cost/route.test.ts @@ -1,5 +1,6 @@ import { readFileSync } from 'node:fs' import { createMockRequest, resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { createRouteContext } from '@sim/testing/helpers/http' import { billingAttributionMock, billingAttributionMockFns, @@ -9,6 +10,7 @@ import { billingPlanMock, billingPlanMockFns } from '@sim/testing/mocks/billing- import { billingUsageLogMock, billingUsageLogMockFns, + MockCumulativeUsageInFlightError, } from '@sim/testing/mocks/billing-usage-log.mock' import { billingUsageMonitorMock, @@ -18,6 +20,7 @@ import { copilotHttpMock, copilotHttpMockFns } from '@sim/testing/mocks/copilot- import { mothershipOtelMock } from '@sim/testing/mocks/mothership-otel.mock' import { sleep } from '@sim/utils/helpers' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { BillingInFlightCode } from '@/lib/mothership/generated/billing' const { mockCheckAndBillOverageThreshold, @@ -748,6 +751,42 @@ describe('POST /api/billing/update-cost — workspaceId attribution', () => { }) }) + it('answers a retryable in-flight 503 while another attempt of the request is recorded', async () => { + const billingRequestId = '0190c03f-9f7d-4b79-8b58-e7f779fd29e1' + mockRecordCumulativeUsage.mockRejectedValueOnce( + new MockCumulativeUsageInFlightError(`update-cost:${billingRequestId}`) + ) + + const res = await POST( + createMockRequest( + 'POST', + { + userId: 'user-1', + cost: 0.5, + model: 'claude-opus-4.8', + source: 'workspace-chat', + idempotencyKey: billingRequestId, + }, + { + 'x-api-key': 'internal', + 'x-sim-billing-protocol': 'direct-v1', + 'x-sim-billing-request-id': billingRequestId, + 'x-sim-billing-account-decision': 'serialized-account-decision', + } + ), + createRouteContext({}) + ) + + expect(res.status).toBe(503) + expect(res.headers.get('retry-after')).toBe('5') + await expect(res.json()).resolves.toMatchObject({ + success: false, + code: BillingInFlightCode, + retryable: true, + }) + expect(mockCheckAndBillPayerOverageThreshold).not.toHaveBeenCalled() + }) + it('retries modern settlement on a duplicate cumulative callback before returning 409', async () => { const billingRequestId = '0190c03f-9f7d-4b79-8b58-e7f779fd29e1' mockRecordCumulativeUsage diff --git a/apps/sim/app/api/billing/update-cost/route.ts b/apps/sim/app/api/billing/update-cost/route.ts index 3c3668cb732..6ff3f097f8e 100644 --- a/apps/sim/app/api/billing/update-cost/route.ts +++ b/apps/sim/app/api/billing/update-cost/route.ts @@ -30,6 +30,7 @@ import { import { type CumulativeUsageContextField, CumulativeUsageContextMismatchError, + CumulativeUsageInFlightError, CumulativeUsagePeriodClosedError, recordCumulativeUsage, } from '@/lib/billing/core/usage-log' @@ -43,6 +44,7 @@ import { isBillingEnabled, isHosted } from '@/lib/core/config/env-flags' import { withinDeadline } from '@/lib/core/utils/deadline' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { BillingInFlightCode } from '@/lib/mothership/generated/billing' import { BILLING_CALLBACK_OUTCOME } from '@/lib/mothership/generated/billing-protocol-v1' import { BillingRouteOutcome } from '@/lib/mothership/generated/trace-attribute-values-v1' import { TraceAttr } from '@/lib/mothership/generated/trace-attributes-v1' @@ -53,7 +55,7 @@ import { withIncomingGoSpan } from '@/lib/mothership/request/otel' const logger = createLogger('BillingUpdateCostAPI') /** * How long a cost callback waits on the payer's standing. The worker gives up on the whole - * callback after 5 s, and a cold gate read can wait on the ledger far longer; past this the + * callback after 15 s, and a cold gate read can wait on the ledger far longer; past this the * callback answers not-exceeded. The abandoned read keeps running and caches its admission, and * the next step or re-check reads a refusal again. */ @@ -510,6 +512,32 @@ async function updateCostInner(req: NextRequest, span: Span): Promise ({ import { CumulativeUsageContextMismatchError, + CumulativeUsageInFlightError, CumulativeUsagePeriodClosedError, getBillingPeriodUsageCost, getBillingPeriodUsageCostByUser, @@ -92,6 +93,18 @@ function usage(cost: number, eventKey = 'update-cost:shared-request'): RecordCum } } +/** Resends a callback refused as in flight, the way the worker's replay does, until it is recorded. */ +async function untilRecorded(params: RecordCumulativeUsageParams) { + for (let attempt = 0; ; attempt++) { + try { + return await recordCumulativeUsage(params) + } catch (error) { + if (!(error instanceof CumulativeUsageInFlightError) || attempt >= 200) throw error + await sleep(50) + } + } +} + async function ledgerRows() { return connection<{ event_key: string; cost: string }[]>` select event_key, cost from usage_log order by event_key @@ -199,13 +212,15 @@ describe('Cumulative billing with PostgreSQL', () => { } ) - it('reproduces a retry timing out behind an idle holder when only lock waits are bounded', async () => { + it('refuses a retry behind a holder of the same request at once, without waiting on its lock', async () => { const pause = pauseNextTransaction(true) const holder = recordCumulativeUsage(usage(0.4)) try { await pause.reached.promise + const startedAt = performance.now() const failure = await recordCumulativeUsage(usage(0.8)).catch((error: unknown) => error) - expect(getPostgresErrorCode(failure)).toBe('55P03') + expect(failure).toBeInstanceOf(CumulativeUsageInFlightError) + expect(performance.now() - startedAt).toBeLessThan(1_000) expect(await ledgerRows()).toEqual([]) } finally { pause.release.resolve() @@ -223,8 +238,8 @@ describe('Cumulative billing with PostgreSQL', () => { try { await pause.reached.promise const failure = await recordCumulativeUsage(usage(0.8)).catch((error: unknown) => error) - expect(getPostgresErrorCode(failure)).toBe('55P03') - const recovered = await recordCumulativeUsage(usage(0.8)) + expect(failure).toBeInstanceOf(CumulativeUsageInFlightError) + const recovered = await untilRecorded(usage(0.8)) expect(recovered.billed).toBe(true) expect(recovered.delta).toBeCloseTo(0.8 - initial, 9) expect(recovered.total).toBe(0.8) @@ -249,7 +264,7 @@ describe('Cumulative billing with PostgreSQL', () => { it('converges concurrent out-of-order callbacks and independent events to their exact totals', async () => { const costs = [0.4, 0.1, 0.8, 0.3, 0.8, 0.6] - const results = await Promise.all(costs.map((cost) => recordCumulativeUsage(usage(cost)))) + const results = await Promise.all(costs.map((cost) => untilRecorded(usage(cost)))) expect(results.reduce((total, result) => total + result.delta, 0)).toBeCloseTo(0.8, 9) await Promise.all( Array.from({ length: 32 }, (_, index) => diff --git a/apps/sim/lib/billing/core/usage-log.test.ts b/apps/sim/lib/billing/core/usage-log.test.ts index 9a1d2d73523..c40202faa5f 100644 --- a/apps/sim/lib/billing/core/usage-log.test.ts +++ b/apps/sim/lib/billing/core/usage-log.test.ts @@ -5,6 +5,7 @@ import { billingSubscriptionUtilsMock, billingSubscriptionUtilsMockFns, } from '@sim/testing/mocks/billing-subscription-utils.mock' +import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' const { @@ -31,6 +32,7 @@ import { USAGE_LEDGER_STATEMENT_TIMEOUT_MS } from '@/lib/billing/constants' import { CUMULATIVE_COST_EPSILON, CumulativeUsageContextMismatchError, + CumulativeUsageInFlightError, getBillingPeriodUsageCost, getBillingPeriodUsageCostByUser, getBillingPeriodUsageCostWithSourceSubset, @@ -245,7 +247,8 @@ describe('recordCumulativeUsage', () => { mockIsOrgScopedSubscription.mockReturnValue(true) }) - const setupTx = (existingRow: Partial | null) => { + /** `locked`: another attempt of the request holds its ledger lock. */ + const setupTx = (existingRow: Partial | null, locked = false) => { const resolvedExistingRow = existingRow ? { ...defaultExistingRow, ...existingRow } : null const limit = vi.fn().mockResolvedValue(resolvedExistingRow ? [resolvedExistingRow] : []) const where = vi.fn().mockReturnValue({ limit }) @@ -255,7 +258,8 @@ describe('recordCumulativeUsage', () => { const updateSet = vi.fn().mockReturnValue({ where: updateWhere }) mockUpdate.mockReturnValue({ set: updateSet }) const tx = { - execute: vi.fn().mockResolvedValue(undefined), + // The configure statement's result is unused; the try-lock reads `acquired`. + execute: vi.fn().mockResolvedValue([{ acquired: !locked }]), select, update: mockUpdate, insert: mockInsert, // recordUsage(tx) reuses the shared insert chain @@ -290,6 +294,27 @@ describe('recordCumulativeUsage', () => { }) }) + it('refuses at once, writing nothing, while another attempt of the request holds its lock', async () => { + const { select } = setupTx(null, true) + await expect( + recordCumulativeUsage({ + userId: 'user-1', + source: 'workspace-chat', + model: 'claude-opus-4.8', + cost: 0.4662453, + eventKey: 'update-cost:msg-1-billing', + }) + ).rejects.toBeInstanceOf(CumulativeUsageInFlightError) + expect(select).not.toHaveBeenCalled() + expect(mockInsert).not.toHaveBeenCalled() + expect(mockUpdate).not.toHaveBeenCalled() + // The route logs the refusal once; it is not a transaction that failed to complete. + expect(getMockLogger('UsageLog').warn).not.toHaveBeenCalledWith( + 'Cumulative usage transaction did not complete promptly', + expect.anything() + ) + }) + it('tops up to the higher cumulative and bills only the delta', async () => { const { updateSet } = setupTx({ id: 'row-1', cost: '0.3474447' }) const result = await recordCumulativeUsage({ diff --git a/apps/sim/lib/billing/core/usage-log.ts b/apps/sim/lib/billing/core/usage-log.ts index 78bfb51ac06..2c29a25e4f1 100644 --- a/apps/sim/lib/billing/core/usage-log.ts +++ b/apps/sim/lib/billing/core/usage-log.ts @@ -30,7 +30,7 @@ import { isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils' import type { InternalUsageLogSource } from '@/lib/billing/usage-sources' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' import { HttpError } from '@/lib/core/utils/http-error' -import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' +import { tryAcquireAdvisoryXactLock } from '@/lib/db/advisory-locks' import type { DbClient, DbOrTx } from '@/lib/db/types' const logger = createLogger('UsageLog') @@ -700,6 +700,18 @@ export class CumulativeUsagePeriodClosedError extends Error { } } +/** + * Another attempt of the same request holds its ledger lock, so this one cannot know whether that + * attempt's charge will land. Waiting would only hold a connection for an answer the caller gets by + * retrying later: the monotonic top-up applies whichever cumulative arrives once. + */ +export class CumulativeUsageInFlightError extends Error { + constructor(readonly eventKey: string) { + super(`Cumulative usage event "${eventKey}" is already being recorded`) + this.name = 'CumulativeUsageInFlightError' + } +} + interface CumulativeUsageLedgerBinding { userId: string workspaceId: string | null @@ -755,11 +767,13 @@ function assertCumulativeUsageLedgerBinding( } /** - * PostgreSQL 17+ bounds the entire transaction below the callback's five-second - * deadline. Older supported servers instead bound each idle interval between - * statements, alongside the per-statement budget. Both policies release an idle - * lock holder without waiting for its application process to resume; only the - * newer policy also limits total elapsed transaction time. + * PostgreSQL 17+ bounds the transaction's statements and the idle time between + * them; older supported servers bound each idle interval instead, alongside the + * per-statement budget. Both release an idle lock holder without waiting for its + * application process to resume. Neither bounds COMMIT: a commit waiting for a + * synchronous standby keeps its locks, including the request's ledger lock, until + * the standby confirms, however long that takes. `lock_timeout` bounds the row + * locks taken under the ledger lock; the ledger lock itself is never waited for. */ const CUMULATIVE_FLUSH_TRANSACTION_TIMEOUT_MS = 4_000 const CUMULATIVE_FLUSH_STATEMENT_TIMEOUT_MS = 3_500 @@ -778,10 +792,12 @@ type CumulativeUsageStage = 'pool' | 'configure' | 'lock' | 'read' | 'write' | ' * (no two flushes can both believe they are first and clobber each other). * An existing row must match the incoming actor, workspace, payer, and billing * period before either a duplicate no-op or a top-up is accepted. - * The billing context is resolved BEFORE the transaction and the lock wait is - * bounded by `lock_timeout`. A server-enforced transaction deadline, or idle - * transaction deadline on older PostgreSQL, releases a stalled holder. The - * critical section uses one SELECT plus one INSERT/UPDATE on a single connection. + * The billing context is resolved BEFORE the transaction. The lock is taken + * without waiting: a held lock means another attempt of the same request is + * still being recorded (possibly committing behind a synchronous standby), and + * this attempt throws {@link CumulativeUsageInFlightError} instead of holding a + * connection while it waits. The critical section uses one SELECT plus one + * INSERT/UPDATE on a single connection. * * Because every leg flushes its cumulative and this converges to the max, * there is no under-billing if the request recovers after a partial flush, no @@ -816,6 +832,7 @@ export async function recordCumulativeUsage( const stageDurationsMs: Partial> = {} let succeeded = false let pgCode: string | undefined + let inFlight = false let toppedUpRowCreatedAt: Date | undefined const enterStage = (nextStage: CumulativeUsageStage) => { const now = Date.now() @@ -841,7 +858,9 @@ export async function recordCumulativeUsage( set_config('lock_timeout', ${`${CUMULATIVE_FLUSH_LOCK_TIMEOUT_MS}ms`}, true) `) enterStage('lock') - await acquireAdvisoryXactLock(tx, 'usage_log_event', eventKey) + if (!(await tryAcquireAdvisoryXactLock(tx, 'usage_log_event', eventKey))) { + throw new CumulativeUsageInFlightError(eventKey) + } enterStage('read') const rows = await tx @@ -997,12 +1016,14 @@ export async function recordCumulativeUsage( return result } catch (error) { pgCode = getPostgresErrorCode(error) + // Another attempt holding the lock is an answer, not a slow transaction; the route logs it. + if (error instanceof CumulativeUsageInFlightError) inFlight = true throw error } finally { const now = Date.now() stageDurationsMs[stage] = now - stageStartedAt const durationMs = now - startedAt - if (!succeeded || durationMs >= CUMULATIVE_FLUSH_SLOW_MS) { + if (!inFlight && (!succeeded || durationMs >= CUMULATIVE_FLUSH_SLOW_MS)) { logger.warn('Cumulative usage transaction did not complete promptly', { eventKey, succeeded, diff --git a/apps/sim/lib/core/application/content-delivery.test.ts b/apps/sim/lib/core/application/content-delivery.test.ts index 5cc74901232..037f25c7057 100644 --- a/apps/sim/lib/core/application/content-delivery.test.ts +++ b/apps/sim/lib/core/application/content-delivery.test.ts @@ -12,6 +12,7 @@ import { ContentDeliveryUnobservedError, ContentRefusedError, type ContentSource, + countWithheldContent, mustClassifyContent, noteContentDeliveryOperation, observeContentDelivery, @@ -509,3 +510,49 @@ describe('observeContentOutsideModel', () => { expect(batches).toEqual([]) }) }) + +describe('countWithheldContent', () => { + it('counts every refused item by kind across the requests and use cases it ran', async () => { + const { gate } = recordingGate() + const row = (id: string): ContentSource => ({ ...source(id, 'unknown'), kind: 'table-row' }) + const { result, withheld } = await observeContentDelivery(gate, () => + countWithheldContent(async () => { + await trackContentDelivery(() => admitContent(CHAT, [row('a'), row('b'), source('clean')])) + const admission = await admitContent(CHAT, [source('file', 'unknown')]) + return admission.refused.length + }) + ) + expect(result).toBe(1) + expect([...withheld].map(([kind, reasons]) => [kind, [...reasons]])).toEqual([ + ['table-row', [['unknown', 2]]], + ['workspace-file', [['unknown', 1]]], + ]) + }) + + it('keeps the reason a gate gave for each refusal', async () => { + const gate: ContentDeliveryGate = { + async admit(sources) { + return sources.map((item) => + item.id === 'raw' ? 'opaque' : item.id === 'later' ? 'registry' : true + ) + }, + } + const { withheld } = await observeContentDelivery(gate, () => + countWithheldContent(() => + admitContent(CHAT, [source('raw'), source('later'), source('clean')]) + ) + ) + expect([...(withheld.get('workspace-file') ?? [])]).toEqual([ + ['opaque', 1], + ['registry', 1], + ]) + }) + + it('counts nothing for a caller that never classifies', async () => { + const { gate } = recordingGate() + const { withheld } = await observeContentDelivery(gate, () => + countWithheldContent(() => admitContent(EXECUTOR, [source('file', 'unknown')])) + ) + expect(withheld.size).toBe(0) + }) +}) diff --git a/apps/sim/lib/core/application/content-delivery.ts b/apps/sim/lib/core/application/content-delivery.ts index 1d6a1bdec59..bc5f3d52259 100644 --- a/apps/sim/lib/core/application/content-delivery.ts +++ b/apps/sim/lib/core/application/content-delivery.ts @@ -37,12 +37,28 @@ export interface ContentSource { view: ContentView } +/** + * Why a gate refused an item. `unknown`: its secret provenance is unknown, so no read returns it. + * `opaque`: it holds recorded secrets in a view no redaction reaches (raw bytes, an excerpt), while + * its full text can be redacted. `registry`, `import`: its provenance could not be checked now. + */ +export type ContentRefusalReason = 'unknown' | 'opaque' | 'registry' | 'import' + +/** `true` admits; a reason refuses. A bare `false` refuses for an unstated reason, read as `unknown`. */ +export type ContentVerdict = boolean | ContentRefusalReason + +/** What a gate refused inside a {@link countWithheldContent} call, by kind and then reason. */ +export type WithheldContent = ReadonlyMap< + ContentSourceKind, + ReadonlyMap +> + /** * Decides, per item, whether content may leave through the surface that installed it. A gate never * throws for a refusal: refusing one item must not cost the call the items it admits. */ export interface ContentDeliveryGate { - admit(sources: readonly ContentSource[]): Promise + admit(sources: readonly ContentSource[]): Promise /** * Redacts text before cutting, or returns `undefined` when it cannot be shown. The current * operation's bounded evidence lets a surface create a redactor only when a cut needs one. @@ -73,6 +89,8 @@ interface ContentLedger { accounted: boolean /** Bounded aggregate of admitted provenance, for a surface that persists the delivered bytes. */ admitted: ContentProvenance[] + /** Only on a {@link countWithheldContent} ledger: the items refused inside it. */ + withheld?: Map> /** A committed write's returned item was refused; the operation answers with a refusal. */ withheldAfterWrite?: ContentSourceKind /** The gate itself failed while this ledger was open, whatever the route then answered. */ @@ -205,7 +223,7 @@ export function mustClassifyContent(principal: Principal): boolean { export class ContentAdmission { private readonly verdicts: ReadonlyMap - constructor(sources: readonly ContentSource[], verdicts: readonly boolean[]) { + constructor(sources: readonly ContentSource[], verdicts: readonly ContentVerdict[]) { this.verdicts = new Map(sources.map((source, index) => [source, verdicts[index] === true])) } @@ -291,7 +309,7 @@ export async function admitContent( const gate = gates.getStore() if (!gate) throw new ContentDeliveryUnobservedError() const ledger = ledgers.getStore() - let verdicts: readonly boolean[] + let verdicts: readonly ContentVerdict[] try { verdicts = await gate.admit(sources) } catch (error) { @@ -303,10 +321,22 @@ export async function admitContent( } markAccounted() if (ledger) { + for (let current: ContentLedger | undefined = ledger; current; current = current.parent) { + const { withheld } = current + if (!withheld) continue + sources.forEach(({ kind }, index) => { + const verdict = verdicts[index] + if (verdict === true) return + const reason = typeof verdict === 'string' ? verdict : 'unknown' + const reasons = withheld.get(kind) ?? new Map() + reasons.set(reason, (reasons.get(reason) ?? 0) + 1) + withheld.set(kind, reasons) + }) + } const batch = mergeContentProvenance( (function* () { for (let index = 0; index < sources.length; index++) { - if (verdicts[index]) yield sources[index].provenance + if (verdicts[index] === true) yield sources[index].provenance } })() ) @@ -459,6 +489,26 @@ export async function trackContentDelivery( } } +/** + * Runs `execute` and counts, by kind, the items a gate refused inside it, across every request and + * use case it ran. A refused item is left out of the result it was read in, so a surface reporting + * to Chat uses this to say what is missing rather than let it read as absent. + */ +export async function countWithheldContent( + execute: () => Promise +): Promise<{ result: T; withheld: WithheldContent }> { + const withheld = new Map>() + const ledger: ContentLedger = { + parent: ledgers.getStore(), + sourcedOperationAdmitted: false, + accounted: false, + admitted: [], + withheld, + } + const result = await ledgers.run(ledger, execute) + return { result, withheld } +} + /** Route admission records that Chat was admitted to an operation, in the request's ledger. */ export function noteContentDeliveryOperation( operation: ApplicationOperation, diff --git a/apps/sim/lib/core/application/index.ts b/apps/sim/lib/core/application/index.ts index 5b91b1fd8f2..47b0b126ef3 100644 --- a/apps/sim/lib/core/application/index.ts +++ b/apps/sim/lib/core/application/index.ts @@ -70,10 +70,13 @@ export { ContentDeliveryUnobservedError, ContentGateFailure, type ContentProvenance, + type ContentRefusalReason, ContentRefusedError, type ContentSource, type ContentSourceKind, + type ContentVerdict, type ContentView, + countWithheldContent, isCopilotPrincipal, mustClassifyContent, noteContentDeliveryOperation, @@ -83,4 +86,5 @@ export { requireAdmittedContent, runDeclaredContentOperation, trackContentDelivery, + type WithheldContent, } from './content-delivery' diff --git a/apps/sim/lib/execution/durable-secret-provenance-telemetry.test.ts b/apps/sim/lib/execution/durable-secret-provenance-telemetry.test.ts index 6d1d2c22870..549e63f6702 100644 --- a/apps/sim/lib/execution/durable-secret-provenance-telemetry.test.ts +++ b/apps/sim/lib/execution/durable-secret-provenance-telemetry.test.ts @@ -2,6 +2,7 @@ import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { describe, expect, it } from 'vitest' import { reportDurableSecretProvenanceRefusal, + reportDurableSecretProvenanceRefusals, reportDurableSecretProvenanceWrite, } from '@/lib/execution/durable-secret-provenance-telemetry' @@ -42,14 +43,66 @@ describe('durable secret provenance telemetry', () => { entries: [{ encryptedValue: 'private ciphertext' }], } reportDurableSecretProvenanceRefusal(report) - expect(logger.error).toHaveBeenCalledExactlyOnceWith( + expect(logger.warn).toHaveBeenCalledExactlyOnceWith( 'Refusing unavailable durable secret provenance', { surface: 'workspace-file', cause: 'workspace-file-opaque-secret-content', + refusedCount: 1, workspaceId: 'workspace-1', resourceId: 'file-1', } ) }) + it('reports a batch once per surface and cause, keeping every id paired with its cause', () => { + logger.warn.mockClear() + const reports = Array.from({ length: 163 }, (_, index) => ({ + surface: 'knowledge' as const, + cause: + index < 160 + ? ('knowledge-chunk-source-unavailable' as const) + : ('knowledge-opaque-secret-content' as const), + workspaceId: 'workspace-1', + resourceId: `chunk-${index}`, + })) + reportDurableSecretProvenanceRefusals(reports) + expect(logger.warn).toHaveBeenCalledTimes(2) + expect(logger.warn).toHaveBeenNthCalledWith( + 1, + 'Refusing unavailable durable secret provenance', + { + surface: 'knowledge', + cause: 'knowledge-chunk-source-unavailable', + refusedCount: 160, + workspaceId: 'workspace-1', + resourceIds: reports.slice(0, 160).map((report) => report.resourceId), + } + ) + expect(logger.warn).toHaveBeenNthCalledWith( + 2, + 'Refusing unavailable durable secret provenance', + { + surface: 'knowledge', + cause: 'knowledge-opaque-secret-content', + refusedCount: 3, + workspaceId: 'workspace-1', + resourceIds: ['chunk-160', 'chunk-161', 'chunk-162'], + } + ) + expect(logger.error).not.toHaveBeenCalled() + }) + + it('caps the ids of one very large group and says so', () => { + logger.warn.mockClear() + reportDurableSecretProvenanceRefusals( + Array.from({ length: 250 }, (_, index) => ({ + surface: 'table-row' as const, + cause: 'table-row-provenance-unavailable' as const, + resourceId: `row-${index}`, + })) + ) + const [[, fields]] = logger.warn.mock.calls + expect(fields).toMatchObject({ refusedCount: 250, resourceIdsTruncated: true }) + expect(fields.resourceIds).toHaveLength(200) + }) }) diff --git a/apps/sim/lib/execution/durable-secret-provenance-telemetry.ts b/apps/sim/lib/execution/durable-secret-provenance-telemetry.ts index 49c4c19d9f3..415665771bd 100644 --- a/apps/sim/lib/execution/durable-secret-provenance-telemetry.ts +++ b/apps/sim/lib/execution/durable-secret-provenance-telemetry.ts @@ -70,18 +70,58 @@ export function reportDurableSecretProvenanceWrite( }) } -/** Reports a durable read refusal without exposing content or secret values. */ +/** + * Reports a durable read refusal without exposing content or secret values. A refusal is policy + * applied to content that lacks provenance, not a fault, so it is a warning. + */ export function reportDurableSecretProvenanceRefusal( report: DurableSecretProvenanceRefusalReport ): void { - persistenceLogger.error('Refusing unavailable durable secret provenance', { + persistenceLogger.warn('Refusing unavailable durable secret provenance', { surface: report.surface, cause: report.cause, + refusedCount: 1, ...(report.workspaceId ? { workspaceId: report.workspaceId } : {}), ...(report.resourceId ? { resourceId: report.resourceId } : {}), }) } +/** Ids one grouped refusal lists, so a page of them stays one line. */ +const REFUSAL_RESOURCE_ID_LIMIT = 200 + +/** + * Reports the refusals of one read with one line per surface and cause, each + * carrying the same top-level `surface` and `cause` as a single refusal, its `refusedCount`, and + * the refused ids, so each id stays paired with its cause. + */ +export function reportDurableSecretProvenanceRefusals( + reports: readonly DurableSecretProvenanceRefusalReport[] +): void { + const groups = new Map() + for (const report of reports) { + const key = JSON.stringify([report.surface, report.cause, report.workspaceId]) + const group = groups.get(key) + if (group) group.push(report) + else groups.set(key, [report]) + } + for (const group of groups.values()) { + const [first] = group + if (group.length === 1) { + reportDurableSecretProvenanceRefusal(first) + continue + } + const resourceIds = group.flatMap((report) => (report.resourceId ? [report.resourceId] : [])) + persistenceLogger.warn('Refusing unavailable durable secret provenance', { + surface: first.surface, + cause: first.cause, + refusedCount: group.length, + ...(first.workspaceId ? { workspaceId: first.workspaceId } : {}), + resourceIds: resourceIds.slice(0, REFUSAL_RESOURCE_ID_LIMIT), + ...(resourceIds.length > REFUSAL_RESOURCE_ID_LIMIT ? { resourceIdsTruncated: true } : {}), + }) + } +} + export interface DurableSecretProvenanceUnrecordedReport { surface: DurableSecretProvenanceSurface workspaceId?: string diff --git a/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts b/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts index a0638b0f092..96a707dbcfa 100644 --- a/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts +++ b/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts @@ -374,7 +374,7 @@ describe('file provenance at the actual CLI and model-result boundary', () => { }) it('refuses only the unknown item of a forwarded response, naming it, without blanking the invocation', async () => { - const refusals = getMockLogger('DurableSecretProvenancePersistence').error + const refusals = getMockLogger('DurableSecretProvenancePersistence').warn refusals.mockClear() const clean = fileSource('clean', { status: 'exact', entries: [] }) const tainted = fileSource('tainted', { status: 'unknown' }) @@ -552,7 +552,7 @@ describe('file provenance at the actual CLI and model-result boundary', () => { }) ) classifySequence('exact', 'unknown') - const refusals = getMockLogger('DurableSecretProvenancePersistence').error + const refusals = getMockLogger('DurableSecretProvenancePersistence').warn refusals.mockClear() const trace = registry() const result = await runCli(grepFiles('CANARY_FOR_LOCAL_TEST'), trace) @@ -569,6 +569,12 @@ describe('file provenance at the actual CLI and model-result boundary', () => { resourceId: tainted.id, }) ) + /** The model is told the file it cannot see exists, so it does not read the gap as absence. */ + expect(result.output).toMatchObject({ + stderr: expect.stringContaining( + 'Note: 1 file was withheld from this result: its secret provenance is unknown, so reading it again will not return it.' + ), + }) }) describe('grep reshapes a file line only after redacting it', () => { diff --git a/apps/sim/lib/mothership/agent-cli/index.ts b/apps/sim/lib/mothership/agent-cli/index.ts index 4d7a7b525dd..bfc76115ffb 100644 --- a/apps/sim/lib/mothership/agent-cli/index.ts +++ b/apps/sim/lib/mothership/agent-cli/index.ts @@ -1,4 +1,5 @@ import { createEmbeddedClient, type EmbeddedCliIdentity } from 'sim/embed' +import { countWithheldContent } from '@/lib/core/application' import { withWorkspaceInvocationScope } from '@/lib/core/application/workspace-invocation-scope' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import { curateBlockDetail } from '@/lib/mothership/agent-cli/curation' @@ -16,6 +17,7 @@ import { createCopilotChatPrincipal, createTrustedOrganizationCopilotPrincipal, } from '@/lib/mothership/auth/application-delegation' +import { describeWithheldContent } from '@/lib/mothership/content-delivery/telemetry' import type { AgentCliRawResult, AgentCliRequest } from '@/lib/mothership/generated/agent-cli' import type { ResourceChange } from '@/lib/mothership/generated/resources' import { TraceSpan } from '@/lib/mothership/generated/trace-spans-v1' @@ -38,6 +40,18 @@ export interface AgentCliExecutionContext extends Omit { + const { result, withheld } = await countWithheldContent(() => + executeUncountedAgentCliRequest(request, context) + ) + const note = describeWithheldContent(withheld) + if (!note) return result + return { ...result, stderr: result.stderr ? `${result.stderr}\n${note}` : note } +} + +async function executeUncountedAgentCliRequest( + request: AgentCliRequest, + context: AgentCliExecutionContext ): Promise { context.signal?.throwIfAborted() if (context.readOnly && !isReadOnlyCliRequest(request)) { diff --git a/apps/sim/lib/mothership/content-delivery/registry-gate.test.ts b/apps/sim/lib/mothership/content-delivery/registry-gate.test.ts index 2eea660c1d7..67abfb84e8c 100644 --- a/apps/sim/lib/mothership/content-delivery/registry-gate.test.ts +++ b/apps/sim/lib/mothership/content-delivery/registry-gate.test.ts @@ -7,6 +7,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/core/security/encryption', () => encryptionMock) const telemetry = vi.hoisted(() => ({ reportDurableSecretProvenanceRefusal: vi.fn(), + reportDurableSecretProvenanceRefusals: vi.fn(), reportDurableSecretProvenanceUnrecordedBatch: vi.fn(), })) vi.mock('@/lib/execution/durable-secret-provenance-telemetry', () => telemetry) @@ -120,19 +121,33 @@ describe('createRegistryContentGate', () => { it('refuses an opaque view of an item holding secrets, which no redaction reaches', async () => { const registry = newRegistry() const gate = createRegistryContentGate({ registry, actorUserId: 'user-1' }) - expect(await gate.admit([item('archive', exact(SECRET), 'opaque')])).toEqual([false]) + expect(await gate.admit([item('archive', exact(SECRET), 'opaque')])).toEqual(['opaque']) expect(registry.isPermanentlyIncomplete()).toBe(false) - expect(telemetry.reportDurableSecretProvenanceRefusal).toHaveBeenCalledWith( + expect(telemetry.reportDurableSecretProvenanceRefusals).toHaveBeenCalledWith([ expect.objectContaining({ resourceId: 'archive', cause: 'workspace-file-opaque-secret-content', - }) + }), + ]) + }) + + it('reports every item one admission refuses as one batch, not one line per item', async () => { + telemetry.reportDurableSecretProvenanceRefusals.mockClear() + const gate = createRegistryContentGate({ registry: newRegistry(), actorUserId: 'user-1' }) + const refused = Array.from({ length: 50 }, (_, index) => + item(`legacy-${index}`, { status: 'unknown' }) ) + const verdicts = await gate.admit([...refused, item('clean', exact())]) + expect(verdicts).toEqual([...refused.map(() => 'unknown'), true]) + expect(telemetry.reportDurableSecretProvenanceRefusals).toHaveBeenCalledTimes(1) + const [reports] = telemetry.reportDurableSecretProvenanceRefusals.mock.calls[0] + expect(reports).toHaveLength(50) + expect(telemetry.reportDurableSecretProvenanceRefusal).not.toHaveBeenCalled() }) it('refuses an item holding secrets when there is no registry to redact with', async () => { const gate = createRegistryContentGate({ actorUserId: 'user-1' }) - expect(await gate.admit([item('config', exact(SECRET))])).toEqual([false]) + expect(await gate.admit([item('config', exact(SECRET))])).toEqual(['registry']) }) it('tracks opaque workbench downloads without relaxing model or excerpt admission', async () => { @@ -144,18 +159,18 @@ describe('createRegistryContentGate', () => { }) expect(await gate.admit([item('download', exact(SECRET), 'opaque')])).toEqual([true]) expect(modelView(SECRET, registry)).not.toContain(SECRET) - expect(await gate.admit([item('snippet', exact(SECRET), 'excerpt')])).toEqual([false]) + expect(await gate.admit([item('snippet', exact(SECRET), 'excerpt')])).toEqual(['opaque']) expect( await createRegistryContentGate({ actorUserId: 'user-1', destination: 'workbench' }).admit([ item('download', exact(SECRET), 'opaque'), ]) - ).toEqual([false]) + ).toEqual(['registry']) }) it('refuses an unknown item without latching the registry', async () => { const registry = newRegistry() const gate = createRegistryContentGate({ registry, actorUserId: 'user-1' }) - expect(await gate.admit([item('tainted', { status: 'unknown' })])).toEqual([false]) + expect(await gate.admit([item('tainted', { status: 'unknown' })])).toEqual(['unknown']) expect(registry.isPermanentlyIncomplete()).toBe(false) }) @@ -169,7 +184,7 @@ describe('createRegistryContentGate', () => { it('refuses an unknown item under taint when no registry could withhold the output', async () => { const gate = createRegistryContentGate({ actorUserId: 'user-1', unknown: 'taint' }) - expect(await gate.admit([item('tainted', { status: 'unknown' })])).toEqual([false]) + expect(await gate.admit([item('tainted', { status: 'unknown' })])).toEqual(['unknown']) }) it('refuses only the item whose secrets cannot be imported, keeping the rest of the call', async () => { @@ -184,7 +199,7 @@ describe('createRegistryContentGate', () => { item('config', exact(SECRET)), item('clean', { status: 'exact', entries: [] }), ]) - expect(verdicts).toEqual([false, true, true]) + expect(verdicts).toEqual(['import', true, true]) expect(registry.isPermanentlyIncomplete()).toBe(false) expect(modelView(`token=${SECRET}`, registry)).not.toContain(SECRET) }) @@ -198,7 +213,7 @@ describe('createRegistryContentGate', () => { const gate = createRegistryContentGate({ registry, actorUserId: 'user-1' }) expect(await gate.admit([item('existing', exact(...values))])).toEqual([true]) - expect(await gate.admit([item('overflow', exact(SECRET, OTHER_SECRET))])).toEqual([false]) + expect(await gate.admit([item('overflow', exact(SECRET, OTHER_SECRET))])).toEqual(['import']) expect(registry.isPermanentlyIncomplete()).toBe(false) expect(modelView(`public notes token=${values[0]}`, registry)).toBe( @@ -221,7 +236,7 @@ describe('createRegistryContentGate', () => { gate.admit([item('second', exact(OTHER_SECRET))]), ]) - expect(verdicts.flat().sort()).toEqual([false, true]) + expect(verdicts.flat().sort()).toEqual(['import', true]) expect(registry.isPermanentlyIncomplete()).toBe(false) expect(modelView(`public notes token=${values[0]}`, registry)).toBe( 'public notes token={{FIXTURE_TOKEN}}' diff --git a/apps/sim/lib/mothership/content-delivery/registry-gate.ts b/apps/sim/lib/mothership/content-delivery/registry-gate.ts index ba6bd14fa90..3937b3d1ae2 100644 --- a/apps/sim/lib/mothership/content-delivery/registry-gate.ts +++ b/apps/sim/lib/mothership/content-delivery/registry-gate.ts @@ -1,4 +1,4 @@ -import type { ContentDeliveryGate, ContentSource } from '@/lib/core/application' +import type { ContentDeliveryGate, ContentSource, ContentVerdict } from '@/lib/core/application' import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' import { type DurableSecretProvenanceUnrecordedReport, @@ -6,8 +6,8 @@ import { } from '@/lib/execution/durable-secret-provenance-telemetry' import { CONTENT_SURFACE, - type ContentRefusalReason, - reportContentRefusal, + type ContentRefusals, + createContentRefusals, } from '@/lib/mothership/content-delivery/telemetry' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -33,13 +33,11 @@ export interface RegistryContentGateScope { * any entry, so a refused item cannot latch the registry for every other item. */ export function createRegistryContentGate(scope: RegistryContentGateScope): ContentDeliveryGate { - const refuse = (source: ContentSource, reason: ContentRefusalReason): false => - reportContentRefusal(source, reason, scope.workspaceId) - const decide = async ( source: ContentSource, - unrecorded: DurableSecretProvenanceUnrecordedReport[] - ): Promise => { + unrecorded: DurableSecretProvenanceUnrecordedReport[], + { refuse }: ContentRefusals + ): Promise => { const { provenance } = source if (provenance.status === 'unknown') { /** Taint needs a registry to latch; without one the bytes would reach the model as is. */ @@ -89,8 +87,13 @@ export function createRegistryContentGate(scope: RegistryContentGateScope): Cont }, async admit(sources) { const unrecorded: DurableSecretProvenanceUnrecordedReport[] = [] - const verdicts: boolean[] = [] - for (const source of sources) verdicts.push(await decide(source, unrecorded)) + const refusals = createContentRefusals(scope.workspaceId) + const verdicts: ContentVerdict[] = [] + try { + for (const source of sources) verdicts.push(await decide(source, unrecorded, refusals)) + } finally { + refusals.report() + } reportDurableSecretProvenanceUnrecordedBatch(unrecorded) return verdicts }, diff --git a/apps/sim/lib/mothership/content-delivery/session-gate.test.ts b/apps/sim/lib/mothership/content-delivery/session-gate.test.ts index 164c2646bb7..6b664b7c48f 100644 --- a/apps/sim/lib/mothership/content-delivery/session-gate.test.ts +++ b/apps/sim/lib/mothership/content-delivery/session-gate.test.ts @@ -8,6 +8,7 @@ const history = vi.hoisted(() => ({ recordExistingSessionFileInput: vi.fn() })) vi.mock('@/lib/execution/remote-sandbox/session-file-provenance', () => history) const telemetry = vi.hoisted(() => ({ reportDurableSecretProvenanceRefusal: vi.fn(), + reportDurableSecretProvenanceRefusals: vi.fn(), reportDurableSecretProvenanceUnrecordedBatch: vi.fn(), })) vi.mock('@/lib/execution/durable-secret-provenance-telemetry', () => telemetry) @@ -42,11 +43,23 @@ beforeEach(() => { describe('createSessionContentGate', () => { it('keeps an unknown item out of the machine and its history', async () => { - expect(await gate().admit([item('tainted', { status: 'unknown' })])).toEqual([false]) + expect(await gate().admit([item('tainted', { status: 'unknown' })])).toEqual(['unknown']) expect(history.recordExistingSessionFileInput).not.toHaveBeenCalled() - expect(telemetry.reportDurableSecretProvenanceRefusal).toHaveBeenCalledWith( - expect.objectContaining({ resourceId: 'tainted' }) + expect(telemetry.reportDurableSecretProvenanceRefusals).toHaveBeenCalledWith([ + expect.objectContaining({ resourceId: 'tainted' }), + ]) + }) + + it('reports every item one admission refuses as one batch, not one line per item', async () => { + telemetry.reportDurableSecretProvenanceRefusals.mockClear() + const refused = Array.from({ length: 50 }, (_, index) => + item(`tainted-${index}`, { status: 'unknown' }) ) + expect(await gate().admit(refused)).toEqual(refused.map(() => 'unknown')) + expect(telemetry.reportDurableSecretProvenanceRefusals).toHaveBeenCalledTimes(1) + const [reports] = telemetry.reportDurableSecretProvenanceRefusals.mock.calls[0] + expect(reports).toHaveLength(50) + expect(telemetry.reportDurableSecretProvenanceRefusal).not.toHaveBeenCalled() }) it('admits secret-bearing bytes, even opaque, after merging their entries into the history', async () => { @@ -55,7 +68,7 @@ describe('createSessionContentGate', () => { item('tainted', { status: 'unknown' }), item('clean', { status: 'exact', entries: [] }), ]) - expect(verdicts).toEqual([true, false, true]) + expect(verdicts).toEqual([true, 'unknown', true]) expect(history.recordExistingSessionFileInput).toHaveBeenCalledTimes(1) expect(history.recordExistingSessionFileInput).toHaveBeenCalledWith('session-1', { status: 'exact', diff --git a/apps/sim/lib/mothership/content-delivery/session-gate.ts b/apps/sim/lib/mothership/content-delivery/session-gate.ts index a5fc9e387f3..89f6b1c439e 100644 --- a/apps/sim/lib/mothership/content-delivery/session-gate.ts +++ b/apps/sim/lib/mothership/content-delivery/session-gate.ts @@ -1,4 +1,4 @@ -import type { ContentDeliveryGate } from '@/lib/core/application' +import type { ContentDeliveryGate, ContentVerdict } from '@/lib/core/application' import { createDurableSecretProvenanceRegistry, type DurableSecretProvenance, @@ -7,7 +7,7 @@ import { } from '@/lib/execution/durable-secret-provenance' import { reportDurableSecretProvenanceUnrecordedBatch } from '@/lib/execution/durable-secret-provenance-telemetry' import { recordExistingSessionFileInput } from '@/lib/execution/remote-sandbox/session-file-provenance' -import { reportContentRefusal } from '@/lib/mothership/content-delivery/telemetry' +import { createContentRefusals } from '@/lib/mothership/content-delivery/telemetry' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' /** The workbench's history could not be widened, so nothing may enter the machine. */ @@ -53,13 +53,14 @@ export function createSessionContentGate(scope: { } }, async admit(sources) { - const verdicts: boolean[] = [] + const verdicts: ContentVerdict[] = [] const admitted: DurableSecretProvenance[] = [] + const refusals = createContentRefusals(scope.workspaceId) let unrecorded = 0 for (const source of sources) { const { provenance } = source if (provenance.status === 'unknown') { - verdicts.push(reportContentRefusal(source, 'unknown', scope.workspaceId)) + verdicts.push(refusals.refuse(source, 'unknown')) continue } if ( @@ -67,7 +68,7 @@ export function createSessionContentGate(scope: { provenance.entries.length > 0 && source.view === 'excerpt' ) { - verdicts.push(reportContentRefusal(source, 'opaque', scope.workspaceId)) + verdicts.push(refusals.refuse(source, 'opaque')) continue } if (provenance.status === 'unrecorded') { @@ -78,6 +79,7 @@ export function createSessionContentGate(scope: { } verdicts.push(true) } + refusals.report() if (admitted.length > 0) { try { await recordExistingSessionFileInput( diff --git a/apps/sim/lib/mothership/content-delivery/telemetry.test.ts b/apps/sim/lib/mothership/content-delivery/telemetry.test.ts new file mode 100644 index 00000000000..b70a864cbf7 --- /dev/null +++ b/apps/sim/lib/mothership/content-delivery/telemetry.test.ts @@ -0,0 +1,40 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import type { ContentRefusalReason, ContentSourceKind } from '@/lib/core/application' +import { describeWithheldContent } from '@/lib/mothership/content-delivery/telemetry' + +function withheld( + entries: [ContentSourceKind, [ContentRefusalReason, number][]][] +): Map> { + return new Map(entries.map(([kind, reasons]) => [kind, new Map(reasons)])) +} + +describe('describeWithheldContent', () => { + it('says nothing when nothing was withheld', () => { + expect(describeWithheldContent(new Map())).toBeUndefined() + }) + + it('tells Chat what each refusal means for its next read, naming no item', () => { + const note = describeWithheldContent( + withheld([ + ['knowledge-document', [['unknown', 3]]], + ['workspace-file', [['opaque', 1]]], + [ + 'table-row', + [ + ['registry', 2], + ['import', 1], + ], + ], + ]) + ) + expect(note?.split('\n')).toEqual([ + 'Note: 3 knowledge documents were withheld from this result: their secret provenance is unknown, so reading them again will not return them. They exist; do not treat them as missing.', + 'Note: 1 file was withheld from this result: it contains recorded secrets and cannot be shown as raw bytes or a preview. Read it as text for a redacted copy.', + 'Note: 2 table rows were withheld from this result: their secret provenance could not be verified right now. They may be readable on a later request.', + 'Note: 1 table row was withheld from this result: its secret provenance could not be verified right now. It may be readable on a later request.', + ]) + }) +}) diff --git a/apps/sim/lib/mothership/content-delivery/telemetry.ts b/apps/sim/lib/mothership/content-delivery/telemetry.ts index 658aee2a55e..d3946026e40 100644 --- a/apps/sim/lib/mothership/content-delivery/telemetry.ts +++ b/apps/sim/lib/mothership/content-delivery/telemetry.ts @@ -1,12 +1,16 @@ -import type { ContentSource, ContentSourceKind } from '@/lib/core/application' +import type { + ContentRefusalReason, + ContentSource, + ContentSourceKind, + WithheldContent, +} from '@/lib/core/application' import { type DurableSecretProvenanceRefusalCause, + type DurableSecretProvenanceRefusalReport, type DurableSecretProvenanceSurface, - reportDurableSecretProvenanceRefusal, + reportDurableSecretProvenanceRefusals, } from '@/lib/execution/durable-secret-provenance-telemetry' -export type ContentRefusalReason = 'unknown' | 'opaque' | 'registry' | 'import' - export const CONTENT_SURFACE: Record = { 'workspace-file': 'workspace-file', 'execution-file': 'workspace-file', @@ -55,17 +59,78 @@ const CAUSES: Record< }, } -/** Names the refused item and why, never its content. */ -export function reportContentRefusal( - source: ContentSource, - reason: ContentRefusalReason, - workspaceId: string | undefined -): false { - reportDurableSecretProvenanceRefusal({ - surface: CONTENT_SURFACE[source.kind], - cause: CAUSES[source.kind][reason], - workspaceId, - resourceId: source.id, - }) - return false +/** + * Collects the items one admission refuses and reports them once per surface and cause, naming + * each item by its id and why, never its content. A page of legacy rows is one line. + */ +export interface ContentRefusals { + /** Records the refusal of `source`; returns the verdict, its reason. */ + refuse(source: ContentSource, reason: ContentRefusalReason): ContentRefusalReason + /** Reports what was recorded. */ + report(): void +} + +export function createContentRefusals(workspaceId: string | undefined): ContentRefusals { + const reports: DurableSecretProvenanceRefusalReport[] = [] + return { + refuse(source, reason) { + reports.push({ + surface: CONTENT_SURFACE[source.kind], + cause: CAUSES[source.kind][reason], + workspaceId, + resourceId: source.id, + }) + return reason + }, + report(): void { + reportDurableSecretProvenanceRefusals(reports) + }, + } +} + +const WITHHELD_LABELS: Record = { + 'workspace-file': ['file', 'files'], + 'execution-file': ['run file', 'run files'], + 'knowledge-document': ['knowledge document', 'knowledge documents'], + 'knowledge-chunk': ['knowledge chunk', 'knowledge chunks'], + 'table-row': ['table row', 'table rows'], + 'tool-result': ['tool result', 'tool results'], +} + +/** What each refusal means for Chat's next read, as `[one item, several items]`. */ +const WITHHELD_REASONS: Record = { + unknown: [ + 'its secret provenance is unknown, so reading it again will not return it. It exists; do not treat it as missing.', + 'their secret provenance is unknown, so reading them again will not return them. They exist; do not treat them as missing.', + ], + opaque: [ + 'it contains recorded secrets and cannot be shown as raw bytes or a preview. Read it as text for a redacted copy.', + 'they contain recorded secrets and cannot be shown as raw bytes or a preview. Read them as text for a redacted copy.', + ], + registry: [ + 'its secret provenance could not be verified right now. It may be readable on a later request.', + 'their secret provenance could not be verified right now. They may be readable on a later request.', + ], + import: [ + 'its secret provenance could not be verified right now. It may be readable on a later request.', + 'their secret provenance could not be verified right now. They may be readable on a later request.', + ], +} + +/** + * Tells Chat's model what a result left out, and what to do about it: a refused item is dropped + * from the page it was read in, which otherwise reads exactly like an item that does not exist, + * and a model that cannot tell why keeps reading it again. Names no item. + */ +export function describeWithheldContent(withheld: WithheldContent): string | undefined { + const lines: string[] = [] + for (const [kind, reasons] of withheld) { + for (const [reason, count] of reasons) { + const form = count === 1 ? 0 : 1 + lines.push( + `Note: ${count} ${WITHHELD_LABELS[kind][form]} ${count === 1 ? 'was' : 'were'} withheld from this result: ${WITHHELD_REASONS[reason][form]}` + ) + } + } + return lines.length > 0 ? lines.join('\n') : undefined } diff --git a/apps/sim/lib/mothership/generated/billing.ts b/apps/sim/lib/mothership/generated/billing.ts index abdec1159ad..cf01b6b4b98 100644 --- a/apps/sim/lib/mothership/generated/billing.ts +++ b/apps/sim/lib/mothership/generated/billing.ts @@ -90,6 +90,8 @@ export const UsageLimitRefusal = z.object({ usageUpgrade: UsageUpgrade.optional().catch(undefined), }); export const BillingDuplicateCode = "DUPLICATE_BILLING_EVENT"; +/** A 503: another attempt of the same request is still being applied, so this one's outcome is unknown. */ +export const BillingInFlightCode = "BILLING_REQUEST_IN_FLIGHT"; /** Sim authors receipts at the paid execution boundary; tool output carries no billing authority. */ export const ServiceUsageReceipt = z.strictObject({ diff --git a/apps/sim/lib/table/rows/secret-provenance.ts b/apps/sim/lib/table/rows/secret-provenance.ts index 1d49c72e252..4e5d1d0cac6 100644 --- a/apps/sim/lib/table/rows/secret-provenance.ts +++ b/apps/sim/lib/table/rows/secret-provenance.ts @@ -988,8 +988,9 @@ function envelopeFromClassifications( rowCount: number, scope: ResolvedSecretTraceScopeV1 ): ResolvedSecretTraceProvenanceV1 { + /** A tracked row whose sidecar is stale, missing or malformed reads as unknown by policy; writer faults log where they write. */ const incomplete = (cause: string): ResolvedSecretTraceProvenanceV1 => { - logger.error('Table row read could not establish secret provenance', { + logger.warn('Table row read could not establish secret provenance', { surface: 'table-row', cause, rowCount, diff --git a/packages/testing/src/mocks/billing-usage-log.mock.ts b/packages/testing/src/mocks/billing-usage-log.mock.ts index 9e1133828fa..ff7def0f720 100644 --- a/packages/testing/src/mocks/billing-usage-log.mock.ts +++ b/packages/testing/src/mocks/billing-usage-log.mock.ts @@ -39,6 +39,14 @@ export class MockCumulativeUsagePeriodClosedError extends Error { } } +/** Stand-in for `CumulativeUsageInFlightError` with the real `name`, `eventKey`, and message. */ +export class MockCumulativeUsageInFlightError extends Error { + constructor(readonly eventKey: string) { + super(`Cumulative usage event "${eventKey}" is already being recorded`) + this.name = 'CumulativeUsageInFlightError' + } +} + /** * Stand-in for `UnknownUsageCursorError` with the real `name`, message, and `statusCode` 400. * It is NOT a subclass of the real `HttpError`, and its `cause` is a plain `Error` carrying @@ -107,7 +115,8 @@ export const billingUsageLogMockFns = { /** * Static mock module for `@/lib/billing/core/usage-log`. Constants carry the real values; * `CumulativeUsageContextMismatchError` is {@link MockCumulativeUsageContextMismatchError}, - * `CumulativeUsagePeriodClosedError` is {@link MockCumulativeUsagePeriodClosedError}, and + * `CumulativeUsagePeriodClosedError` is {@link MockCumulativeUsagePeriodClosedError}, + * `CumulativeUsageInFlightError` is {@link MockCumulativeUsageInFlightError}, and * `UnknownUsageCursorError` is {@link MockUnknownUsageCursorError}. * * @example @@ -122,6 +131,7 @@ export const billingUsageLogMock = { UNKNOWN_CURSOR_MESSAGE, CumulativeUsageContextMismatchError: MockCumulativeUsageContextMismatchError, CumulativeUsagePeriodClosedError: MockCumulativeUsagePeriodClosedError, + CumulativeUsageInFlightError: MockCumulativeUsageInFlightError, UnknownUsageCursorError: MockUnknownUsageCursorError, isUnbilledUsageCategory: billingUsageLogMockFns.mockIsUnbilledUsageCategory, stableEventKey: billingUsageLogMockFns.mockStableEventKey, diff --git a/packages/testing/src/mocks/index.ts b/packages/testing/src/mocks/index.ts index 212acf24b63..f32c784e589 100644 --- a/packages/testing/src/mocks/index.ts +++ b/packages/testing/src/mocks/index.ts @@ -146,6 +146,7 @@ export { billingUsageLogMock, billingUsageLogMockFns, MockCumulativeUsageContextMismatchError, + MockCumulativeUsageInFlightError, MockCumulativeUsagePeriodClosedError, MockUnknownUsageCursorError, } from './billing-usage-log.mock'