diff --git a/apps/sim/lib/internal/file/operations.test.ts b/apps/sim/lib/internal/file/operations.test.ts index aa228f2efb1..1c476fc66c5 100644 --- a/apps/sim/lib/internal/file/operations.test.ts +++ b/apps/sim/lib/internal/file/operations.test.ts @@ -1278,55 +1278,6 @@ describe('file manage operations', () => { expect(mockUploadWorkspaceFile).toHaveBeenCalled() }) - it('downgrades provenance when overwriting a file owned by another user', async () => { - const existing = workspaceFile('report', 'other-user') - mockResolveWorkspaceFileReference.mockResolvedValue(existing) - mockUpdateWorkspaceFileContent.mockResolvedValue(existing) - - const response = await POST( - createMockRequest( - 'POST', - { - operation: 'write', - workspaceId: 'workspace-1', - fileName: 'report.txt', - content: 'secret-value', - overwrite: true, - __privateSecretProvenance: { - version: 1, - complete: true, - selections: [ - { - key: 'content', - provenance: { - version: 1, - complete: true, - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], - scope: { userId: 'user-1', workspaceId: 'workspace-1' }, - }, - }, - ], - }, - }, - PRIVATE_SECRET_PROVENANCE_HEADER - ) - ) - - expect(response.status).toBe(200) - expect(mockUpdateWorkspaceFileContent).toHaveBeenCalledWith( - 'workspace-1', - 'report', - 'user-1', - Buffer.from('secret-value'), - 'text/plain', - { - version: { source: 'workflow', authorUserId: 'user-1' }, - expectedUpdatedAt: CONTENT_UPDATED_AT, - secretProvenancePolicy: { mode: 'replace', provenance: { status: 'unknown' } }, - } - ) - }) - it('atomically binds append provenance to the exact predecessor version', async () => { const existing = workspaceFile('file-1') mockResolveWorkspaceFileReference.mockResolvedValue(existing) diff --git a/apps/sim/lib/internal/file/operations.ts b/apps/sim/lib/internal/file/operations.ts index aacdd31bc1f..2fc26305e70 100644 --- a/apps/sim/lib/internal/file/operations.ts +++ b/apps/sim/lib/internal/file/operations.ts @@ -1412,14 +1412,6 @@ export async function executeFileManageOperation( leafName, }) if (existing) { - // Writing into a file someone else owns must not hand its owner an - // exact, re-resolvable secret lineage, exactly as appending does. - const overwriteProvenance = - writeProvenance?.status === 'exact' && - writeProvenance.entries.length > 0 && - existing.uploadedBy !== userId - ? { status: 'unknown' as const } - : writeProvenance const { file: overwritten } = await updateWorkspaceFileContent.execute({ principal, input: { @@ -1431,7 +1423,7 @@ export async function executeFileManageOperation( provenanceMode: 'replace_empty', expectedUpdatedAt: existing.contentUpdatedAt ?? undefined, expectedRevision, - ...(overwriteProvenance ? { secretProvenance: overwriteProvenance } : {}), + ...(writeProvenance ? { secretProvenance: writeProvenance } : {}), }, }) @@ -1657,14 +1649,9 @@ export async function executeFileManageOperation( ) } const appendedProvenance = appendedResolution.provenanceBySelection?.get('content') - const secretProvenance = - appendedProvenance?.status === 'exact' && - appendedProvenance.entries.length > 0 && - existing.uploadedBy !== userId - ? { status: 'unknown' as const } - : appendedProvenance - ? mergeWorkspaceFileSecretProvenance(existingProvenance, appendedProvenance) - : undefined + const secretProvenance = appendedProvenance + ? mergeWorkspaceFileSecretProvenance(existingProvenance, appendedProvenance) + : undefined const { content: existingBuffer } = await readWorkspaceFileContent.execute({ principal, input: { diff --git a/apps/sim/lib/knowledge/__integration__/connector-save-concurrency.integration.ts b/apps/sim/lib/knowledge/__integration__/connector-save-concurrency.integration.ts index 300a146f850..77eac5d685a 100644 --- a/apps/sim/lib/knowledge/__integration__/connector-save-concurrency.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/connector-save-concurrency.integration.ts @@ -15,6 +15,7 @@ import { workspace, } from '@sim/db/schema' import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { sha256Hex } from '@sim/security/hash' import { generateId } from '@sim/utils/id' import { and, eq, inArray, isNotNull, isNull, sql } from 'drizzle-orm' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -113,6 +114,78 @@ describe('independent connector saves in one knowledge base', () => { await db.$client.end() }) + it('keeps retained chunk tags consistent during metadata-only connector refresh', async () => { + const item = await source('retry hash') + const chunkId = generateId() + const refreshedTags = { + tag1: 'current', + tag2: null, + number1: 3.5, + boolean1: true, + date1: new Date('2026-10-01T00:00:00.000Z'), + } + await db + .update(document) + .set({ tag1: 'old', tag2: 'cleared', chunkCount: 1 }) + .where(eq(document.id, item.documentId)) + await db.insert(embedding).values({ + id: chunkId, + knowledgeBaseId: ids.knowledgeBaseId, + documentId: item.documentId, + chunkIndex: 0, + chunkHash: sha256Hex('retained content'), + content: 'retained content', + contentLength: 16, + tokenCount: 4, + startOffset: 0, + endOffset: 16, + embedding: Array(1536).fill(0.2), + tag1: 'old', + tag2: 'cleared', + }) + await persistHashOnlyUpdates( + ids.knowledgeBaseId, + ids.connectorId, + [ + { + existingId: item.documentId, + externalId: item.extDoc.externalId, + contentHash: 'refreshed-hash', + sourceMetadata: { sourceUrl: null, sourceModifiedAt: null, ...refreshedTags }, + }, + ], + { stillHeld: () => stillHoldsSyncLock(ids.connectorId, ids.lockId) } + ) + expect( + await db + .select({ + tag1: document.tag1, + tag2: document.tag2, + number1: document.number1, + boolean1: document.boolean1, + date1: document.date1, + count: document.chunkCount, + status: document.processingStatus, + }) + .from(document) + .where(eq(document.id, item.documentId)) + ).toEqual([{ ...refreshedTags, count: 1, status: 'completed' }]) + expect( + await db + .select({ + tag1: embedding.tag1, + tag2: embedding.tag2, + number1: embedding.number1, + boolean1: embedding.boolean1, + date1: embedding.date1, + content: embedding.content, + id: embedding.id, + }) + .from(embedding) + .where(eq(embedding.documentId, item.documentId)) + ).toEqual([{ ...refreshedTags, content: 'retained content', id: chunkId }]) + }) + async function source(kind: SaveKind) { const item = { documentId: generateId(), diff --git a/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts b/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts index 7a411efc12a..833fdf234ee 100644 --- a/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts @@ -12,6 +12,7 @@ import { knowledgeBase, organization, outboxEvent, + permissions, user, workspace, workspaceFiles, @@ -19,7 +20,7 @@ import { import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' import { remoteSandboxMock, remoteSandboxMockFns } from '@sim/testing/mocks/remote-sandbox.mock' import { generateId } from '@sim/utils/id' -import { eq, inArray, sql } from 'drizzle-orm' +import { and, eq, inArray, sql } from 'drizzle-orm' import JSZip from 'jszip' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' @@ -64,7 +65,10 @@ vi.mock('@/lib/embeddings', async () => ({ })) import { functionExecuteBodySchema } from '@/lib/api/contracts' -import { fileManageDecompressBodySchema } from '@/lib/api/contracts/tools/file' +import { + fileManageBodySchema, + fileManageDecompressBodySchema, +} from '@/lib/api/contracts/tools/file' import { observeContentDelivery } from '@/lib/core/application' import { processOutboxEventById } from '@/lib/core/outbox/service' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' @@ -990,9 +994,121 @@ describe('execution archive durable provenance', () => { expect(await isOpaqueWorkspaceFileEgressSafe(ids.workspaceId, source.identity)).toBe(true) }) - it.each([false, true])( - 'refuses tracked unknown execution attachments with historical metadata (archivedOnly=%s)', - async (archivedOnly) => { + it.each(['append', 'write'] as const)( + 'preserves encrypted origins when another member performs %s', + async (operation) => { + const ids = await seed() + await db + .update(permissions) + .set({ permissionType: 'write' }) + .where(and(eq(permissions.userId, ids.bobId), eq(permissions.entityId, ids.workspaceId))) + const priorSecret = 'fixture-prior-origin-secret' + const previous = { + status: 'exact' as const, + entries: [ + { + encryptedValue: (await encryptSecret(priorSecret)).encrypted, + sourceUserId: ids.aliceId, + sourceWorkspaceId: ids.workspaceId, + name: 'PRIOR_TOKEN', + }, + ], + } + const source = await uploadWorkspaceFile( + ids.workspaceId, + ids.aliceId, + Buffer.from(priorSecret), + `${generateId()}.txt`, + 'text/plain', + { secretProvenance: previous, notifyWorkspaceChange: false } + ) + const response = await executeFileManageOperation( + fileManageBodySchema.parse({ + operation, + workspaceId: ids.workspaceId, + fileName: source.name, + content: FIXTURE_SECRET, + ...(operation === 'write' ? { overwrite: true } : {}), + [PRIVATE_SECRET_PROVENANCE_FIELD]: { + version: 1, + complete: true, + selections: [ + { + key: 'content', + provenance: { + version: 1, + complete: true, + scope: { userId: ids.bobId, workspaceId: ids.workspaceId }, + entries: [ + { + name: 'NEW_TOKEN', + encryptedValue: (await encryptSecret(FIXTURE_SECRET)).encrypted, + }, + ], + }, + }, + ], + }, + }), + { + principal: createWorkspaceFileDelegatedPrincipal({ + serviceId: 'executor', + subjectUserId: ids.bobId, + workspaceId: ids.workspaceId, + delegationId: generateId(), + executionId: ids.executionId, + }), + workspaceId: ids.workspaceId, + attributedUserId: ids.bobId, + fileAccessUserId: ids.bobId, + workflowId: ids.workflowId, + executionId: ids.executionId, + headers: new Headers({ + [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, + }), + requestId: generateId(), + } + ) + expect(response.status, await response.text()).toBe(200) + const record = await getWorkspaceFile(ids.workspaceId, source.id) + if (!record) throw new Error('Edited file disappeared') + const bytes = await downloadFile({ key: record.key, context: 'workspace' }) + expect(bytes.toString()).toBe( + operation === 'append' ? priorSecret + FIXTURE_SECRET : FIXTURE_SECRET + ) + const registry = new ResolvedSecretTraceRegistry([], { + userId: ids.bobId, + workspaceId: ids.workspaceId, + }) + expect( + await importWorkspaceFileSecretProvenanceForRuntime({ + workspaceId: ids.workspaceId, + identity: { + fileId: record.id, + key: record.key, + context: 'workspace', + contentUpdatedAt: record.contentUpdatedAt ?? undefined, + }, + registry, + }) + ).toBe(true) + const shown = projectResolvedSecretModelContent(bytes.toString(), registry) + expect(shown.safe).toBe(true) + expect(JSON.stringify(shown)).not.toContain(FIXTURE_SECRET) + expect(JSON.stringify(shown)).not.toContain(priorSecret) + expect(JSON.stringify(shown)).toContain('{{NEW_TOKEN}}') + if (operation === 'append') expect(JSON.stringify(shown)).toContain('[REDACTED_SECRET]') + } + ) + + it.each([ + ['execution', false], + ['execution', true], + ['workspace', true], + ['mothership', true], + ] as const)( + 'retains refusal evidence for %s attachments (archivedOnly=%s)', + async (context, archivedOnly) => { const ids = await seed() const file = await uploadExecutionFile( ids, @@ -1005,7 +1121,7 @@ describe('execution archive durable provenance', () => { if (archivedOnly) { await db .update(workspaceFiles) - .set({ deletedAt: new Date() }) + .set({ deletedAt: new Date(), context }) .where(eq(workspaceFiles.key, file.key)) } else { await db.insert(workspaceFiles).values({ diff --git a/apps/sim/lib/knowledge/__integration__/processing-lock-scope.integration.ts b/apps/sim/lib/knowledge/__integration__/processing-lock-scope.integration.ts index c2586ebf5ce..4d82f09d945 100644 --- a/apps/sim/lib/knowledge/__integration__/processing-lock-scope.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/processing-lock-scope.integration.ts @@ -7,7 +7,16 @@ import { rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' import { db } from '@sim/db' -import { document, embedding, knowledgeBase, organization, user, workspace } from '@sim/db/schema' +import { + document, + embedding, + knowledgeBase, + knowledgeBaseTagDefinitions, + organization, + user, + workspace, +} from '@sim/db/schema' +import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure' import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' import { generateId } from '@sim/utils/id' import { eq, inArray } from 'drizzle-orm' @@ -26,14 +35,21 @@ vi.mock('@/lib/knowledge/documents/processor', () => ({ vi.mock('@/lib/knowledge/embeddings', () => ({ generateEmbeddings: fixtures.embeddings })) import { resolveBillingAttribution } from '@/lib/billing/core/attribution' +import { observeContentDelivery } from '@/lib/core/application' import * as embeddingClient from '@/lib/embeddings/client' import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { KNOWLEDGE_DELEGATION_AUDIENCE } from '@/lib/knowledge/application/authorization' +import { createKnowledgeChunk, listKnowledgeChunks } from '@/lib/knowledge/application/chunks' import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' import { addDocument } from '@/lib/knowledge/connectors/sync-persistence' -import { processDocumentAsync } from '@/lib/knowledge/documents/service' +import { processDocumentAsync, updateDocument } from '@/lib/knowledge/documents/service' +import { deleteAllTagDefinitions } from '@/lib/knowledge/tags/service' +import { createCopilotChatPrincipal } from '@/lib/mothership/auth/application-delegation' +import { createRegistryContentGate } from '@/lib/mothership/content-delivery/registry-gate' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' /** Advisory lock key that parks a processing transaction inside its embedding insert. */ const EMBEDDING_GATE = 7_310_191 @@ -137,6 +153,170 @@ describe('document processing commit lock scope', () => { return resolveBillingAttribution({ actorUserId: ids.aliceId, workspaceId: ids.workspaceId }) } + async function expectCurrentChunks( + documentId: string, + contents: string[], + expectedTag: string | null = 'current' + ) { + const principal = createCopilotChatPrincipal( + { userId: ids.aliceId, workspaceId: ids.workspaceId }, + KNOWLEDGE_DELEGATION_AUDIENCE + ) + const result = await observeContentDelivery( + createRegistryContentGate({ + registry: new ResolvedSecretTraceRegistry([], { + userId: ids.aliceId, + workspaceId: ids.workspaceId, + }), + actorUserId: ids.aliceId, + workspaceId: ids.workspaceId, + }), + () => + listKnowledgeChunks.execute({ + principal, + input: { knowledgeBaseId: ids.knowledgeBaseId, documentId }, + }) + ) + expect(result.chunks.map((row) => row.content).sort()).toEqual([...contents].sort()) + expect(result.chunks.every((row) => row.tag1 === expectedTag)).toBe(true) + } + + it('uses metadata committed while processing waits for embeddings', async () => { + const file = await addConnectorDocument('metadata-during-provider') + await db.update(document).set({ connectorId: null }).where(eq(document.id, file.documentId)) + fixtures.embeddings.mockImplementationOnce(async () => { + await updateDocument(file.documentId, { tag1: 'current' }, 'metadata-during-provider') + return embeddingResult + }) + expect( + await processDocumentAsync(ids.knowledgeBaseId, file.documentId, file, {}, await billing()) + ).toEqual({ outcome: 'indexed' }) + await expectCurrentChunks( + file.documentId, + chunks.map((chunk) => chunk.text) + ) + }) + + it.each(['metadata edit', 'tag deletion'] as const)( + 'applies a waiting %s to the chunks committed by processing', + async (kind) => { + const file = await addConnectorDocument('metadata-during-commit') + await db.update(document).set({ connectorId: null }).where(eq(document.id, file.documentId)) + if (kind === 'tag deletion') { + await db + .insert(knowledgeBaseTagDefinitions) + .values([ + { + id: generateId(), + knowledgeBaseId: ids.knowledgeBaseId, + tagSlot: 'tag1', + displayName: 'Fixture text', + fieldType: 'text', + }, + { + id: generateId(), + knowledgeBaseId: ids.knowledgeBaseId, + tagSlot: 'number1', + displayName: 'Fixture number', + fieldType: 'number', + }, + ]) + .onConflictDoNothing() + await db + .update(document) + .set({ tag1: 'old', number1: 3 }) + .where(eq(document.id, file.documentId)) + } + const gate = postgres(readTestDatabaseUrl(), { max: 1, onnotice: () => undefined }) + let processing: ReturnType | undefined + let editing: Promise | undefined + try { + await gate`SELECT pg_advisory_lock(${EMBEDDING_GATE})` + await installEmbeddingProbe( + file.documentId, + `PERFORM pg_advisory_xact_lock_shared(${EMBEDDING_GATE});` + ) + processing = processDocumentAsync( + ids.knowledgeBaseId, + file.documentId, + file, + {}, + await billing() + ) + let processingPid = 0 + await vi.waitFor( + async () => { + const [row] = await gate>` + SELECT pid FROM pg_locks WHERE locktype = 'advisory' AND objid = ${EMBEDDING_GATE} AND NOT granted` + expect(row?.pid).toBeGreaterThan(0) + processingPid = row?.pid ?? 0 + }, + { timeout: 10_000, interval: 20 } + ) + editing = + kind === 'metadata edit' + ? updateDocument(file.documentId, { tag1: 'current' }, 'metadata-during-commit') + : deleteAllTagDefinitions(ids.knowledgeBaseId, 'tag-deletion-during-commit') + await vi.waitFor( + async () => { + const [row] = await gate>` + SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE ${processingPid} = ANY(pg_blocking_pids(pid))) AS waiting` + expect(row?.waiting).toBe(true) + }, + { timeout: 10_000, interval: 20 } + ) + } finally { + await gate`SELECT pg_advisory_unlock(${EMBEDDING_GATE})` + try { + await Promise.all([processing, editing]) + } finally { + await gate.end() + } + } + await expectCurrentChunks( + file.documentId, + chunks.map((chunk) => chunk.text), + kind === 'metadata edit' ? 'current' : null + ) + if (kind === 'tag deletion') { + const retained = await db + .select({ tag1: embedding.tag1, number1: embedding.number1 }) + .from(embedding) + .where(eq(embedding.documentId, file.documentId)) + expect(retained).toHaveLength(chunks.length) + expect(retained.every((row) => row.tag1 === null && row.number1 === null)).toBe(true) + expect( + await db + .select() + .from(knowledgeBaseTagDefinitions) + .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, ids.knowledgeBaseId)) + ).toEqual([]) + } + } + ) + + it('uses current document tags when a manual chunk finishes its provider request', async () => { + const file = await addConnectorDocument('manual-metadata-during-provider') + await db + .update(document) + .set({ connectorId: null, processingStatus: 'completed' }) + .where(eq(document.id, file.documentId)) + fixtures.embeddings.mockImplementationOnce(async () => { + await updateDocument(file.documentId, { tag1: 'current' }, 'manual-metadata-during-provider') + return { ...embeddingResult, embeddings: [embeddingResult.embeddings[0]] } + }) + await createKnowledgeChunk.execute({ + principal: { kind: 'session', userId: ids.aliceId, sessionId: generateId() }, + input: { + knowledgeBaseId: ids.knowledgeBaseId, + documentId: file.documentId, + content: 'Manual chunk content', + resolveContentProvenance: () => ({ status: 'exact', entries: [] }), + }, + }) + await expectCurrentChunks(file.documentId, ['Manual chunk content']) + }) + it('holds no knowledge_connector lock while writing embeddings', async () => { const file = await addConnectorDocument('lock-scope-fixture') await installConnectorLockProbe() @@ -174,8 +354,8 @@ describe('document processing commit lock scope', () => { * The embedding insert waits on an advisory lock the test holds, so the processing * transaction is parked mid-write while a second connection commits the deletion. */ - const gate = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) - const deleter = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) + const gate = postgres(readTestDatabaseUrl(), { max: 1, onnotice: () => undefined }) + const deleter = postgres(readTestDatabaseUrl(), { max: 1, onnotice: () => undefined }) try { await gate`SELECT pg_advisory_lock(${EMBEDDING_GATE})` await installEmbeddingProbe( diff --git a/apps/sim/lib/knowledge/__integration__/upload-read-provenance.integration.ts b/apps/sim/lib/knowledge/__integration__/upload-read-provenance.integration.ts index b5f53a8a433..5f386ac758e 100644 --- a/apps/sim/lib/knowledge/__integration__/upload-read-provenance.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/upload-read-provenance.integration.ts @@ -27,10 +27,16 @@ vi.mock('@/lib/uploads/core/setup.server', () => ({ }, })) +import { encryptSecret } from '@/lib/core/security/encryption' import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { + executeChatFileBlobCopies, + persistChatFileCopies, + planChatFileCopies, +} from '@/lib/mothership/chat/fork-chat-files' import { generateWorkspaceFileKey, trackChatUpload, @@ -48,8 +54,11 @@ import { snapshotWorkspaceFileSecretProvenanceInTx, type WorkspaceFileSecretProvenance, } from '@/lib/uploads/contexts/workspace/file-secret-provenance' -import { uploadFile } from '@/lib/uploads/core/storage-service' +import { downloadFile, uploadFile } from '@/lib/uploads/core/storage-service' import { readWorkspaceFileText } from '@/lib/workspace-files/application/read-workspace-file-text' +import { executeForkFileBlobCopies } from '@/ee/workspace-forking/lib/copy/copy-files' +import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const fixtures: ReturnType[] = [] const CONTENT = 'The same uploaded bytes remain readable after saving.\n' @@ -139,6 +148,104 @@ afterAll(async () => { }) describe('chat upload reads racing with save_upload', () => { + it.each(['creator', 'workspace'] as const)( + 'keeps byte-identical forked files usable with anonymized encrypted origins across a different %s', + async (kind) => { + const ids = await seedUpload() + const encryptedValue = (await encryptSecret(CONTENT)).encrypted + await db.transaction((tx) => + replaceWorkspaceFileSecretProvenanceInTx(tx, ids.file.id, ids.file.contentUpdatedAt, { + status: 'exact', + entries: [ + { + encryptedValue, + sourceUserId: ids.aliceId, + sourceWorkspaceId: ids.workspaceId, + name: 'ORIGINAL_TOKEN', + }, + ], + }) + ) + const [source] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.id, ids.file.id)) + let copyId: string | undefined + let targetWorkspaceId = ids.workspaceId + let recipientId = ids.bobId + if (kind === 'workspace') { + const target = await seedUpload() + targetWorkspaceId = target.workspaceId + recipientId = target.aliceId + copyId = `wf_${generateId()}` + const copied = await executeForkFileBlobCopies( + [ + { + sourceFileId: source.id, + sourceContentUpdatedAtMs: source.contentUpdatedAt.getTime(), + sourceKey: source.key, + targetKey: generateWorkspaceFileKey(targetWorkspaceId, 'copied.txt'), + targetFileId: copyId, + workspaceId: targetWorkspaceId, + userId: recipientId, + context: 'mothership', + fileName: 'copied.txt', + displayName: null, + contentType: 'text/plain', + size: Buffer.byteLength(CONTENT), + }, + ], + 'copy-provenance-regression' + ) + expect(copied).toMatchObject({ copied: 1, failed: 0 }) + } else { + const newChatId = generateId() + await db.insert(copilotChats).values({ + id: newChatId, + userId: recipientId, + workspaceId: targetWorkspaceId, + type: 'mothership', + }) + const plan = planChatFileCopies({ + rows: [source], + newChatId, + userId: recipientId, + now: new Date(), + }) + const copied = await executeChatFileBlobCopies(plan.blobTasks) + expect(copied.failed).toBe(0) + await db.transaction((tx) => persistChatFileCopies(tx, plan, new Set(copied.failedCopyIds))) + copyId = plan.idMap.get(source.id) + } + if (!copyId) throw new Error('Fork did not map its file') + const [copy] = await db.select().from(workspaceFiles).where(eq(workspaceFiles.id, copyId)) + const bytes = await downloadFile({ key: copy.key, context: 'mothership' }) + expect(bytes.toString()).toBe(CONTENT) + const registry = new ResolvedSecretTraceRegistry([], { + userId: recipientId, + workspaceId: targetWorkspaceId, + }) + expect( + await importWorkspaceFileSecretProvenanceForModelView({ + workspaceId: targetWorkspaceId, + identity: { + fileId: copy.id, + key: copy.key, + context: 'mothership', + contentUpdatedAt: copy.contentUpdatedAt, + }, + registry, + view: 'complete', + value: bytes.toString(), + }) + ).toBe(true) + expect(projectResolvedSecretModelContent(bytes.toString(), registry)).toMatchObject({ + safe: true, + value: '[REDACTED_SECRET]', + }) + } + ) + it.each(['legacy', 'exact', 'unrecorded'] as const)( 'keeps a %s authorized upload read valid across promotion', async (kind) => { diff --git a/apps/sim/lib/knowledge/application/chunks.ts b/apps/sim/lib/knowledge/application/chunks.ts index 719f87d56ed..47a680e495a 100644 --- a/apps/sim/lib/knowledge/application/chunks.ts +++ b/apps/sim/lib/knowledge/application/chunks.ts @@ -107,29 +107,6 @@ function requireChunkWritable(context: ActiveKnowledgeDocumentContext): void { } } -function documentTags(context: ActiveKnowledgeDocumentContext) { - const document = context.document - return { - tag1: document.tag1 ?? null, - tag2: document.tag2 ?? null, - tag3: document.tag3 ?? null, - tag4: document.tag4 ?? null, - tag5: document.tag5 ?? null, - tag6: document.tag6 ?? null, - tag7: document.tag7 ?? null, - number1: document.number1 ?? null, - number2: document.number2 ?? null, - number3: document.number3 ?? null, - number4: document.number4 ?? null, - number5: document.number5 ?? null, - date1: document.date1 ?? null, - date2: document.date2 ?? null, - boolean1: document.boolean1 ?? null, - boolean2: document.boolean2 ?? null, - boolean3: document.boolean3 ?? null, - } -} - export const listKnowledgeChunks = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listChunks, resolveContext: ({ @@ -226,7 +203,6 @@ export const createKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ createChunk( context.knowledgeBaseId, context.documentId, - documentTags(context), { content: input.content, enabled: input.enabled }, generateRequestId(), context.workspaceId, diff --git a/apps/sim/lib/knowledge/chunks/service.ts b/apps/sim/lib/knowledge/chunks/service.ts index b276f06398a..3f8b2bcc6f7 100644 --- a/apps/sim/lib/knowledge/chunks/service.ts +++ b/apps/sim/lib/knowledge/chunks/service.ts @@ -14,7 +14,6 @@ import { searchFilter, textKey, } from '@/lib/api/list-query' -import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' import type { DurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' @@ -181,29 +180,10 @@ export async function queryChunks( } } -/** - * Create a new chunk for a document. - * - * Assigns `chunkIndex` as `max(chunkIndex) + 1` under a transactional - * `pg_advisory_xact_lock` keyed on the document, so concurrent calls for the - * same document serialize instead of computing the same index and colliding - * on the `(document_id, chunk_index)` unique constraint. A `SELECT ... FOR - * UPDATE` on the current max row doesn't prevent that collision, since the - * row it would lock is unrelated to the not-yet-inserted next row. A row - * lock on `document` instead of an advisory lock would also work, but would - * invert the embedding-before-document lock order every other chunk - * mutation path uses (see lock-order.test.ts) — the advisory lock is a - * separate namespace, so it can't deadlock against that convention. - * - * `pg_advisory_xact_lock` auto-releases at transaction end, so there's no - * session lock to leak onto a pooled connection, and `lock_timeout` bounds - * the wait (it raises SQLSTATE 55P03 instead of hanging a pooled connection) - * if a same-document holder is stuck. - */ +/** Assigns the next chunk index under the document row lock, shared by its metadata and chunk writers. */ export async function createChunk( knowledgeBaseId: string, documentId: string, - docTags: Record, chunkData: CreateChunkData, requestId: string, workspaceId?: string | null, @@ -240,10 +220,28 @@ export async function createChunk( await tx.execute( sql`select set_config('lock_timeout', ${`${KB_CHUNK_LOCK_TIMEOUT_MS}ms`}, true)` ) - await acquireAdvisoryXactLock(tx, 'kb_chunk_seq', `kb_chunk_seq:${documentId}`) const activeDocument = await tx - .select({ id: document.id }) + .select({ + id: document.id, + tag1: document.tag1, + tag2: document.tag2, + tag3: document.tag3, + tag4: document.tag4, + tag5: document.tag5, + tag6: document.tag6, + tag7: document.tag7, + number1: document.number1, + number2: document.number2, + number3: document.number3, + number4: document.number4, + number5: document.number5, + date1: document.date1, + date2: document.date2, + boolean1: document.boolean1, + boolean2: document.boolean2, + boolean3: document.boolean3, + }) .from(document) .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) .where( @@ -256,11 +254,14 @@ export async function createChunk( ) ) .limit(1) + .for('update', { of: document }) if (activeDocument.length === 0) { throw new Error('Document not found') } + const docTags = activeDocument[0] + const lastChunk = await tx .select({ chunkIndex: embedding.chunkIndex }) .from(embedding) @@ -284,23 +285,23 @@ export async function createChunk( embeddingModel: kbEmbeddingModel, startOffset: 0, // Manual chunks don't have document offsets endOffset: chunkData.content.length, - tag1: docTags.tag1 as string | null, - tag2: docTags.tag2 as string | null, - tag3: docTags.tag3 as string | null, - tag4: docTags.tag4 as string | null, - tag5: docTags.tag5 as string | null, - tag6: docTags.tag6 as string | null, - tag7: docTags.tag7 as string | null, - number1: docTags.number1 as number | null, - number2: docTags.number2 as number | null, - number3: docTags.number3 as number | null, - number4: docTags.number4 as number | null, - number5: docTags.number5 as number | null, - date1: docTags.date1 as Date | null, - date2: docTags.date2 as Date | null, - boolean1: docTags.boolean1 as boolean | null, - boolean2: docTags.boolean2 as boolean | null, - boolean3: docTags.boolean3 as boolean | null, + tag1: docTags.tag1, + tag2: docTags.tag2, + tag3: docTags.tag3, + tag4: docTags.tag4, + tag5: docTags.tag5, + tag6: docTags.tag6, + tag7: docTags.tag7, + number1: docTags.number1, + number2: docTags.number2, + number3: docTags.number3, + number4: docTags.number4, + number5: docTags.number5, + date1: docTags.date1, + date2: docTags.date2, + boolean1: docTags.boolean1, + boolean2: docTags.boolean2, + boolean3: docTags.boolean3, enabled: chunkData.enabled ?? true, createdAt: now, updatedAt: now, @@ -371,6 +372,11 @@ export async function batchChunkOperation( if (operation === 'delete') { // Handle batch delete with transaction for consistency await db.transaction(async (tx) => { + await tx + .select({ id: document.id }) + .from(document) + .where(eq(document.id, documentId)) + .for('update') // Get chunks to delete for statistics update const chunksToDelete = await tx .select({ @@ -509,6 +515,11 @@ export async function updateChunk( } const result = await db.transaction(async (tx) => { + await tx + .select({ id: document.id }) + .from(document) + .where(eq(document.id, preRead.documentId)) + .for('update') const currentChunk = await tx .select({ documentId: embedding.documentId, @@ -661,6 +672,11 @@ export async function deleteChunk( requestId: string ): Promise { await db.transaction(async (tx) => { + await tx + .select({ id: document.id }) + .from(document) + .where(eq(document.id, documentId)) + .for('update') // Get chunk data before deletion for statistics update const chunkToDelete = await tx .select({ diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index 2355a421be8..700ff71a025 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -28,7 +28,7 @@ import { leaseTransaction, type SyncWriteLease, } from '@/lib/knowledge/connectors/sync-lock' -import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' +import { ALL_TAG_SLOTS, MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' import type { DocumentData } from '@/lib/knowledge/documents/service' import { enqueueKnowledgeStorageCleanup } from '@/lib/knowledge/documents/storage-cleanup' import { @@ -345,6 +345,8 @@ async function isKnowledgeBaseActiveInTx( return rows.length > 0 } +type MappedDocumentTags = Partial> + /** * Resolves tag values from connector metadata using the connector's mapTags function. * Translates semantic keys returned by mapTags to actual DB slots using the @@ -354,7 +356,7 @@ export function resolveTagMapping( connectorType: string, metadata: Record, sourceConfig?: Record -): Partial | undefined { +): MappedDocumentTags | undefined { const config = CONNECTOR_REGISTRY[connectorType] if (!config?.mapTags || !metadata) return undefined @@ -362,7 +364,7 @@ export function resolveTagMapping( const mapping = sourceConfig?.tagSlotMapping as Record | undefined if (!mapping || !semanticTags) return undefined - const result: Partial = {} + const result: MappedDocumentTags = {} for (const [semanticKey, slot] of Object.entries(mapping)) { const value = semanticTags[semanticKey] ;(result as Record)[slot] = @@ -552,7 +554,7 @@ export async function persistSkippedDocuments( } /** Source-derived fields that can change without the indexed text changing. */ -export type SourceMetadataFields = Partial & { +export type SourceMetadataFields = MappedDocumentTags & { sourceUrl: string | null sourceModifiedAt: Date | null } @@ -607,6 +609,29 @@ export async function persistHashOnlyUpdates( .returning({ id: document.id }) if (persisted.length === 0) { missedExternalIds.push(update.externalId) + } else if (update.sourceMetadata) { + const tags = Object.fromEntries( + ALL_TAG_SLOTS.filter((slot) => update.sourceMetadata?.[slot] !== undefined).map( + (slot) => [slot, update.sourceMetadata?.[slot]] + ) + ) + const changed = ALL_TAG_SLOTS.filter((slot) => tags[slot] !== undefined) + if (changed.length > 0) { + await tx + .update(embedding) + .set(tags) + .where( + and( + eq(embedding.documentId, update.existingId), + or( + ...changed.map( + (slot) => + sql`${embedding[slot]} IS DISTINCT FROM ${sql.param(tags[slot], embedding[slot])}` + ) + ) + ) + ) + } } } }) diff --git a/apps/sim/lib/knowledge/documents/lock-order.test.ts b/apps/sim/lib/knowledge/documents/lock-order.test.ts deleted file mode 100644 index b622e0fa6fd..00000000000 --- a/apps/sim/lib/knowledge/documents/lock-order.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Lock-order regression guard: `updateDocument` must lock the document's - * embedding rows BEFORE the document row when cascading tag updates, matching - * the embedding → document order every chunk-mutation path uses - * (chunks/service.ts). The opposite order deadlocks a document tag edit against - * a concurrent chunk edit of the same document. - */ -import { document, embedding } from '@sim/db/schema' -import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' -import { beforeEach, describe, expect, it } from 'vitest' -import { updateDocument } from '@/lib/knowledge/documents/service' - -/** invocationCallOrder of the first `tx.update(table)` call. */ -function updateOrderForTable(table: unknown): number { - const { calls, invocationCallOrder } = dbChainMockFns.update.mock - for (let i = 0; i < calls.length; i++) { - if (calls[i][0] === table) return invocationCallOrder[i] - } - return -1 -} - -describe('updateDocument lock ordering', () => { - beforeEach(() => { - resetDbChainMock() - queueTableRows(document, [ - { id: 'doc-1', knowledgeBaseId: 'kb-1', secretProvenanceVersion: null }, - ]) - dbChainMockFns.returning.mockResolvedValue([ - { id: 'doc-1', knowledgeBaseId: 'kb-1', secretProvenanceVersion: null }, - ]) - }) - - it('updates embeddings before the document row when cascading tag changes', async () => { - await updateDocument('doc-1', { tag1: 'priority' }, 'req-1') - - const embeddingWriteOrder = updateOrderForTable(embedding) - const documentWriteOrder = updateOrderForTable(document) - - expect(embeddingWriteOrder).toBeGreaterThan(0) - expect(documentWriteOrder).toBeGreaterThan(0) - expect(embeddingWriteOrder).toBeLessThan(documentWriteOrder) - }) -}) diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 017aca1c451..a60e2b43d94 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -89,6 +89,7 @@ import { requiresConnectorIndexing, } from '@/lib/knowledge/connectors/indexing-policy' import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' +import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' import { documentConnectorIsActive } from '@/lib/knowledge/documents/connector-lifecycle' import { createEmbeddingCheckpoints } from '@/lib/knowledge/documents/embedding-checkpoints' import { @@ -2066,7 +2067,6 @@ export async function processDocumentAsync( `Embedding generation returned ${embeddings.length} vectors for ${processed.chunks.length} chunks` ) } - const documentTags = ctx logger.info( `[${documentId}] Embeddings generated, creating embedding records with tags` @@ -2082,40 +2082,6 @@ export async function processDocumentAsync( : EXACT_EMPTY_DURABLE_SECRET_PROVENANCE : undefined ) - const embeddingRecords = processed.chunks.map((chunk, chunkIndex) => ({ - id: generateId(), - knowledgeBaseId, - documentId, - chunkIndex, - chunkHash: sha256Hex(chunk.text), - content: chunk.text, - secretProvenanceVersion: chunkProvenances[chunkIndex] ? 1 : null, - contentLength: chunk.text.length, - tokenCount: chunkTokenCounts[chunkIndex], - ...embeddingVectorValues(kbEmbedding.dimensions, embeddings[chunkIndex]), - embeddingModel: kbEmbeddingModel, - startOffset: chunk.metadata.startIndex, - endOffset: chunk.metadata.endIndex, - tag1: documentTags.tag1, - tag2: documentTags.tag2, - tag3: documentTags.tag3, - tag4: documentTags.tag4, - tag5: documentTags.tag5, - tag6: documentTags.tag6, - tag7: documentTags.tag7, - number1: documentTags.number1, - number2: documentTags.number2, - number3: documentTags.number3, - number4: documentTags.number4, - number5: documentTags.number5, - date1: documentTags.date1, - date2: documentTags.date2, - boolean1: documentTags.boolean1, - boolean2: documentTags.boolean2, - boolean3: documentTags.boolean3, - createdAt: now, - updatedAt: now, - })) signal.throwIfAborted() /** @@ -2149,7 +2115,26 @@ export async function processDocumentAsync( * through the embedding foreign key and projection triggers. */ const activeDocument = await tx - .select({ id: document.id }) + .select({ + id: document.id, + tag1: document.tag1, + tag2: document.tag2, + tag3: document.tag3, + tag4: document.tag4, + tag5: document.tag5, + tag6: document.tag6, + tag7: document.tag7, + number1: document.number1, + number2: document.number2, + number3: document.number3, + number4: document.number4, + number5: document.number5, + date1: document.date1, + date2: document.date2, + boolean1: document.boolean1, + boolean2: document.boolean2, + boolean3: document.boolean3, + }) .from(document) .where( and( @@ -2169,6 +2154,42 @@ export async function processDocumentAsync( return false } + const documentTags = activeDocument[0] + const embeddingRecords = processed.chunks.map((chunk, chunkIndex) => ({ + id: generateId(), + knowledgeBaseId, + documentId, + chunkIndex, + chunkHash: sha256Hex(chunk.text), + content: chunk.text, + secretProvenanceVersion: chunkProvenances[chunkIndex] ? 1 : null, + contentLength: chunk.text.length, + tokenCount: chunkTokenCounts[chunkIndex], + ...embeddingVectorValues(kbEmbedding.dimensions, embeddings[chunkIndex]), + embeddingModel: kbEmbeddingModel, + startOffset: chunk.metadata.startIndex, + endOffset: chunk.metadata.endIndex, + tag1: documentTags.tag1, + tag2: documentTags.tag2, + tag3: documentTags.tag3, + tag4: documentTags.tag4, + tag5: documentTags.tag5, + tag6: documentTags.tag6, + tag7: documentTags.tag7, + number1: documentTags.number1, + number2: documentTags.number2, + number3: documentTags.number3, + number4: documentTags.number4, + number5: documentTags.number5, + date1: documentTags.date1, + date2: documentTags.date2, + boolean1: documentTags.boolean1, + boolean2: documentTags.boolean2, + boolean3: documentTags.boolean3, + createdAt: now, + updatedAt: now, + })) + /** * A pass that indexes nothing (a decorative image) still replaces what an * earlier pass indexed, or the document would stay searchable by stale text. @@ -4008,25 +4029,6 @@ export async function updateDocument( boolean2: boolean | null boolean3: boolean | null }> = {} - const ALL_TAG_SLOTS = [ - 'tag1', - 'tag2', - 'tag3', - 'tag4', - 'tag5', - 'tag6', - 'tag7', - 'number1', - 'number2', - 'number3', - 'number4', - 'number5', - 'date1', - 'date2', - 'boolean1', - 'boolean2', - 'boolean3', - ] as const type TagSlot = (typeof ALL_TAG_SLOTS)[number] if (updateData.filename !== undefined) dbUpdateData.filename = updateData.filename @@ -4073,6 +4075,14 @@ export async function updateDocument( }) const doc = await db.transaction(async (tx) => { + const [current] = await tx + .select() + .from(document) + .where(eq(document.id, documentId)) + .limit(1) + .for('update') + if (!current) return undefined + const hasTagUpdates = ALL_TAG_SLOTS.some((field) => typedUpdateData[field] !== undefined) if (hasTagUpdates) { @@ -4091,14 +4101,6 @@ export async function updateDocument( .where(eq(embedding.documentId, documentId)) } - const [current] = await tx - .select() - .from(document) - .where(eq(document.id, documentId)) - .limit(1) - .for('update') - if (!current) return undefined - const [sidecar] = current.secretProvenanceVersion === 1 ? await tx @@ -4484,29 +4486,24 @@ async function hardDeleteDocumentBatch( * embedding delete and the document delete are scoped to this re-verified * ID set rather than the stale `existingIds`. */ - const stillTargetedIds = - scopedConnectorId || scopedKnowledgeBaseId || accessCondition - ? ( - await tx - .select({ id: document.id }) - .from(document) - .where( - and( - inArray(document.id, existingIds), - scopedConnectorId ? eq(document.connectorId, scopedConnectorId) : undefined, - scopedKnowledgeBaseId - ? eq(document.knowledgeBaseId, scopedKnowledgeBaseId) - : undefined, - requireEligibleDocument ? eq(document.userExcluded, false) : undefined, - requireEligibleDocument ? isNull(document.archivedAt) : undefined, - requireVisibleDocument ? isNull(document.deletedAt) : undefined, - accessCondition - ) - ) - .orderBy(asc(document.id)) - .for('update') - ).map((d) => d.id) - : existingIds + const stillTargetedIds = ( + await tx + .select({ id: document.id }) + .from(document) + .where( + and( + inArray(document.id, existingIds), + scopedConnectorId ? eq(document.connectorId, scopedConnectorId) : undefined, + scopedKnowledgeBaseId ? eq(document.knowledgeBaseId, scopedKnowledgeBaseId) : undefined, + requireEligibleDocument ? eq(document.userExcluded, false) : undefined, + requireEligibleDocument ? isNull(document.archivedAt) : undefined, + requireVisibleDocument ? isNull(document.deletedAt) : undefined, + accessCondition + ) + ) + .orderBy(asc(document.id)) + .for('update') + ).map((d) => d.id) await tx.delete(embedding).where(inArray(embedding.documentId, stillTargetedIds)) const deletedRows = await tx diff --git a/apps/sim/lib/knowledge/tags/secret-provenance-delete.test.ts b/apps/sim/lib/knowledge/tags/secret-provenance-delete.test.ts index b9d99d731f7..151e46d66e5 100644 --- a/apps/sim/lib/knowledge/tags/secret-provenance-delete.test.ts +++ b/apps/sim/lib/knowledge/tags/secret-provenance-delete.test.ts @@ -2,7 +2,6 @@ import { document, knowledgeBase, knowledgeBaseTagDefinitions } from '@sim/db/sc import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it } from 'vitest' import { - deleteAllTagDefinitions, deleteTagDefinition, KnowledgeTagProvenanceConflictError, } from '@/lib/knowledge/tags/service' @@ -36,19 +35,4 @@ describe('knowledge tag deletion provenance', () => { expect(dbChainMockFns.update).not.toHaveBeenCalled() expect(dbChainMockFns.delete).not.toHaveBeenCalled() }) - - it('clears every bounded tag slot through the same guarded mutation path', async () => { - queueTableRows(knowledgeBase, [{ id: KNOWLEDGE_BASE_ID }]) - queueTableRows(knowledgeBaseTagDefinitions, [ - { id: 'tag-definition-1', tagSlot: 'tag1' }, - { id: 'tag-definition-2', tagSlot: 'number1' }, - ]) - queueTableRows(document, []) - - await expect(deleteAllTagDefinitions(KNOWLEDGE_BASE_ID, 'request-1')).resolves.toBe(2) - - expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(1) - expect(dbChainMockFns.execute).toHaveBeenCalledTimes(1) - expect(dbChainMockFns.set).toHaveBeenCalledWith({ tag1: null, number1: null }) - }) }) diff --git a/apps/sim/lib/knowledge/tags/service.test.ts b/apps/sim/lib/knowledge/tags/service.test.ts index 068428a5de5..2d6c9c227c7 100644 --- a/apps/sim/lib/knowledge/tags/service.test.ts +++ b/apps/sim/lib/knowledge/tags/service.test.ts @@ -182,19 +182,6 @@ describe('createOrUpdateTagDefinitionsBulk', () => { expect(result.errors).toEqual(['Display name "CLITEST-CAT" already exists']) expect(dbChainMockFns.insert).not.toHaveBeenCalled() }) - - it('reads the snapshot it compares against under the knowledge base row lock', async () => { - queueTableRows(knowledgeBaseTagDefinitions, []) - - await createOrUpdateTagDefinitionsBulk( - 'kb-1', - { definitions: [{ tagSlot: 'tag1', displayName: 'clitest-locked', fieldType: 'text' }] }, - 'request-10' - ) - - expect(dbChainMockFns.transaction).toHaveBeenCalled() - expect(dbChainMockFns.for).toHaveBeenCalledWith('update') - }) }) describe('getTagUsageStats', () => { diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index 6e7fb2381cb..03091a79567 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -73,7 +73,7 @@ async function lockKnowledgeBaseForTagMutation( .from(knowledgeBase) .where(eq(knowledgeBase.id, knowledgeBaseId)) .limit(1) - .for('update') + .for('no key update') } /** Prevents tag cleanup from demoting nonempty or unverifiable provenance to legacy state. */ @@ -112,6 +112,15 @@ async function clearTagSlotsInTx( const clearedTagValues: ClearedTagValues = {} for (const tagSlot of tagSlots) clearedTagValues[tagSlot] = null + await tx.execute(sql` + SELECT count(*) FROM ( + SELECT ${document.id} FROM ${document} + WHERE ${document.knowledgeBaseId} = ${knowledgeBaseId} + ORDER BY ${document.id} + FOR UPDATE + ) AS locked_documents + `) + await assertKnowledgeBaseTagsCanBeClearedInTx(tx, knowledgeBaseId) await tx diff --git a/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts b/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts index 70c74fca5a3..98997432a34 100644 --- a/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts +++ b/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts @@ -141,6 +141,8 @@ import { generateImageServerTool } from '@/lib/mothership/tools/server/image/gen import { generateAudioServerTool } from '@/lib/mothership/tools/server/media/generate-audio' import { generateVideoServerTool } from '@/lib/mothership/tools/server/media/generate-video' import { getShareForResource } from '@/lib/public-shares/manager' +import { TABLE_DELEGATION_AUDIENCE } from '@/lib/table/application/authorization' +import { queryTableRows } from '@/lib/table/application/rows' import { createOwnedWorkspaceFile, getWorkspaceFile, @@ -582,6 +584,45 @@ describe('content reaching Chat is admitted per item', () => { expectPerItem(read, { secretAdmitted: true }) }) + it('exports private evidence only for rows delivered by the mixed Chat page', async () => { + const scope = { userId: fixture.ids.aliceId, workspaceId: ws() } + const registry = new ResolvedSecretTraceRegistry([], scope) + const result = await observeContentDelivery( + createRegistryContentGate({ registry, actorUserId: scope.userId, workspaceId: ws() }), + () => + queryTableRows.execute({ + principal: createCopilotChatPrincipal(scope, TABLE_DELEGATION_AUDIENCE, { + tableId: fixture.table.tableId, + }), + input: { tableId: fixture.table.tableId, includePersistedSecretProvenance: true }, + }) + ) + expect(result.rows.map((row) => row.id).sort()).toEqual( + [fixture.table.rows.clean, fixture.table.rows.entries].sort() + ) + expect(result.secretProvenance?.complete).toBe(true) + const receiver = new ResolvedSecretTraceRegistry([], scope) + expect( + await receiver.importProvenance(result.secretProvenance, { + trusted: true, + origin: 'delivered-table-page', + }) + ).toBe(true) + const projected = projectResolvedSecretModelContent( + result.rows.map((row) => row.data), + receiver + ) + expect(projected.safe).toBe(true) + expect(JSON.stringify(projected)).not.toContain(CANARY.secret) + expect(JSON.stringify(projected)).toContain('{{FIXTURE_TOKEN}}') + const ordinary = await queryTableRows.execute({ + principal: createSessionPrincipal({ userId: scope.userId }), + input: { tableId: fixture.table.tableId, includePersistedSecretProvenance: true }, + }) + expect(ordinary.rows).toHaveLength(3) + expect(ordinary.secretProvenance?.complete).toBe(false) + }) + it('filters and sorts mixed-provenance rows without refusing unrelated content (#18b)', async () => { const table = `/api/v2/tables/${fixture.table.tableId}` const predicate = { field: 'note', op: 'contains', value: 'second' } diff --git a/apps/sim/lib/projects/files/__integration__/content-bulk.integration.ts b/apps/sim/lib/projects/files/__integration__/content-bulk.integration.ts index 1ece5a545e8..9f2f940e04e 100644 --- a/apps/sim/lib/projects/files/__integration__/content-bulk.integration.ts +++ b/apps/sim/lib/projects/files/__integration__/content-bulk.integration.ts @@ -36,6 +36,8 @@ vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock) vi.mock('@/lib/messaging/email/mailer', () => emailMailerMock) import { observeContentDelivery } from '@/lib/core/application/content-delivery' +import { encryptSecret } from '@/lib/core/security/encryption' +import { createDurableSecretProvenanceRegistry } from '@/lib/execution/durable-secret-provenance' import { createProjectFile, readProjectFileContent, @@ -45,6 +47,7 @@ import { createProjectFileFolder } from '@/lib/projects/files/application/folder import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/file-secret-provenance' import * as storage from '@/lib/uploads/core/storage-service' import { observeWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' +import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) @@ -1414,7 +1417,7 @@ describe('Project ZIP extraction against PostgreSQL and local storage', () => { status: 'exact' as const, entries: [ { - encryptedValue: 'archive-fixture', + encryptedValue: (await encryptSecret('extracted-secret-value')).encrypted, sourceUserId: 'origin', }, ], @@ -1422,7 +1425,7 @@ describe('Project ZIP extraction against PostgreSQL and local storage', () => { ] for (const classification of classifications) { const f = await fixture() - const source = await archive(f, { 'data.txt': 'extracted' }, classification) + const source = await archive(f, { 'data.txt': 'extracted-secret-value' }, classification) await extractProjectFile.execute({ principal: f.principal, input: { projectId: f.projectId, fileId: source.file.id }, @@ -1433,15 +1436,23 @@ describe('Project ZIP extraction against PostgreSQL and local storage', () => { .select() .from(workspaceFileSecretProvenance) .where(eq(workspaceFileSecretProvenance.fileId, child.id)) - const expected = - classification.status === 'exact' - ? classification.entries.length - ? 'unknown' - : 'exact' - : classification.status - expect(provenance.status).toBe(expected) - expect(provenance.entries).toEqual([]) - expect(await readFile(join(localStorageRoot, child.key), 'utf8')).toBe('extracted') + expect(provenance.status).toBe(classification.status) + const read = await readProjectFileContent.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: child.id, includeSecretProvenance: true }, + }) + expect(read.content.toString()).toBe('extracted-secret-value') + if (classification.status === 'exact' && classification.entries.length > 0) { + if (read.secretProvenance?.status !== 'exact') + throw new Error('Child lost its recorded evidence') + const registry = await createDurableSecretProvenanceRegistry(read.secretProvenance, { + userId: f.ownerId, + workspaceId: f.workspaceId, + }) + expect( + projectResolvedSecretModelContent(read.content.toString(), registry) + ).toMatchObject({ safe: true, value: '[REDACTED_SECRET]' }) + } else expect(provenance.entries).toEqual([]) } } ) diff --git a/apps/sim/lib/table/application/rows.ts b/apps/sim/lib/table/application/rows.ts index b10607ebbbc..948076cc1ee 100644 --- a/apps/sim/lib/table/application/rows.ts +++ b/apps/sim/lib/table/application/rows.ts @@ -654,7 +654,7 @@ export const queryTableRows = defineAuthorizedTableUseCase({ rowCount: isCopilotPrincipal(principal) ? rows.length : result.rowCount, rows: rowsForReader(principal, rows), ...(input.includePersistedSecretProvenance && readProvenance - ? { secretProvenance: readProvenance.exportProvenance() } + ? { secretProvenance: readProvenance.exportProvenance(rows.map((row) => row.id)) } : {}), } } catch (error) { diff --git a/apps/sim/lib/table/rows/secret-provenance.ts b/apps/sim/lib/table/rows/secret-provenance.ts index 1d49c72e252..e3c24d69664 100644 --- a/apps/sim/lib/table/rows/secret-provenance.ts +++ b/apps/sim/lib/table/rows/secret-provenance.ts @@ -1028,7 +1028,7 @@ function envelopeFromClassifications( export class TableRowProvenanceReader { private readonly accumulator: ResolvedSecretTraceProvenanceAccumulator /** Every row captured, classified on its own. A row never captured has no classification. */ - private readonly rows = new Map() + private readonly rows = new Map() constructor( private readonly scope: ResolvedSecretTraceScopeV1, @@ -1060,25 +1060,28 @@ export class TableRowProvenanceReader { executor ) for (const [rowId, classification] of classifications) { - this.rows.set( - rowId, - classification.status === 'unknown' - ? { status: 'unknown' } - : { - status: 'exact', - entries: classification.entries.map(({ columnId: _column, ...entry }) => entry), - } - ) + this.rows.set(rowId, classification) } this.accumulator.record(envelopeFromClassifications(classifications, rows.length, this.scope)) } - exportProvenance(): ResolvedSecretTraceProvenanceV1 { - return this.accumulator.exportProvenance() + exportProvenance(deliveredRowIds?: readonly string[]): ResolvedSecretTraceProvenanceV1 { + if (!deliveredRowIds) return this.accumulator.exportProvenance() + const delivered = new Map() + for (const rowId of deliveredRowIds) { + delivered.set(rowId, this.rows.get(rowId) ?? { status: 'unknown', cause: 'uncaptured-row' }) + } + return envelopeFromClassifications(delivered, deliveredRowIds.length, this.scope) } /** One captured row's classification; a row this reader never captured is unknown. */ rowProvenance(rowId: string): DurableSecretProvenance { - return this.rows.get(rowId) ?? { status: 'unknown' } + const classification = this.rows.get(rowId) + return !classification || classification.status === 'unknown' + ? { status: 'unknown' } + : { + status: 'exact', + entries: classification.entries.map(({ columnId: _column, ...entry }) => entry), + } } } diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index b0ce412afdc..1c0c18a1ff1 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -881,7 +881,7 @@ export async function upsertRow( mutate: async () => { const [row] = await trx .update(userTableRows) - .set({ data: data.data, updatedAt: now }) + .set(rowContentUpdate(sql`${JSON.stringify(data.data)}::jsonb`, now)) .where( and( eq(userTableRows.id, matchedRowId), @@ -1903,6 +1903,14 @@ function jsonbMergePatch(changedColumnIds: string[], coercedRow: RowData): SQL { return sql`${userTableRows.data} || ${JSON.stringify(patch)}::jsonb` } +function rowContentUpdate(data: SQL, now: Date) { + return { + data, + updatedAt: sql`CASE WHEN ${userTableRows.data} IS DISTINCT FROM ${data} + THEN ${now.toISOString()}::timestamp ELSE ${userTableRows.updatedAt} END`, + } +} + export async function updateRow( data: UpdateRowData, table: TableDefinition, @@ -2064,7 +2072,7 @@ export async function updateRow( patchedColumnIds.size > 0 ? await trx .update(userTableRows) - .set({ data: persistedData, updatedAt: now }) + .set(rowContentUpdate(persistedData, now)) .where(condition) .returning() : await trx.select().from(userTableRows).where(condition).for('update') @@ -2403,10 +2411,7 @@ async function persistBulkUpdateBatch(params: { const batchIds = validIds.slice(index, index + TABLE_LIMITS.UPDATE_BATCH_SIZE) const updated = await trx .update(userTableRows) - .set({ - data: sql`${userTableRows.data} || ${patchJson}::jsonb`, - updatedAt: now, - }) + .set(rowContentUpdate(sql`${userTableRows.data} || ${patchJson}::jsonb`, now)) .where( and( eq(userTableRows.tableId, table.id), @@ -2891,7 +2896,7 @@ export async function batchUpdateRows( const dataPromises = batch.map(({ rowId, changedColumnIds, mergedData }) => trx .update(userTableRows) - .set({ data: jsonbMergePatch(changedColumnIds, mergedData), updatedAt: now }) + .set(rowContentUpdate(jsonbMergePatch(changedColumnIds, mergedData), now)) .where( and( eq(userTableRows.id, rowId), diff --git a/apps/sim/lib/table/rows/writes.integration.ts b/apps/sim/lib/table/rows/writes.integration.ts index 985fb2e4d72..f9bcdb9b9ed 100644 --- a/apps/sim/lib/table/rows/writes.integration.ts +++ b/apps/sim/lib/table/rows/writes.integration.ts @@ -4,7 +4,7 @@ * everywhere; the `rows_version` cases also need the migrated deferred trigger and skip without it. */ import { db } from '@sim/db' -import { userTableDefinitions, userTableRows } from '@sim/db/schema' +import { userTableDefinitions, userTableRowSecretProvenance, userTableRows } from '@sim/db/schema' import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure' import { storageServiceMock, storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' import { tableBillingMock, tableBillingMockFns } from '@sim/testing/mocks/table-billing.mock' @@ -12,7 +12,7 @@ import { tableTriggerMock, tableTriggerMockFns } from '@sim/testing/mocks/table- import { tableWorkflowColumnsMock } from '@sim/testing/mocks/table-workflow-columns.mock' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' -import { sql } from 'drizzle-orm' +import { eq, sql } from 'drizzle-orm' import postgres from 'postgres' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -21,6 +21,7 @@ vi.mock('@/lib/table/trigger', () => tableTriggerMock) vi.mock('@/lib/table/workflow-columns', () => tableWorkflowColumnsMock) vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) +import { encryptSecret } from '@/lib/core/security/encryption' import { deleteColumn, updateColumnConstraints } from '@/lib/table/columns/service' import { getMaxRowSizeBytes } from '@/lib/table/constants' import { bulkInsertImportBatch, importAppendRows, importReplaceRows } from '@/lib/table/import-data' @@ -29,6 +30,7 @@ import type { DbTransaction } from '@/lib/table/planner' import { readCurrentRowsVersion } from '@/lib/table/row-changes' import { decodeCursor } from '@/lib/table/rows/cursor' import { lockLiveTableSchema } from '@/lib/table/rows/live-schema' +import { loadTableRowSecretProvenance } from '@/lib/table/rows/secret-provenance' import { batchInsertRows, batchUpdateRows, @@ -45,6 +47,8 @@ import { getTableById } from '@/lib/table/service' import { getOrCreateTableSnapshot } from '@/lib/table/snapshot-cache' import type { ColumnDefinition, JsonValue, RowData, TableDefinition } from '@/lib/table/types' import { checkBatchUniqueConstraintsDb } from '@/lib/table/validation' +import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const url = readTestDatabaseUrl() if (process.env.DATABASE_URL !== url) { @@ -90,7 +94,9 @@ async function seedRows( tableId: string, rows: Array<{ id: string; data: RowData; orderKey: string | null; position?: number }> ) { - await db.insert(userTableRows).values(rows.map((row) => ({ ...row, tableId, workspaceId }))) + await db + .insert(userTableRows) + .values(rows.map((row) => ({ ...row, tableId, workspaceId, createdAt: new Date(0) }))) } async function rowsVersion(tableId: string): Promise { @@ -229,6 +235,105 @@ describe('table row writes against real PostgreSQL', () => { await control.end() }) + it.each(['upsert', 'single', 'filter', 'batch'] as const)( + 'preserves the content binding of an identical %s write without healing existing faults', + async (kind) => { + const secret = 'fixture-no-op-content-secret' + const encryptedValue = (await encryptSecret(secret)).encrypted + for (const state of ['exact', 'unknown', 'stale'] as const) { + const table = await createTable([ + { id: 'key', name: 'key', type: 'string', unique: true }, + ...textColumns('note'), + ]) + const rowId = generateId() + const data = { key: 'stable', note: secret } + const revision = new Date('2026-10-01T00:00:00.000Z') + await db.insert(userTableRows).values({ + id: rowId, + tableId: table.id, + workspaceId, + data, + orderKey: 'a0', + createdAt: revision, + updatedAt: revision, + secretProvenanceVersion: 1, + }) + await db.insert(userTableRowSecretProvenance).values({ + rowId, + contentUpdatedAt: state === 'stale' ? new Date(0) : revision, + status: state === 'unknown' ? 'unknown' : 'exact', + entries: + state === 'unknown' + ? [] + : [ + { + columnId: 'note', + encryptedValue, + name: 'ROW_TOKEN', + sourceUserId: userId, + sourceWorkspaceId: workspaceId, + }, + ], + }) + const base = { + tableId: table.id, + workspaceId, + capabilityGovernedUserId: null, + secretProvenance: undefined, + } + const write = async (note: string) => { + if (kind === 'upsert') + await upsertRow( + { ...base, data: { ...data, note }, conflictTarget: 'key' }, + table, + 'noop-provenance' + ) + else if (kind === 'single') + await updateRow({ ...base, rowId, data: { note } }, table, 'noop-provenance') + else if (kind === 'filter') { + const result = await updateRowsByFilter( + table, + { + filter: { key: 'stable' }, + data: { note }, + secretProvenance: undefined, + capabilityGovernedUserId: null, + }, + 'noop-provenance' + ) + expect(result.affectedRowIds).toEqual([rowId]) + } else + await batchUpdateRows( + { ...base, updates: [{ rowId, data: { note } }] }, + table, + 'noop-provenance' + ) + } + await write(secret) + const [stored] = await db.select().from(userTableRows).where(eq(userTableRows.id, rowId)) + expect(stored.updatedAt).toEqual(revision) + const scope = { userId, workspaceId } + const evidence = await loadTableRowSecretProvenance([stored], scope) + expect(evidence.complete).toBe(state === 'exact') + if (state === 'exact') { + const registry = new ResolvedSecretTraceRegistry([], scope) + expect( + await registry.importProvenance(evidence, { trusted: true, origin: 'no-op-read' }) + ).toBe(true) + const shown = projectResolvedSecretModelContent(stored.data, registry) + expect(shown).toMatchObject({ safe: true, value: { note: '{{ROW_TOKEN}}' } }) + } + if (migrated) { + await write('changed content') + const [changed] = await db.select().from(userTableRows).where(eq(userTableRows.id, rowId)) + expect(changed.data).toEqual({ key: 'stable', note: 'changed content' }) + expect(changed.updatedAt.getTime()).toBeGreaterThan(revision.getTime()) + expect(changed.secretProvenanceVersion).toBeNull() + } + } + } + ) + describe('unique columns under concurrent writes', () => { const uniqueColumns: ColumnDefinition[] = [ { id: 'email', name: 'email', type: 'string', unique: true }, diff --git a/apps/sim/lib/uploads/contexts/workspace/file-secret-provenance.test.ts b/apps/sim/lib/uploads/contexts/workspace/file-secret-provenance.test.ts index a45b0a0191c..4d92ebe9fa5 100644 --- a/apps/sim/lib/uploads/contexts/workspace/file-secret-provenance.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/file-secret-provenance.test.ts @@ -889,13 +889,13 @@ describe('workspace file secret provenance', () => { ) }) - it('does not remint named provenance into a copied file with a different owner scope', async () => { + it('rejects malformed copied evidence without a source identity', async () => { const targetContentUpdatedAt = new Date('2026-08-04T00:00:02.000Z') dbChainMockFns.limit .mockResolvedValueOnce([ { - userId: 'target-user', - workspaceId: 'target-workspace', + userId: 'source-user', + workspaceId: 'source-workspace', contentUpdatedAt: targetContentUpdatedAt, }, ]) diff --git a/apps/sim/lib/uploads/contexts/workspace/file-secret-provenance.ts b/apps/sim/lib/uploads/contexts/workspace/file-secret-provenance.ts index d31282b58da..eabd9064035 100644 --- a/apps/sim/lib/uploads/contexts/workspace/file-secret-provenance.ts +++ b/apps/sim/lib/uploads/contexts/workspace/file-secret-provenance.ts @@ -4,6 +4,7 @@ import { type WorkspaceFileSecretProvenanceEntry, workspaceFileSecretProvenance, workspaceFiles, + workspaceFileVersion, } from '@sim/db/schema' import { compareStrings } from '@sim/utils/string' import { and, desc, eq, gte, inArray, isNull, lt, or, sql } from 'drizzle-orm' @@ -710,7 +711,7 @@ export async function applyWorkspaceFileSecretProvenancePolicyInTx( return snapshotWorkspaceFileSecretProvenanceInTx(tx, fileId, nextContentUpdatedAt, 1) } -/** Copies exact provenance for a byte-identical, same-owner-scope file copy; otherwise unknown. */ +/** Preserves encrypted origin entries for an authorized copy of the captured file revision. */ export async function copyWorkspaceFileSecretProvenanceInTx( tx: DbTransaction, expectedSource: WorkspaceFileSecretProvenanceCopySource | undefined, @@ -718,8 +719,6 @@ export async function copyWorkspaceFileSecretProvenanceInTx( ): Promise { const [target] = await tx .select({ - userId: workspaceFiles.userId, - workspaceId: workspaceFiles.workspaceId, contentUpdatedAt: workspaceFiles.contentUpdatedAt, }) .from(workspaceFiles) @@ -739,8 +738,6 @@ export async function copyWorkspaceFileSecretProvenanceInTx( const [source] = await tx .select({ key: workspaceFiles.key, - userId: workspaceFiles.userId, - workspaceId: workspaceFiles.workspaceId, secretProvenanceVersion: workspaceFiles.secretProvenanceVersion, fileContentUpdatedAt: workspaceFiles.contentUpdatedAt, provenanceContentUpdatedAt: workspaceFileSecretProvenance.contentUpdatedAt, @@ -780,28 +777,13 @@ export async function copyWorkspaceFileSecretProvenanceInTx( }) return } - /** - * An absence copies as an absence. Folding it in with the refusals below would hand the target a - * taint the source never carried, and a fork or a chat copy would turn a readable file into a - * permanently refused one — the pathology this surface exists to undo, reached by copying. - * - * Safe to carry across the scope checks below, which exist to stop one workspace's secret entries - * landing in another's file. A recorded absence has no entries to carry. - */ if (source.status === 'unrecorded') { await replaceWorkspaceFileSecretProvenanceInTx(tx, targetFileId, target.contentUpdatedAt, { status: 'unrecorded', }) return } - if ( - source.status !== 'exact' || - (source.entries.length > 0 && - (!source.workspaceId || - !target.workspaceId || - source.userId !== target.userId || - source.workspaceId !== target.workspaceId)) - ) { + if (source.status !== 'exact') { await replaceWorkspaceFileSecretProvenanceInTx(tx, targetFileId, target.contentUpdatedAt, { status: 'unknown', }) @@ -818,7 +800,7 @@ export async function markWorkspaceFileSecretProvenanceUnknown( workspaceId: string, fileIds: readonly string[] ): Promise { - const uniqueIds = [...new Set(fileIds.filter((fileId) => fileId.length > 0))] + const uniqueIds = [...new Set(fileIds.filter((fileId) => fileId.length > 0))].sort(compareStrings) if (uniqueIds.length === 0) return await db.transaction(async (tx) => { @@ -833,7 +815,11 @@ export async function markWorkspaceFileSecretProvenanceUnknown( for (let index = 0; index < uniqueIds.length; index += FILE_PROVENANCE_QUERY_CHUNK_SIZE) { const chunk = uniqueIds.slice(index, index + FILE_PROVENANCE_QUERY_CHUNK_SIZE) const rows = await tx - .select({ id: workspaceFiles.id, contentUpdatedAt: workspaceFiles.contentUpdatedAt }) + .select({ + id: workspaceFiles.id, + key: workspaceFiles.key, + contentUpdatedAt: workspaceFiles.contentUpdatedAt, + }) .from(workspaceFiles) .where( and( @@ -843,11 +829,26 @@ export async function markWorkspaceFileSecretProvenanceUnknown( isNull(workspaceFiles.deletedAt) ) ) + .orderBy(workspaceFiles.id) + .for('update') bound += rows.length for (const row of rows) { await replaceWorkspaceFileSecretProvenanceInTx(tx, row.id, row.contentUpdatedAt, { status: 'unknown', }) + await tx + .update(workspaceFileVersion) + .set({ + secretProvenanceStatus: 'unknown', + secretProvenanceEntries: [], + }) + .where( + and( + eq(workspaceFileVersion.fileId, row.id), + eq(workspaceFileVersion.key, row.key), + isNull(workspaceFileVersion.supersededAt) + ) + ) } } /** @@ -1443,12 +1444,7 @@ async function loadModelSafeWorkspaceFileRowChunk( workspaceFileSecretProvenance, eq(workspaceFileSecretProvenance.fileId, workspaceFiles.id) ) - .where( - and( - inArray(workspaceFiles.key, [...keys]), - or(isNull(workspaceFiles.deletedAt), eq(workspaceFiles.context, 'execution')) - ) - ) + .where(inArray(workspaceFiles.key, [...keys])) .orderBy( workspaceFiles.key, sql`${workspaceFiles.deletedAt} IS NULL DESC`, diff --git a/apps/sim/lib/workspace-files/__integration__/copilot-file-versions.integration.ts b/apps/sim/lib/workspace-files/__integration__/copilot-file-versions.integration.ts index 3bd9f212779..db69adc32f0 100644 --- a/apps/sim/lib/workspace-files/__integration__/copilot-file-versions.integration.ts +++ b/apps/sim/lib/workspace-files/__integration__/copilot-file-versions.integration.ts @@ -46,7 +46,10 @@ import { updateWorkspaceFileContent, uploadWorkspaceFile, } from '@/lib/uploads/contexts/workspace/file-manager' -import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { + markWorkspaceFileSecretProvenanceUnknown, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/file-secret-provenance' import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' import { downloadWorkspaceFileStream } from '@/lib/workspace-files/application/download-workspace-file' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' @@ -86,7 +89,10 @@ describe('chat-delegated file version history', () => { }) /** Version 2 holds a named secret; version 3 replaced it with public text. */ - async function seedVersionedFile(historicalProvenance?: WorkspaceFileSecretProvenance) { + async function seedVersionedFile( + historicalProvenance?: WorkspaceFileSecretProvenance, + markHistoricalUnknown = false + ) { const ids = createKnowledgeAclFixtureIds() fixtures.push(ids) await seedKnowledgeAclFixture(ids) @@ -124,6 +130,9 @@ describe('chat-delegated file version history', () => { secretProvenancePolicy: { mode: 'replace', provenance }, } ) + if (markHistoricalUnknown && content.startsWith('token=')) { + await markWorkspaceFileSecretProvenanceUnknown(ids.workspaceId, [file.id]) + } } return { ...ids, fileId: file.id } } @@ -397,6 +406,31 @@ describe('chat-delegated file version history', () => { expect(registry.isPermanentlyIncomplete()).toBe(false) }) + it('retains a classification-only refusal when the current version becomes historical and is reverted', async () => { + const fixture = await seedVersionedFile(undefined, true) + const chat = chatTransport(fixture, fixture.aliceId, registryFor(fixture, fixture.aliceId)) + const older = await chat( + `/api/v2/files/${fixture.fileId}/versions/1/text?workspaceId=${fixture.workspaceId}` + ) + expect(older.status).toBe(200) + const historical = await chat( + `/api/v2/files/${fixture.fileId}/versions/2/text?workspaceId=${fixture.workspaceId}` + ) + expect(historical.status).toBe(403) + expect(await historical.text()).not.toContain(SECRET) + const reverted = await chat(`/api/v2/files/${fixture.fileId}/versions/2/revert`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ workspaceId: fixture.workspaceId }), + }) + expect(reverted.status).toBe(200) + const current = await chat( + `/api/v2/files/${fixture.fileId}/text?workspaceId=${fixture.workspaceId}` + ) + expect(current.status).toBe(403) + expect(await current.text()).not.toContain(SECRET) + }) + it('keeps a secret redacted after Chat reverts the file to the version holding it', async () => { const fixture = await seedVersionedFile() const reverted = await chatTransport(fixture, fixture.aliceId)( diff --git a/apps/sim/lib/workspace-files/archive-extraction.ts b/apps/sim/lib/workspace-files/archive-extraction.ts index 54a389d6de2..2530f27821d 100644 --- a/apps/sim/lib/workspace-files/archive-extraction.ts +++ b/apps/sim/lib/workspace-files/archive-extraction.ts @@ -129,11 +129,6 @@ export async function commitFileArchiveInTx( tx ) const folders = new Map([['[]', root.id]]) - const provenance: WorkspaceFileSecretProvenance = - args.secretProvenance.status === 'unrecorded' || - (args.secretProvenance.status === 'exact' && args.secretProvenance.entries.length === 0) - ? args.secretProvenance - : { status: 'unknown' } for (const entry of args.staged.entries) { args.signal.throwIfAborted() let parentId = root.id @@ -161,7 +156,7 @@ export async function commitFileArchiveInTx( staged: entry.content, folderId: parentId, exactName: false, - secretProvenance: provenance, + secretProvenance: args.secretProvenance, }) } args.signal.throwIfAborted() diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index af850e30cb3..1307ccb7f40 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -1475,7 +1475,6 @@ "apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts": 3, "apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts": 22, "apps/sim/lib/knowledge/__integration__/ocr-input-failures.integration.ts": 2, - "apps/sim/lib/knowledge/__integration__/processing-lock-scope.integration.ts": 2, "apps/sim/lib/knowledge/__integration__/provider-admission.integration.ts": 1, "apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts": 3, "apps/sim/lib/knowledge/__integration__/providers-live.integration.ts": 10,