diff --git a/apps/sim/app/api/copilot/chat/resources/route.ts b/apps/sim/app/api/copilot/chat/resources/route.ts index 1cb4f71eea8..c05e09cdd80 100644 --- a/apps/sim/app/api/copilot/chat/resources/route.ts +++ b/apps/sim/app/api/copilot/chat/resources/route.ts @@ -5,12 +5,13 @@ import { } from '@/lib/api/contracts/copilot' import { defineInternalJsonRoute, - internalJsonPresenters, internalOrchestrationErrorPolicy, internalRateLimits, internalSessionAuth, } from '@/lib/api/server/routes' import { changeChatResources } from '@/lib/mothership/chat/application/change-resources' +import { presentChatResourceForBrowser } from '@/lib/mothership/resources/presentation' +import type { MothershipResource } from '@/lib/mothership/resources/types' const policy = { auth: internalSessionAuth, @@ -20,7 +21,10 @@ const policy = { }), errorPolicy: internalOrchestrationErrorPolicy, useCase: changeChatResources, - present: internalJsonPresenters.withSuccess, + present: ({ resources }: { resources: MothershipResource[] }) => ({ + success: true as const, + resources: resources.map(presentChatResourceForBrowser), + }), } export const POST = defineInternalJsonRoute({ @@ -53,7 +57,14 @@ export const DELETE = defineInternalJsonRoute({ chatId: body.chatId, change: { kind: 'remove' as const, - resources: [{ type: body.resourceType, id: body.resourceId, workspaceId: body.workspaceId }], + resources: [ + { + type: body.resourceType, + id: body.resourceId, + workspaceId: body.workspaceId, + owner: body.owner, + }, + ], }, }), }) diff --git a/apps/sim/app/api/mothership/chats/[chatId]/route.ts b/apps/sim/app/api/mothership/chats/[chatId]/route.ts index eacabc1f83d..b6c61f53fe8 100644 --- a/apps/sim/app/api/mothership/chats/[chatId]/route.ts +++ b/apps/sim/app/api/mothership/chats/[chatId]/route.ts @@ -30,6 +30,7 @@ import { createInternalServerErrorResponse, createUnauthorizedResponse, } from '@/lib/mothership/request/http' +import { presentChatResourceForBrowser } from '@/lib/mothership/resources/presentation' import { captureServerEvent } from '@/lib/posthog/server' const logger = createLogger('MothershipChatAPI') @@ -107,7 +108,9 @@ export const GET = withRouteHandler( mode: chat.mode, messages: effectiveMessages, activeStreamId: liveStreamId, - resources: Array.isArray(chat.resources) ? chat.resources : [], + resources: Array.isArray(chat.resources) + ? chat.resources.map(presentChatResourceForBrowser) + : [], effort: chat.effort, olderCursor: chat.olderCursor, windowStartCursor: chat.windowStartCursor, diff --git a/apps/sim/app/api/v2/files/[fileId]/metadata/route.test.ts b/apps/sim/app/api/v2/files/[fileId]/metadata/route.test.ts index 1534cc41728..697796ca2ce 100644 --- a/apps/sim/app/api/v2/files/[fileId]/metadata/route.test.ts +++ b/apps/sim/app/api/v2/files/[fileId]/metadata/route.test.ts @@ -5,20 +5,18 @@ import { v2RateLimiterModuleMock, v2RouteMocks, } from '@sim/testing' +import { + readWorkspaceFileMetadataMock, + readWorkspaceFileMetadataMockFns, +} from '@sim/testing/mocks/read-workspace-file-metadata.mock' import { usersQueriesMock, usersQueriesMockFns } from '@sim/testing/mocks/users-queries.mock' import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' -const mocks = vi.hoisted(() => ({ - readMetadata: vi.fn(), -})) - -vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', () => ({ - readWorkspaceFileMetadataWithVersion: { - operation: { id: 'files.read_metadata', minimumRole: 'read', workspaceApiKey: 'allow' }, - execute: mocks.readMetadata, - }, -})) +vi.mock( + '@/lib/workspace-files/application/read-workspace-file-metadata', + () => readWorkspaceFileMetadataMock +) vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => v2ApiKeyAuthModuleMock) vi.mock('@/lib/core/rate-limiter', () => v2RateLimiterModuleMock) @@ -30,6 +28,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { GET } from '@/app/api/v2/files/[fileId]/metadata/route' const { mockGetUserEmailsByIds } = usersQueriesMockFns +const readMetadata = readWorkspaceFileMetadataMockFns.mockReadWorkspaceFileMetadataWithVersion const WORKSPACE_ID = 'workspace-1' const FILE_ID = 'wf_1' @@ -89,7 +88,7 @@ describe('GET /api/v2/files/[fileId]/metadata', () => { v2RouteMocks.authenticate.mockResolvedValue(auth) v2RouteMocks.preauthRate.mockResolvedValue(V2_PREAUTH_RATE_LIMIT_ALLOWED) v2RouteMocks.operationRate.mockResolvedValue(V2_OPERATION_RATE_LIMIT_ALLOWED) - mocks.readMetadata.mockResolvedValue({ + readMetadata.mockResolvedValue({ file: { ...buildRecord(), currentVersion: 3 }, share: SHARE, }) @@ -97,7 +96,7 @@ describe('GET /api/v2/files/[fileId]/metadata', () => { }) it('conceals cross-workspace authorization as not found', async () => { - mocks.readMetadata.mockRejectedValue(new NoWorkspaceAccessError()) + readMetadata.mockRejectedValue(new NoWorkspaceAccessError()) const response = await callGet(`workspaceId=${WORKSPACE_ID}`) @@ -106,13 +105,13 @@ describe('GET /api/v2/files/[fileId]/metadata', () => { }) it('leaves an archived file unreachable when scope is omitted', async () => { - mocks.readMetadata.mockImplementation(archivedFileUseCase) + readMetadata.mockImplementation(archivedFileUseCase) const response = await callGet(`workspaceId=${WORKSPACE_ID}`) expect(response.status).toBe(404) expect((await response.json()).error.code).toBe('NOT_FOUND') - expect(mocks.readMetadata).toHaveBeenCalledWith( + expect(readMetadata).toHaveBeenCalledWith( expect.objectContaining({ input: { fileId: FILE_ID, assertedWorkspaceId: WORKSPACE_ID, includeDeleted: false }, }) @@ -120,13 +119,13 @@ describe('GET /api/v2/files/[fileId]/metadata', () => { }) it('leaves an archived file unreachable under an explicit scope=active', async () => { - mocks.readMetadata.mockImplementation(archivedFileUseCase) + readMetadata.mockImplementation(archivedFileUseCase) const response = await callGet(`workspaceId=${WORKSPACE_ID}&scope=active`) expect(response.status).toBe(404) expect((await response.json()).error.code).toBe('NOT_FOUND') - expect(mocks.readMetadata).toHaveBeenCalledWith( + expect(readMetadata).toHaveBeenCalledWith( expect.objectContaining({ input: { fileId: FILE_ID, assertedWorkspaceId: WORKSPACE_ID, includeDeleted: false }, }) @@ -134,7 +133,7 @@ describe('GET /api/v2/files/[fileId]/metadata', () => { }) it('returns archived metadata when scope=archived opts into the archived set', async () => { - mocks.readMetadata.mockImplementation(archivedFileUseCase) + readMetadata.mockImplementation(archivedFileUseCase) const response = await callGet(`workspaceId=${WORKSPACE_ID}&scope=archived`) @@ -157,7 +156,7 @@ describe('GET /api/v2/files/[fileId]/metadata', () => { revision: expect.any(String), }, }) - expect(mocks.readMetadata).toHaveBeenCalledWith( + expect(readMetadata).toHaveBeenCalledWith( expect.objectContaining({ principal: auth.principal, input: { fileId: FILE_ID, assertedWorkspaceId: WORKSPACE_ID, includeDeleted: true }, diff --git a/apps/sim/app/api/v2/projects/[projectId]/files/route.ts b/apps/sim/app/api/v2/projects/[projectId]/files/route.ts index b4a81170233..332c4d8dc8b 100644 --- a/apps/sim/app/api/v2/projects/[projectId]/files/route.ts +++ b/apps/sim/app/api/v2/projects/[projectId]/files/route.ts @@ -11,10 +11,10 @@ import { v2OrchestrationErrorPolicy, v2RateLimits, } from '@/lib/api/server/routes' +import { toCreateProjectFileInput } from '@/lib/projects/files/api' import { toV2ProjectFile } from '@/lib/projects/files/api/presenters' import { createProjectFile, listProjectFiles } from '@/lib/projects/files/application' import { projectFileOperations } from '@/lib/projects/files/application/operations' -import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file' import { MAX_WORKSPACE_FILE_INLINE_BODY_BYTES } from '@/lib/workspace-files/orchestration' import { readSortedCursor, writeSortedCursor } from '@/app/api/v2/lib/response' @@ -68,15 +68,7 @@ export const POST = defineV2JsonRoute({ rateLimit: v2RateLimits.publicApi, errorPolicy: v2OrchestrationErrorPolicy, parseOptions: { maxBodyBytes: MAX_WORKSPACE_FILE_INLINE_BODY_BYTES }, - mapInput: ({ params, body }) => ({ - projectId: params.projectId, - name: body.name, - contentType: body.contentType ?? getMimeTypeFromExtension(getFileExtension(body.name)), - content: body.content, - encoding: body.encoding, - folderPath: body.folderPath ?? '/', - exactName: true, - }), + mapInput: ({ params, body }) => toCreateProjectFileInput(params.projectId, body), useCase: createProjectFile, present: ({ file }) => ({ data: toV2ProjectFile(file) }), }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/chat-context-kind-registry/chat-context-kind-registry.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/chat-context-kind-registry/chat-context-kind-registry.tsx index 80ca0db6ef1..d7f59e60af6 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/chat-context-kind-registry/chat-context-kind-registry.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/chat-context-kind-registry/chat-context-kind-registry.tsx @@ -108,6 +108,10 @@ export const CHAT_CONTEXT_KIND_REGISTRY: Record , }, + project: { + label: 'Project', + renderIcon: ({ className }) => , + }, workspace: { label: 'Workspace', renderIcon: ({ className }) => , diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts index a8c60f3d8d1..e0f457806eb 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts @@ -467,6 +467,34 @@ describe('organization resource stream', () => { }) }) +describe('owner-addressed effects on a legacy tab', () => { + const legacyTab: MothershipResource = { type: 'file', id: 'f', title: 'notes' } + const owner = { entityType: 'workspace' as const, entityId: 'ws' } + + function applyOwnerEffect(op: 'upsert' | 'remove') { + const deps = makeStreamLoopDeps({ workspaceId: 'ws', resourcesRef: ref([legacyTab]) }) + handleResourceEvent( + { deps } as StreamLoopContext, + { + ...removeEvent('file', 'f'), + payload: { op, effectId: `effect:${op}`, resource: { type: 'file', id: 'f', owner } }, + } as ResourceEvent + ) + const updater = (deps.setResources as ReturnType).mock.calls[0][0] as ( + current: MothershipResource[] + ) => MothershipResource[] + return updater([legacyTab]) + } + + it('closes the tab when the removal names its workspace owner', () => { + expect(applyOwnerEffect('remove')).toEqual([]) + }) + + it('updates the tab in place instead of opening a duplicate', () => { + expect(applyOwnerEffect('upsert')).toHaveLength(1) + }) +}) + it('seeds only fresh matching search effects and never seeds replayed or foreign evidence', () => { const deps = makeStreamLoopDeps({ workspaceId: undefined, diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts index 63b5c13ce35..1930055f347 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts @@ -8,7 +8,7 @@ import type { FilePreviewSession } from '@/lib/mothership/request/session' import type { PersistedStreamEventEnvelope } from '@/lib/mothership/request/session/contract' import { searchResourceMatchesOwner } from '@/lib/mothership/resources/search' import { - getChatResourceKey, + chatResourceKeyFor, getChatResourceSelectionId, mergeChatResource, } from '@/lib/mothership/resources/types' @@ -102,6 +102,7 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven } = ctx.deps const onResourceEvent = onResourceEventRef.current const payload = parsed.payload + const keyOf = chatResourceKeyFor(chatWorkspaceId) if (payload.op === 'upsert' && payload.readOnly) return if (payload.resource.type === 'settings') { const settings = payload.resource @@ -135,9 +136,7 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven search, } if (payload.op === 'remove') { - setResources((current) => - current.filter((item) => getChatResourceKey(item) !== getChatResourceKey(resource)) - ) + setResources((current) => current.filter((item) => keyOf(item) !== keyOf(resource))) return } const scopeKey = @@ -181,9 +180,7 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven if (ctx.deps.citedSourcesEnabled) return if (payload.effectId) setResources((current) => { - const found = current.find( - (item) => getChatResourceKey(item) === getChatResourceKey(resource) - ) + const found = current.find((item) => keyOf(item) === keyOf(resource)) return found ? current.map((item) => (item === found ? mergeChatResource(item, resource) : item)) : [...current, resource] @@ -257,7 +254,7 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven const resourceType = resource.type if (payload.effectId) { applyResourceEffect(ctx, (current) => - current.filter((item) => getChatResourceKey(item) !== getChatResourceKey(resource)) + current.filter((item) => keyOf(item) !== keyOf(resource)) ) ctx.deps.setActiveResourceId((current) => current === getChatResourceSelectionId(resource) ? null : current @@ -308,13 +305,11 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven !shouldAutoActivatePreviewSession(previewForResource))) const wasAdded = shouldSuppressFileResourceActivation || payload.effectId - ? !resourcesRef.current.some((r) => getChatResourceKey(r) === getChatResourceKey(resource)) + ? !resourcesRef.current.some((r) => keyOf(r) === keyOf(resource)) : addResource(resourceUpdate) if (payload.effectId) { applyResourceEffect(ctx, (current) => { - const previous = current.find( - (item) => getChatResourceKey(item) === getChatResourceKey(resource) - ) + const previous = current.find((item) => keyOf(item) === keyOf(resource)) const next = { ...previous, ...resource, @@ -326,9 +321,7 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven }) } else if (shouldSuppressFileResourceActivation && wasAdded) { setResources((current) => - current.some((r) => getChatResourceKey(r) === getChatResourceKey(resource)) - ? current - : [...current, resource] + current.some((r) => keyOf(r) === keyOf(resource)) ? current : [...current, resource] ) } if (completedPreviewHandoff && isCompletedPreviewHandoffCurrent) { @@ -341,14 +334,8 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven // report "added", while only the first updater actually inserted — the // updater is idempotent, so it simply runs every time. setResources((current) => - current.some( - (r) => getChatResourceKey(r) === getChatResourceKey(resource) && r.viewId !== pinnedViewId - ) - ? current.map((r) => - getChatResourceKey(r) === getChatResourceKey(resource) - ? { ...r, viewId: pinnedViewId } - : r - ) + current.some((r) => keyOf(r) === keyOf(resource) && r.viewId !== pinnedViewId) + ? current.map((r) => (keyOf(r) === keyOf(resource) ? { ...r, viewId: pinnedViewId } : r)) : current ) // Consumed by the embedded table once its views list carries the view — diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx index 6e4ab2eb4e0..c8c733515d7 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx @@ -90,7 +90,10 @@ import { } from '@/lib/core/config/deployment-shape' import { MothershipHandoffStorage } from '@/lib/core/utils/browser-storage' import { MOTHERSHIP_STREAM_REPLAY_HEADER } from '@/lib/mothership/constants' -import type { MothershipStreamV1EventEnvelope } from '@/lib/mothership/generated/mothership-stream-v1' +import { + MothershipStreamV1CompletionStatus, + type MothershipStreamV1EventEnvelope, +} from '@/lib/mothership/generated/mothership-stream-v1' import type { PersistedStreamEventEnvelope } from '@/lib/mothership/request/session/contract' import { getChatResourceSelectionId } from '@/lib/mothership/resources/types' import { ChatSurfaceProvider } from '@/app/workspace/[workspaceId]/home/components/chat-surface-context' @@ -942,7 +945,9 @@ describe('useChat remount send recovery', () => { id: 'chat-reconnecting-search', title: 'Search', activeStreamId: 'accepted-search', - messages: [{ id: 'accepted-search', role: 'user', content: 'Find the policy' }], + messages: [ + { id: 'accepted-search', role: 'user', content: 'Find the policy', timestamp: '' }, + ], resources: [ { type: 'search', @@ -4860,7 +4865,12 @@ describe('useChat remount send recovery', () => { title: 'Trace inspection', activeStreamId: status === 'active' ? 'accepted-request' : null, messages: [ - { id: 'accepted-request', role: 'user', content: 'check the trace for this req' }, + { + id: 'accepted-request', + role: 'user', + content: 'check the trace for this req', + timestamp: '', + }, ...(status === 'active' ? [] : [ @@ -4868,11 +4878,14 @@ describe('useChat remount send recovery', () => { id: 'accepted-answer', role: 'assistant' as const, content: '', + timestamp: '', contentBlocks: [ { type: 'complete' as const, status: - status === 'cancelled' ? ('cancelled' as const) : ('success' as const), + status === 'cancelled' + ? MothershipStreamV1CompletionStatus.cancelled + : MothershipStreamV1CompletionStatus.complete, }, ], }, @@ -4898,7 +4911,12 @@ describe('useChat remount send recovery', () => { title: 'Trace inspection', activeStreamId: null, messages: [ - { id: 'different-request', role: 'user', content: 'check the trace for this req' }, + { + id: 'different-request', + role: 'user', + content: 'check the trace for this req', + timestamp: '', + }, ], resources: [], }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts index d0cfccfd34c..c8879bc4ca0 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts @@ -70,7 +70,7 @@ import type { FilePreviewSession } from '@/lib/mothership/request/session/file-p import { canDisplayResource } from '@/lib/mothership/resources/availability' import { ResourcePersistenceQueue } from '@/lib/mothership/resources/client-persistence-queue' import { - getChatResourceKey, + chatResourceKeyFor, getChatResourceSelectionId, isAddressableResource, isEphemeralResource, @@ -836,13 +836,13 @@ export function useChat( return { ...current, resources: pendingOrder - ? (reorderStoredChatResources(resources, pendingOrder) ?? resources) + ? (reorderStoredChatResources(resources, pendingOrder, workspaceId) ?? resources) : resources, } }) void queryClient.invalidateQueries(query) }, - [queryClient] + [queryClient, workspaceId] ) if (!resourcePersistenceQueueRef.current) { resourcePersistenceQueueRef.current = new ResourcePersistenceQueue({ @@ -1428,9 +1428,8 @@ export function useChat( }) return false } - const existing = resourcesRef.current.find( - (r) => getChatResourceKey(r) === getChatResourceKey(resourceUpdate) - ) + const keyOf = chatResourceKeyFor(workspaceId) + const existing = resourcesRef.current.find((r) => keyOf(r) === keyOf(resourceUpdate)) const resource = mergeChatResource(existing, resourceUpdate) const persistChatId = chatIdRef.current ?? selectedChatIdRef.current if (persistChatId && !isEphemeralResource(resource)) { @@ -1438,17 +1437,13 @@ export function useChat( mothershipChatKeys.detail(persistChatId), (current) => { if (!current) return current - const cached = current.resources.find( - (item) => getChatResourceKey(item) === getChatResourceKey(resource) - ) + const cached = current.resources.find((item) => keyOf(item) === keyOf(resource)) const merged = mergeChatResource(cached, resourceUpdate) if (cached === merged) return current return { ...current, resources: cached - ? current.resources.map((item) => - getChatResourceKey(item) === getChatResourceKey(resource) ? merged : item - ) + ? current.resources.map((item) => (keyOf(item) === keyOf(resource) ? merged : item)) : [...current.resources, merged], } } @@ -1459,12 +1454,12 @@ export function useChat( } setResources((prev) => { - const current = prev.find((r) => getChatResourceKey(r) === getChatResourceKey(resource)) + const current = prev.find((r) => keyOf(r) === keyOf(resource)) if (!current) return [...prev, resource] const merged = mergeChatResource(current, resourceUpdate) return merged === current ? prev - : prev.map((r) => (getChatResourceKey(r) === getChatResourceKey(resource) ? merged : r)) + : prev.map((r) => (keyOf(r) === keyOf(resource) ? merged : r)) }) // Synthetic result/preview panels are in-memory only. The browser tab // metadata is persisted even though its live page remains desktop-owned. @@ -1476,7 +1471,7 @@ export function useChat( resourcePersistenceQueue.enqueue(resourceUpdate, persistChatId, persistenceScopeId, existing) return existing === undefined }, - [queryClient, resourcePersistenceQueue] + [queryClient, resourcePersistenceQueue, workspaceId] ) const removeResource = useCallback( @@ -2121,6 +2116,7 @@ export function useChat( const persistedResources = sanitizeChatResources( chatHistory.resources.filter((r) => r.id !== 'streaming-file') ) + const keyOf = chatResourceKeyFor(workspaceId) let updatedResources = resourcePersistenceQueue.applyPendingUpdates( chatHistory.id, persistedResources @@ -2135,9 +2131,7 @@ export function useChat( if ( resource.type === 'search' && getChatResourceSelectionId(resource) !== selectedResourceIdRef.current && - !resourcesRef.current.some( - (visible) => getChatResourceKey(visible) === getChatResourceKey(resource) - ) + !resourcesRef.current.some((visible) => keyOf(visible) === keyOf(resource)) ) { removeResource('search', resource.id, resource.workspaceId) } @@ -2152,7 +2146,8 @@ export function useChat( // desktop app still gets it back. const pendingOrder = pendingResourceReorders.get(chatHistory.id) const projectedResources = pendingOrder - ? (reorderStoredChatResources(updatedResources, pendingOrder) ?? updatedResources) + ? (reorderStoredChatResources(updatedResources, pendingOrder, workspaceId) ?? + updatedResources) : updatedResources const keepSearchPanelStable = requestModeRef.current === 'assistant' && @@ -2162,9 +2157,7 @@ export function useChat( .filter(canDisplayResource) .flatMap((resource) => { if (!keepSearchPanelStable || resource.type !== 'search') return [resource] - const visible = resourcesRef.current.find( - (item) => getChatResourceKey(item) === getChatResourceKey(resource) - ) + const visible = resourcesRef.current.find((item) => keyOf(item) === keyOf(resource)) if (visible) return [visible] return getChatResourceSelectionId(resource) === selectedResourceIdRef.current ? [resource] @@ -2173,11 +2166,11 @@ export function useChat( undisplayableResourcesRef.current = persistedResources.filter((r) => !canDisplayResource(r)) // Keyed on everything the server holds, not just what is restorable, so a // resource being hidden cannot make it look local-only and get re-added. - const serverKeys = new Set(persistedResources.map(getChatResourceKey)) + const serverKeys = new Set(persistedResources.map((r) => keyOf(r))) const localOnly = resourcesRef.current.filter( (r) => r.id !== 'streaming-file' && - !serverKeys.has(getChatResourceKey(r)) && + !serverKeys.has(keyOf(r)) && (isEphemeralResource(r) || resourcePersistenceQueue.hasPendingUpsert(chatHistory.id, r.type, r.id, r.workspaceId)) ) @@ -2186,14 +2179,10 @@ export function useChat( // keep their current on-screen position — hydration reruns on every send // and stream completion, and appending them at the end made those tabs // visibly jump/flash each time. - const localOnlyKeys = new Set(localOnly.map(getChatResourceKey)) - const mergedResources = restorableResources.filter( - (r) => !localOnlyKeys.has(getChatResourceKey(r)) - ) + const localOnlyKeys = new Set(localOnly.map((r) => keyOf(r))) + const mergedResources = restorableResources.filter((r) => !localOnlyKeys.has(keyOf(r))) for (const resource of localOnly) { - const currentIndex = resourcesRef.current.findIndex( - (r) => getChatResourceKey(r) === getChatResourceKey(resource) - ) + const currentIndex = resourcesRef.current.findIndex((r) => keyOf(r) === keyOf(resource)) const insertAt = currentIndex < 0 ? mergedResources.length : Math.min(currentIndex, mergedResources.length) mergedResources.splice(insertAt, 0, resource) diff --git a/apps/sim/executor/utils/resolved-secret-trace-registry.ts b/apps/sim/executor/utils/resolved-secret-trace-registry.ts index d5408ec0d88..04f804a32ad 100644 --- a/apps/sim/executor/utils/resolved-secret-trace-registry.ts +++ b/apps/sim/executor/utils/resolved-secret-trace-registry.ts @@ -77,6 +77,7 @@ export type ResolvedSecretIncompletenessReason = | 'table-run-state-provenance-unavailable' | 'mounted-file-provenance-unavailable' | 'workspace-file-provenance-unknown' + | 'project-file-provenance-unavailable' | 'file-source-unidentified' | 'table-snapshot-unsafe-for-mount' | 'restored-provenance-untrusted' diff --git a/apps/sim/lib/api/contracts/copilot.ts b/apps/sim/lib/api/contracts/copilot.ts index 72a5e3fbb27..dab30ada290 100644 --- a/apps/sim/lib/api/contracts/copilot.ts +++ b/apps/sim/lib/api/contracts/copilot.ts @@ -1,7 +1,10 @@ import { z } from 'zod' import { persistedContentBlockSchema } from '@/lib/api/contracts/copilot-messages' import { workspaceSearchFiltersSchema } from '@/lib/api/contracts/knowledge/search' -import { mothershipResourceSchema } from '@/lib/api/contracts/mothership-resources' +import { + mothershipResourceRemovalSchema, + mothershipResourceSchema, +} from '@/lib/api/contracts/mothership-resources' import { requiredFieldSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' import { usageUpgradePayloadSchema } from '@/lib/api/contracts/subscription' import { type ContractJsonResponse, defineRouteContract } from '@/lib/api/contracts/types' @@ -120,11 +123,8 @@ export const addCopilotChatResourceBodySchema = z }) export type AddCopilotChatResourceBody = z.input -export const removeCopilotChatResourceBodySchema = z.object({ - workspaceId: workspaceIdSchema.optional(), +export const removeCopilotChatResourceBodySchema = mothershipResourceRemovalSchema.safeExtend({ chatId: z.string(), - resourceType: mothershipResourceSchema.shape.type, - resourceId: z.string(), }) export type RemoveCopilotChatResourceBody = z.input diff --git a/apps/sim/lib/api/contracts/mothership-assistant-tools.ts b/apps/sim/lib/api/contracts/mothership-assistant-tools.ts index 50e5cc740c0..ec1a9d5217e 100644 --- a/apps/sim/lib/api/contracts/mothership-assistant-tools.ts +++ b/apps/sim/lib/api/contracts/mothership-assistant-tools.ts @@ -347,8 +347,22 @@ export const listWorkspacesInputSchema = z.object({ cursor: z.string().uuid().optional().describe('Copy nextCursor from the preceding page.'), }) +/** Project discovery accepts pagination only; the conversation determines the accessible scope. */ +export const listUserProjectsInputSchema = z.object({ + cursor: z.string().min(1).optional(), + limit: z.number().int().min(1).max(100).default(50), +}) +export type ListUserProjectsInput = z.output + /** Executor identity belongs to Sim; model-facing profile guidance belongs to its caller. */ export const assistantToolContracts = [ + { + id: 'list_user_projects', + route: 'sim', + description: + 'List accessible Projects within this conversation, with visible environments and current capabilities. Follow nextCursor for more results; discovery does not grant file write access.', + inputSchema: listUserProjectsInputSchema, + }, { id: 'list_workspaces', route: 'sim', diff --git a/apps/sim/lib/api/contracts/mothership-chats.ts b/apps/sim/lib/api/contracts/mothership-chats.ts index 3fccdbf253d..5f1a7461a50 100644 --- a/apps/sim/lib/api/contracts/mothership-chats.ts +++ b/apps/sim/lib/api/contracts/mothership-chats.ts @@ -1,7 +1,9 @@ import { z } from 'zod' import { addCopilotChatResourceBodySchema } from '@/lib/api/contracts/copilot' -import { mothershipResourceSchema } from '@/lib/api/contracts/mothership-resources' -import { workspaceIdSchema } from '@/lib/api/contracts/primitives' +import { + mothershipResourceRemovalSchema, + mothershipResourceSchema, +} from '@/lib/api/contracts/mothership-resources' import { scheduleContextSchema } from '@/lib/api/contracts/schedules' import { mountedSecretNamesSchema, @@ -234,11 +236,8 @@ const reorderMothershipChatResourcesBodySchema = z.object({ resources: z.array(mothershipResourceSchema), }) -const removeMothershipChatResourceBodySchema = z.object({ - workspaceId: workspaceIdSchema.optional(), +const removeMothershipChatResourceBodySchema = mothershipResourceRemovalSchema.safeExtend({ chatId: z.string().min(1), - resourceType: mothershipResourceSchema.shape.type, - resourceId: z.string().min(1), }) export const addMothershipChatResourceContract = defineRouteContract({ diff --git a/apps/sim/lib/api/contracts/mothership-resource-tools.ts b/apps/sim/lib/api/contracts/mothership-resource-tools.ts index d4224c0724b..901ffef3d27 100644 --- a/apps/sim/lib/api/contracts/mothership-resource-tools.ts +++ b/apps/sim/lib/api/contracts/mothership-resource-tools.ts @@ -1,28 +1,46 @@ import { z } from 'zod' +import { FileOperationOwner } from '@/lib/mothership/generated/file-owner' -/** Canonical resources in one selected workspace; titles come from authorized reads. */ -export const openResourceInputSchema = z.strictObject({ - workspaceId: z - .string() - .min(1) - .optional() - .describe('Explicit owning workspace in organization chat.'), - resources: z - .array( - z.strictObject({ - type: z.enum(['workflow', 'table', 'knowledgebase', 'file', 'dashboard', 'log', 'test']), - id: z.string().trim().min(1), - viewId: z - .string() - .trim() - .min(1) - .optional() - .describe('Saved table view; table resources only.'), +/** Requested targets only; canonical metadata comes from authorized reads. */ +export const openResourceInputSchema = z + .strictObject({ + workspaceId: z + .string() + .min(1) + .optional() + .describe('Explicit owning workspace in organization chat.'), + resources: z + .array( + z.strictObject({ + type: z.enum(['workflow', 'table', 'knowledgebase', 'file', 'dashboard', 'log', 'test']), + id: z.string().trim().min(1), + owner: FileOperationOwner.optional(), + viewId: z + .string() + .trim() + .min(1) + .optional() + .describe('Saved table view; table resources only.'), + }) + ) + .min(1) + .max(20), + }) + .superRefine((input, ctx) => { + const projects = input.resources.filter((resource) => resource.owner?.entityType === 'project') + if (input.resources.some((resource) => resource.owner && resource.type !== 'file')) + ctx.addIssue({ + code: 'custom', + path: ['resources'], + message: 'Explicit file owners apply only to files', }) - ) - .min(1) - .max(20), -}) + if (projects.length && (projects.length !== input.resources.length || input.workspaceId)) + ctx.addIssue({ + code: 'custom', + path: ['resources'], + message: 'Project files cannot be mixed with workspace targets', + }) + }) export const openResourceOutputSchema = z.object({ resources: z.array( z.object({ @@ -31,6 +49,7 @@ export const openResourceOutputSchema = z.object({ title: z.string(), viewId: z.string().optional(), executionId: z.string().optional(), + owner: FileOperationOwner.optional(), }) ), }) diff --git a/apps/sim/lib/api/contracts/mothership-resources.ts b/apps/sim/lib/api/contracts/mothership-resources.ts index 65c9add4770..cd4cbfdbd4e 100644 --- a/apps/sim/lib/api/contracts/mothership-resources.ts +++ b/apps/sim/lib/api/contracts/mothership-resources.ts @@ -3,16 +3,30 @@ import { requiredFieldSchema, workspaceIdSchema } from '@/lib/api/contracts/prim import { predicateInputSchema, sortSpecSchema } from '@/lib/api/contracts/tables' import { ResourceAddress, SearchResource } from '@/lib/mothership/generated/resources' import { + hasValidChatResourceOwner, type MothershipResource, MothershipResourceType, PERSISTED_RESOURCE_TYPES, } from '@/lib/mothership/resources/types' +function refineResourceOwner( + resource: Parameters[0], + ctx: z.RefinementCtx +): void { + if (!hasValidChatResourceOwner(resource)) + ctx.addIssue({ + code: 'custom', + path: ['owner'], + message: 'Resource owner conflicts with its address', + }) +} + /** Keep the complete panel address through request parsing, persistence and hydration. */ const resourceAddressSchema = z .object({ type: z.enum(PERSISTED_RESOURCE_TYPES), id: requiredFieldSchema('resource.id cannot be empty'), + owner: ResourceAddress.shape.owner, workspaceId: workspaceIdSchema.optional(), workspaceName: z.string().max(256).optional(), path: z.string().optional(), @@ -22,6 +36,7 @@ const resourceAddressSchema = z sources: ResourceAddress.shape.sources, }) .superRefine((resource, ctx) => { + refineResourceOwner(resource, ctx) if ((resource.type === 'sources') !== (resource.sources !== undefined)) ctx.addIssue({ code: 'custom', @@ -56,6 +71,18 @@ export const mothershipResourceSchema = resourceAddressSchema.safeExtend({ title: z.string(), }) satisfies z.ZodType +/** Removal uses the same file owner constraints as resource creation and hydration. */ +export const mothershipResourceRemovalSchema = z + .object({ + owner: ResourceAddress.shape.owner, + workspaceId: workspaceIdSchema.optional(), + resourceType: z.enum(PERSISTED_RESOURCE_TYPES), + resourceId: requiredFieldSchema('resourceId cannot be empty'), + }) + .superRefine((resource, ctx) => + refineResourceOwner({ ...resource, type: resource.resourceType }, ctx) + ) + /** The query visible in an embedded table, including changes not saved yet. */ export const mothershipTableViewContextSchema = z .object({ diff --git a/apps/sim/lib/api/server/routes/copilot-project-file-request.test.ts b/apps/sim/lib/api/server/routes/copilot-project-file-request.test.ts new file mode 100644 index 00000000000..3aed26faf89 --- /dev/null +++ b/apps/sim/lib/api/server/routes/copilot-project-file-request.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from 'vitest' +import { copilotRequestPrincipal, isCopilotRequest } from '@/lib/api/server/routes/copilot-request' +import { requireResourceDelegation } from '@/lib/core/application/resource-delegation' +import { markCopilotProjectFileRequest } from '@/lib/mothership/application/execute-project-file-use-case' +import { + type CopilotExecutionContext, + createCopilotResourceAdmission, +} from '@/lib/mothership/auth/application-delegation' +import { getProjectFileMetadata, listProjectFiles } from '@/lib/projects/files/application' +import { projectFileOperations } from '@/lib/projects/files/application/operations' +import { listAllWorkspaceFiles } from '@/lib/workspace-files/application/list-workspace-files' + +function authoringContext(): CopilotExecutionContext { + return { + userId: 'actor', + chatId: 'private-chat', + toolCallId: 'tool-call', + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: 'actor', + invocation: { kind: 'chat', chatId: 'private-chat' }, + }), + } +} + +function request() { + return new Request('https://sim.test/api/v2/projects/project/files/file/metadata', { + headers: { + 'x-mothership-project-id': 'project', + 'x-mothership-user-id': 'actor', + 'x-api-key': 'mothership-in-process', + }, + }) +} + +describe('private v2 Project file authority', () => { + it('retains the admitted human and exact target without a workspace delegation', () => { + const req = request() + const context = authoringContext() + const target = { projectId: 'project', fileId: 'file' } + markCopilotProjectFileRequest(req, context, target) + context.userId = 'later-user' + target.projectId = 'later-project' + const principal = copilotRequestPrincipal( + req, + projectFileOperations.readMetadata, + getProjectFileMetadata + ) + if (!principal) throw new Error('The admitted Project file request lost its principal') + expect(principal).toMatchObject({ + kind: 'resource_delegated', + subjectUserId: 'actor', + invocation: { kind: 'chat', chatId: 'private-chat' }, + scope: { kind: 'entity', entityType: 'project', entityId: 'project', fileId: 'file' }, + }) + expect(principal).not.toHaveProperty('workspaceId') + expect( + requireResourceDelegation(principal, { + audience: 'sim:project-files', + services: ['copilot'], + scope: { kind: 'entity', entityType: 'project', entityId: 'project', fileId: 'file' }, + maxTtlMs: 60_000, + }) + ).toBe('actor') + }) + + it('does not transfer private admission through headers or a cloned Request', () => { + const req = request() + expect(isCopilotRequest(req)).toBe(false) + expect( + copilotRequestPrincipal(req, projectFileOperations.readMetadata, getProjectFileMetadata) + ).toBeUndefined() + markCopilotProjectFileRequest(req, authoringContext(), { projectId: 'project', fileId: 'file' }) + const cloned = new Request(req) + expect(isCopilotRequest(cloned)).toBe(false) + expect( + copilotRequestPrincipal(cloned, projectFileOperations.readMetadata, getProjectFileMetadata) + ).toBeUndefined() + }) + + it('refuses to mark a request without current authoring admission', () => { + const req = request() + const context = authoringContext() + context.copilotResourceAdmission = structuredClone(context.copilotResourceAdmission) + expect(() => markCopilotProjectFileRequest(req, context, { projectId: 'project' })).toThrow( + 'Copilot resource operations require current authoring admission' + ) + expect(isCopilotRequest(req)).toBe(false) + }) + + it('never changes a file-bound request into collection or workspace authority', () => { + const req = request() + markCopilotProjectFileRequest(req, authoringContext(), { projectId: 'project', fileId: 'file' }) + expect( + copilotRequestPrincipal(req, projectFileOperations.list, listProjectFiles) + ).toBeUndefined() + expect( + copilotRequestPrincipal(req, listAllWorkspaceFiles.operation, listAllWorkspaceFiles) + ).toBeUndefined() + const collection = request() + markCopilotProjectFileRequest(collection, authoringContext(), { projectId: 'project' }) + expect( + copilotRequestPrincipal(collection, projectFileOperations.list, listProjectFiles) + ).toMatchObject({ + scope: { kind: 'entity', entityType: 'project', entityId: 'project' }, + }) + expect( + copilotRequestPrincipal( + collection, + projectFileOperations.readMetadata, + getProjectFileMetadata + ) + ).toBeUndefined() + }) + + it('rejects unregistered operation identities and mismatched use-case policies', () => { + const req = request() + markCopilotProjectFileRequest(req, authoringContext(), { projectId: 'project', fileId: 'file' }) + const clone = { ...projectFileOperations.readMetadata } + expect( + copilotRequestPrincipal(req, clone, { ...getProjectFileMetadata, operation: clone }) + ).toBeUndefined() + expect( + copilotRequestPrincipal(req, projectFileOperations.readMetadata, listProjectFiles) + ).toBeUndefined() + expect( + copilotRequestPrincipal(req, projectFileOperations.readMetadata, { + ...getProjectFileMetadata, + delegationAudience: 'sim:other', + }) + ).toBeUndefined() + expect(copilotRequestPrincipal(req, projectFileOperations.readMetadata)).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/api/server/routes/copilot-refused-routes.json b/apps/sim/lib/api/server/routes/copilot-refused-routes.json index 2cc89d250fa..0720d6ac7fa 100644 --- a/apps/sim/lib/api/server/routes/copilot-refused-routes.json +++ b/apps/sim/lib/api/server/routes/copilot-refused-routes.json @@ -229,6 +229,31 @@ "path": "/api/v2/organizations/[organizationId]/workspaces", "operation": "organizations.workspaces.list" }, + { + "method": "GET", + "path": "/api/v2/projects/[projectId]/files/[fileId]/versions", + "operation": "project_files.versions.list" + }, + { + "method": "GET", + "path": "/api/v2/projects/[projectId]/files/[fileId]/versions/[version]", + "operation": "project_files.versions.read" + }, + { + "method": "DELETE", + "path": "/api/v2/projects/[projectId]/files/[fileId]/versions/[version]", + "operation": "project_files.versions.delete" + }, + { + "method": "GET", + "path": "/api/v2/projects/[projectId]/files/[fileId]/versions/[version]/content", + "operation": "project_files.versions.read_content" + }, + { + "method": "POST", + "path": "/api/v2/projects/[projectId]/files/[fileId]/versions/[version]/revert", + "operation": "project_files.versions.revert" + }, { "method": "GET", "path": "/api/v2/tables/[tableId]/exports/[exportId]/download", diff --git a/apps/sim/lib/api/server/routes/copilot-request.ts b/apps/sim/lib/api/server/routes/copilot-request.ts index 0cb4bc3052a..4a6045618dd 100644 --- a/apps/sim/lib/api/server/routes/copilot-request.ts +++ b/apps/sim/lib/api/server/routes/copilot-request.ts @@ -1,4 +1,4 @@ -import type { DelegatedPrincipal } from '@sim/auth/principal' +import type { DelegatedPrincipal, ResourceDelegatedPrincipal } from '@sim/auth/principal' import { noteContentDeliveryOperation } from '@/lib/core/application' import { markCopilotWorkspaceInvocation } from '@/lib/core/application/copilot-workspace-invocation' import type { ApplicationOperation, OperationUseCase } from '@/lib/core/application/operation' @@ -7,14 +7,36 @@ import { createCopilotChatPrincipal, } from '@/lib/mothership/auth/application-delegation' +/** + * Mints the resource grant one admitted request carries for an operation the route runs under + * the given audience, or none when that operation is outside the grant. Owned by the domain + * adapter that bound the request, so this route layer stays free of domain policy. + */ +type CopilotResourceRequestGrant = ( + operation: ApplicationOperation, + audience: string +) => ResourceDelegatedPrincipal | undefined + +type CopilotRequestInvocation = + | { kind: 'resource'; grant: CopilotResourceRequestGrant } + | { kind: 'workspace'; context: Readonly } + /** A private in-process invocation, installed only after current chat/target authorization. Never a wire header. */ -const INVOCATIONS = new WeakMap>() +const INVOCATIONS = new WeakMap() export function markCopilotRequest( request: Request, invocation: CopilotChatDelegationContext ): void { - INVOCATIONS.set(request, Object.freeze({ ...invocation })) + INVOCATIONS.set(request, { kind: 'workspace', context: Object.freeze({ ...invocation }) }) +} + +/** Binds one request to a domain adapter's resource grant; the use case still reauthorizes current access. */ +export function markCopilotResourceRequest( + request: Request, + grant: CopilotResourceRequestGrant +): void { + INVOCATIONS.set(request, { kind: 'resource', grant }) } export function isCopilotRequest(request: Request): boolean { @@ -35,19 +57,31 @@ export function copilotRouteAudience( operation: ApplicationOperation, useCase?: CopilotRouteUseCase ): string | undefined { - return useCase?.operation === operation ? useCase.delegationAudience || undefined : undefined + if (useCase?.operation !== operation) return undefined + // An operation that names its delegated services without `copilot` is refused before minting. + const services: readonly string[] | undefined = + 'delegatedServices' in operation + ? (operation.delegatedServices as readonly string[]) + : undefined + if (services && !services.includes('copilot')) return undefined + return useCase.delegationAudience || undefined } export function copilotRequestPrincipal( request: Request, operation: ApplicationOperation, useCase?: CopilotRouteUseCase -): DelegatedPrincipal | undefined { +): DelegatedPrincipal | ResourceDelegatedPrincipal | undefined { const invocation = INVOCATIONS.get(request) if (!invocation) return undefined const audience = copilotRouteAudience(operation, useCase) if (!audience) return undefined - const principal = createCopilotChatPrincipal(invocation, audience) + if (invocation.kind === 'resource') { + const principal = invocation.grant(operation, audience) + if (principal) noteContentDeliveryOperation(operation, principal) + return principal + } + const principal = createCopilotChatPrincipal(invocation.context, audience) markCopilotWorkspaceInvocation(principal) noteContentDeliveryOperation(operation, principal) return principal diff --git a/apps/sim/lib/api/server/routes/in-process-transport.ts b/apps/sim/lib/api/server/routes/in-process-transport.ts index 0a78a5972dc..e1e0275b744 100644 --- a/apps/sim/lib/api/server/routes/in-process-transport.ts +++ b/apps/sim/lib/api/server/routes/in-process-transport.ts @@ -30,7 +30,7 @@ interface CompiledRoute { load: () => Promise } -interface MatchedRoute { +export interface MatchedRoute { pattern: string params: Record literals: number diff --git a/apps/sim/lib/api/server/routes/v2-route-pattern.ts b/apps/sim/lib/api/server/routes/v2-route-pattern.ts new file mode 100644 index 00000000000..02e4bd8d4f9 --- /dev/null +++ b/apps/sim/lib/api/server/routes/v2-route-pattern.ts @@ -0,0 +1,24 @@ +import { type MatchedRoute, matchV2Route } from '@/lib/api/server/routes/in-process-transport' + +/** A contract's path in the generated route table's `{param}` pattern syntax. */ +export function v2RoutePattern(contract: { path: string }): string { + return contract.path.replace(/\[([^\]]+)\]/g, '{$1}') +} + +/** Whether a matched request is the method and route a contract declares. */ +export function isV2ContractRoute( + method: string, + route: Pick, + contract: { method: string; path: string } +): boolean { + return method === contract.method && route.pattern === v2RoutePattern(contract) +} + +/** {@link matchV2Route}, reading a malformed percent encoding as no route. */ +export function tryMatchV2Route(pathname: string): MatchedRoute | null { + try { + return matchV2Route(pathname) + } catch { + return null + } +} diff --git a/apps/sim/lib/mothership/agent-cli/engines/file-read.ts b/apps/sim/lib/mothership/agent-cli/engines/file-read.ts index 9d4b2cd8c42..be1f7ade79a 100644 --- a/apps/sim/lib/mothership/agent-cli/engines/file-read.ts +++ b/apps/sim/lib/mothership/agent-cli/engines/file-read.ts @@ -2,17 +2,15 @@ import { v2ReadFileTextContract, v2ReadFileTextQuerySchema } from '@/lib/api/con import { readOriginalKnowledgeDocument } from '@/lib/knowledge/application/read-original-document' import { enginePrincipal } from '@/lib/mothership/agent-cli/engine-principal' import { fileReadFailure, readFileVisual } from '@/lib/mothership/agent-cli/engines/file-view' -import { observePrivateFile } from '@/lib/mothership/agent-cli/engines/observe-private-file' +import { + isDefaultVisualMediaType, + observePrivateFile, +} from '@/lib/mothership/agent-cli/engines/observe-private-file' import { readScratchFile } from '@/lib/mothership/agent-cli/engines/scratch-file-read' import { type AgentCliEngine, agentCliFail, agentCliOk } from '@/lib/mothership/agent-cli/types' import { readChatAttachment } from '@/lib/mothership/chat/application/read-attachment' import { workspaceFileVfsPath } from '@/lib/uploads/contexts/workspace/file-manager' -import { - getFileExtension, - getMimeTypeFromExtension, - needsRenderedArtifact, - resolveEffectiveMimeType, -} from '@/lib/uploads/utils/file' +import { observedMediaType } from '@/lib/uploads/utils/file' import { fileOperations } from '@/lib/workspace-files/application/operations' import { resolveWorkspaceFileReference } from '@/lib/workspace-files/application/resolve-workspace-file-reference' import { workspaceFileTextFormat } from '@/lib/workspace-files/text-format' @@ -120,15 +118,8 @@ export const fileReadCommand: AgentCliEngine = { ...(runtime.chatId !== undefined ? { chatId: runtime.chatId } : {}), }) runtime.signal?.throwIfAborted() - const type = - (needsRenderedArtifact(file.type, file.name) - ? getMimeTypeFromExtension(getFileExtension(file.name)) - : resolveEffectiveMimeType(file.type, file.name) - ) - .split(';')[0] - ?.trim() - .toLowerCase() ?? '' - if (visual || (!textRange && (type.startsWith('image/') || type === 'application/pdf'))) { + const type = observedMediaType(file.type, file.name) + if (visual || (!textRange && isDefaultVisualMediaType(type))) { const { metadata, ...result } = await readFileVisual( file.id, runtime, diff --git a/apps/sim/lib/mothership/agent-cli/engines/observe-private-file.ts b/apps/sim/lib/mothership/agent-cli/engines/observe-private-file.ts index 4e2ce13bece..5cbc5b8617e 100644 --- a/apps/sim/lib/mothership/agent-cli/engines/observe-private-file.ts +++ b/apps/sim/lib/mothership/agent-cli/engines/observe-private-file.ts @@ -1,12 +1,19 @@ -import { ContentRefusedError, redactBeforeCutting } from '@/lib/core/application' +import { ContentRefusedError } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { AgentCliFlags } from '@/lib/mothership/agent-cli/types' import { agentCliOk } from '@/lib/mothership/agent-cli/types' import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file' -import { parseFileText } from '@/lib/workspace-files/application/read-workspace-file-text' +import { + type ExtractedFileText, + extractFileTextFromBytes, +} from '@/lib/workspace-files/application/read-workspace-file-text' import { decodeFileVisual } from '@/lib/workspace-files/decode-file-visual' import { workspaceFileTextFormat } from '@/lib/workspace-files/text-format' -import { sliceFileTextLines } from '@/lib/workspace-files/text-lines' + +/** Images and PDFs are read visually unless a line range asks for their text. */ +export function isDefaultVisualMediaType(type: string): boolean { + return type.startsWith('image/') || type === 'application/pdf' +} /** Shared representation selection for authorized private file bytes. */ export async function observePrivateFile( @@ -14,7 +21,8 @@ export async function observePrivateFile( buffer: Buffer name: string path: string - source: 'sandbox' | 'upload' | 'knowledge' + source: 'sandbox' | 'upload' | 'knowledge' | 'project' + fileId?: string contentType?: string /** The bytes hold secrets whose text redaction a rendered image would bypass. */ hasRecordedSecrets?: boolean @@ -27,7 +35,7 @@ export async function observePrivateFile( } }, flags: AgentCliFlags, - options: { offset?: number; limit?: number }, + options: { offset?: number; limit?: number; visual?: boolean }, signal?: AbortSignal ) { const type = file.contentType ?? getMimeTypeFromExtension(getFileExtension(file.name)) @@ -36,11 +44,12 @@ export async function observePrivateFile( path: file.path, type, source: file.source, + ...(file.fileId ? { fileId: file.fileId } : {}), ...file.knowledge, } const textRange = options.offset !== undefined || options.limit !== undefined - const visual = flags.render !== undefined || flags.pages !== undefined - if (visual || (!textRange && (type.startsWith('image/') || type === 'application/pdf'))) { + const visual = options.visual || flags.render !== undefined || flags.pages !== undefined + if (visual || (!textRange && isDefaultVisualMediaType(type))) { if (file.hasRecordedSecrets) { throw new ContentRefusedError('workspace-file') } @@ -66,6 +75,7 @@ export async function observePrivateFile( observations: [ { name: file.name, + ...(file.fileId ? { resourceId: file.fileId } : {}), mediaType: decoded.mediaType, data: decoded.buffer.toString('base64'), ...(decoded.pages ? { pageCount: decoded.pages.last - decoded.pages.first + 1 } : {}), @@ -75,35 +85,10 @@ export async function observePrivateFile( } const format = workspaceFileTextFormat({ name: file.name, type }) if (format) { - const parsed = await parseFileText(file.buffer, format, file.name) - const truncated = parsed.metadata?.truncated === true - const ranged = options.offset !== undefined || options.limit !== undefined - const { text, lineRange } = sliceFileTextLines( - ranged - ? await redactBeforeCutting( - undefined, - parsed.content, - file.source === 'knowledge' ? 'knowledge-document' : 'workspace-file' - ) - : parsed.content, - options.offset, - options.limit, - truncated - ) - return agentCliOk( - JSON.stringify({ - ...metadata, - representation: 'text', - text, - truncated, - degraded: parsed.metadata?.degraded === true, - degradedReason: - parsed.metadata?.degraded === true ? (parsed.metadata.warning ?? null) : null, - charCount: text.length, - byteCount: file.buffer.length, - ...(lineRange ? { lineRange } : {}), - }) - ) + const extracted = await extractFileTextFromBytes(file.buffer, format, file.name, options, { + kind: file.source === 'knowledge' ? 'knowledge-document' : 'workspace-file', + }) + return presentObservedText(metadata, extracted) } if (textRange) throw new OrchestrationError('validation', 'This file has no text decoder') return agentCliOk( @@ -113,9 +98,31 @@ export async function observePrivateFile( bytes: file.buffer.length, contentAvailable: false, note: - file.source === 'knowledge' - ? 'Content was not inspected: this original has no model-readable decoder.' - : 'Content was not inspected: this file has no model-readable decoder. Process its mounted sandbox path with run_code.', + file.source === 'project' + ? 'Content was not inspected: this file has no model-readable decoder. Download its stored bytes to inspect it.' + : file.source === 'knowledge' + ? 'Content was not inspected: this original has no model-readable decoder.' + : 'Content was not inspected: this file has no model-readable decoder. Process its mounted sandbox path with run_code.', + }) + ) +} + +/** The `files read` text representation, shared by every owner's text read. */ +export function presentObservedText( + metadata: Record, + extracted: ExtractedFileText +) { + return agentCliOk( + JSON.stringify({ + ...metadata, + representation: 'text', + text: extracted.text, + truncated: extracted.truncated, + degraded: extracted.degraded, + degradedReason: extracted.degradedReason, + charCount: extracted.text.length, + byteCount: extracted.byteCount, + ...(extracted.lineRange ? { lineRange: extracted.lineRange } : {}), }) ) } diff --git a/apps/sim/lib/mothership/agent-cli/engines/universal-grep.ts b/apps/sim/lib/mothership/agent-cli/engines/universal-grep.ts index 088964a2d1e..83a668f8cb3 100644 --- a/apps/sim/lib/mothership/agent-cli/engines/universal-grep.ts +++ b/apps/sim/lib/mothership/agent-cli/engines/universal-grep.ts @@ -49,9 +49,9 @@ const MAX_LINE_CHARS = 2_000 const FETCH_CONCURRENCY = 8 /** Past any catalog's size: the use case builds the full list in memory and slices it. */ const CATALOG_ALL = 100_000 -const MAX_FILES = 300 +export const MAX_GREP_FILES = 300 const FILE_READ_CONCURRENCY = 5 -const MAX_BYTES_PER_FILE = 262_144 +export const MAX_GREP_BYTES_PER_FILE = 262_144 /** Bound scanning after materialization without charging backend read latency. */ const MAX_SCAN_TIME_MS = 5_000 const SCAN_YIELD_INTERVAL_MS = 10 @@ -71,7 +71,11 @@ const NESTED_REQUEST_CONCURRENCY = 8 let activeRequests = 0 const waiting: Array<() => void> = [] -async function gated(work: () => Promise, signal?: AbortSignal): Promise { +/** Runs one backend read in the shared grep read queue; a released slot passes to the next waiter. */ +export async function withGrepReadSlot( + work: () => Promise, + signal?: AbortSignal +): Promise { signal?.throwIfAborted() if (activeRequests >= NESTED_REQUEST_CONCURRENCY) { await new Promise((resolve, reject) => { @@ -105,12 +109,12 @@ function gatedRuntime(runtime: GrepRuntime): GrepRuntime { ...runtime, client: { request: (path, options) => - gated(() => runtime.client.request(path, options), runtime.signal), + withGrepReadSlot(() => runtime.client.request(path, options), runtime.signal), }, } } -interface Materialized { +export interface GrepMaterialized { scope: Scope /** Display identity: the resource's name when it has one, else its id. */ label: string @@ -127,10 +131,18 @@ function str(value: unknown): string | undefined { return typeof value === 'string' ? value : undefined } +/** Inventory pages one scope lists before reporting the search incomplete. */ +export const MAX_GREP_LIST_PAGES = 50 + +/** The incomplete-search note for a scope whose inventory hit {@link MAX_GREP_LIST_PAGES}. */ +export function grepListingStoppedMessage(scope: string): string { + return `${scope}: listing stopped after ${MAX_GREP_LIST_PAGES} pages; use ${scope} list to continue discovery.` +} + async function listAll(runtime: GrepRuntime, path: string): Promise[]> { const out: Record[] = [] let cursor: string | undefined - for (let pages = 0; pages < 50; pages++) { + for (let pages = 0; pages < MAX_GREP_LIST_PAGES; pages++) { const page = await runtime.client.request(path, { query: { workspaceId: runtime.workspaceId, limit: '100', ...(cursor ? { cursor } : {}) }, }) @@ -138,13 +150,15 @@ async function listAll(runtime: GrepRuntime, path: string): Promise(items: T[], read: (item: T) => Promise): Promise { + return mapConcurrent(items, FILE_READ_CONCURRENCY, read) +} + async function mapConcurrent( items: T[], limit: number, @@ -176,7 +190,7 @@ async function settleAll(tasks: Promise[]): Promise { * The searchable text leads with the resource's own name and description: a workflow's * state carries neither, so `grep fx-` in an `fx-*` workspace found nothing. */ -function render(scope: Scope, id: string, label: string, value: unknown): Materialized { +function render(scope: Scope, id: string, label: string, value: unknown): GrepMaterialized { const description = isRecordLike(value) && typeof (value as Record).description === 'string' ? (value as Record).description @@ -226,7 +240,7 @@ const INDEXERS: Record Promise> = // runs answers in one call, same authorization, same projection. if (runtime.principal) { const principal = enginePrincipal(runtime, listCatalogTools)! - const page = await gated( + const page = await withGrepReadSlot( () => listCatalogTools.execute({ principal, @@ -288,7 +302,10 @@ const INDEXERS: Record Promise> = } /** One fetch per scope; a null text preserves the identity of an unreadable resource. */ -const FETCHERS: Record Promise> = { +const FETCHERS: Record< + Scope, + (runtime: GrepRuntime, item: IndexEntry) => Promise +> = { workflows: async (runtime, item) => { // The draft state, not the export: export is sanitized for sharing and nulls // workspace-specific fields (a Table block's `tableId`), so a grep for a table id @@ -317,7 +334,7 @@ const FETCHERS: Record Promis try { const response = await runtime.client.request( `/api/v2/files/${encodeURIComponent(item.id)}/text`, - { query: { workspaceId: runtime.workspaceId, maxBytes: String(MAX_BYTES_PER_FILE) } } + { query: { workspaceId: runtime.workspaceId, maxBytes: String(MAX_GREP_BYTES_PER_FILE) } } ) if (response.data.degraded || response.data.truncated) { runtime.reportIncomplete( @@ -361,21 +378,21 @@ async function fetchAll( runtime: GrepRuntime, scope: Scope, entries: IndexEntry[] -): Promise { - if (scope === 'files' && entries.length > MAX_FILES) { +): Promise { + if (scope === 'files' && entries.length > MAX_GREP_FILES) { runtime.reportIncomplete( - `files: searched the first ${MAX_FILES} of ${entries.length} selected files; narrow with --in files/.` + `files: searched the first ${MAX_GREP_FILES} of ${entries.length} selected files; narrow with --in files/.` ) } - const selected = scope === 'files' ? entries.slice(0, MAX_FILES) : entries + const selected = scope === 'files' ? entries.slice(0, MAX_GREP_FILES) : entries return mapConcurrent(selected, concurrencyFor(scope), (item) => FETCHERS[scope](runtime, item)) } /** A whole world, for a search with no `--in`: every resource the index lists. */ -async function materializeScope(runtime: GrepRuntime, scope: Scope): Promise { +async function materializeScope(runtime: GrepRuntime, scope: Scope): Promise { if (scope === 'blocks' && runtime.principal) { const principal = enginePrincipal(runtime, readBlockCatalog)! - const { blocks } = await gated( + const { blocks } = await withGrepReadSlot( () => readBlockCatalog.execute({ principal, @@ -402,10 +419,10 @@ async function materializeWithin( runtime: GrepRuntime, scopes: Scope[], nameFilter: string -): Promise { +): Promise { // Cheapest platform world first: a block id answers from 65 definitions, and only a // miss there pays for the tool catalog's 5,000 entries. - const platform: Materialized[] = [] + const platform: GrepMaterialized[] = [] for (const scope of scopes.filter((scope) => PLATFORM_SCOPES.has(scope))) { const corpus = await materializeScope(runtime, scope) const exact = corpus.filter((m) => m.id.toLowerCase() === nameFilter) @@ -539,142 +556,176 @@ function parseContext(flags: AgentCliFlags): ContextWindow | string { return { before: before ?? around ?? 0, after: after ?? around ?? 0 } } -export const universalGrepCommand: AgentCliEngine = { - async execute(positionals, runtime, flags) { - const pattern = positionals[0] - if (!pattern) { - return agentCliFail( - 'Usage: sim grep [--scope workflows,blocks,...] [--in ] [-i] [-C n] [-A n] [-B n] [--count] [--limit n]' +/** + * An owner-specific corpus for {@link executeGrep}: `materialize` returns only authorized text + * and reports skipped items through `reportIncomplete`; `scopeLimit` refuses any other + * `--scope`/`--in`. + */ +export interface GrepSource { + signal?: AbortSignal + registry?: AgentCliRuntime['resolvedSecretTraceRegistry'] + /** The only scopes this corpus holds, and the refusal for a search that names another. */ + scopeLimit?: { scopes: readonly Scope[]; refusal: string } + materialize( + scopes: Scope[], + nameFilter: string | undefined, + reportIncomplete: (message: string) => void + ): Promise +} + +/** Shared bounded matching over a corpus authorized by its owner-specific source. */ +export async function executeGrep(positionals: string[], flags: AgentCliFlags, source: GrepSource) { + const pattern = positionals[0] + if (!pattern) { + return agentCliFail( + 'Usage: sim grep [--scope workflows,blocks,...] [--in ] [-i] [-C n] [-A n] [-B n] [--count] [--limit n]' + ) + } + const scopes = parseScopes(flags) + if (typeof scopes === 'string') return agentCliFail(scopes) + const limit = parseLimit(flags) + if (typeof limit === 'string') return agentCliFail(limit) + const context = parseContext(flags) + if (typeof context === 'string') return agentCliFail(context) + const ignoreCase = flags.i === true + const countOnly = flags.count === true + // `--in tables` reads as "search the tables world", so a world name narrows the scope; + // `--in blocks/table_v2` is the path a match line prints (world, then resource); a bare + // value is a resource id or name inside the searched worlds. + const within = typeof flags.in === 'string' ? flags.in : undefined + const [withinHead, ...withinRest] = within ? within.toLowerCase().split('/') : [] + if (within && withinHead && KNOWLEDGE_SELECTOR_HEADS.has(withinHead)) { + return agentCliFail(knowledgeWithin(within)) + } + const withinScope = SCOPES.find((scope) => scope === withinHead) + const withinResource = withinScope ? withinRest.join('/') : within?.toLowerCase() + const searched: Scope[] = withinScope ? [withinScope] : scopes + const scopeLimit = source.scopeLimit + if ( + scopeLimit && + (scopes.length === 0 || + scopes.some((scope) => !scopeLimit.scopes.includes(scope)) || + searched.some((scope) => !scopeLimit.scopes.includes(scope))) + ) { + return agentCliFail(scopeLimit.refusal) + } + const nameFilter = withinResource || undefined + /** + * Refused before any fetch: the model learns the accepted forms without paying for + * a full materialization it would only get an empty result from. + */ + if (within && nameFilter && PREFIX_SELECTOR.test(nameFilter)) { + return agentCliFail(unknownWithin(within)) + } + const matches = compilePattern(pattern, ignoreCase) + const issues: string[] = [] + let issueCount = 0 + const reportIncomplete = (message: string) => { + issueCount++ + if (issues.length < 10) issues.push(clip(message)) + } + const finish = (stdout: string) => + issueCount === 0 + ? agentCliOk(stdout) + : { + exitCode: 1, + stdout, + stderr: `Search incomplete (${issueCount} issue${issueCount === 1 ? '' : 's'}); matches and counts cover only the content read.\n${issues.join('\n')}${issueCount > issues.length ? `\n${issueCount - issues.length} further issues omitted.` : ''}`, + } + const candidates = await source.materialize(searched, nameFilter, reportIncomplete) + source.signal?.throwIfAborted() + /** + * A resource nothing in the searched worlds answers to is a wrong selector, not a + * search with no hits — a silent "No matches" would hide the misspelling. + */ + if (within && nameFilter && candidates.length === 0) { + return issueCount ? finish('') : agentCliFail(unknownWithin(within)) + } + + const out: string[] = [] + let total = 0 + let shownMatches = 0 + const perScope = new Map() + const scanStartedAt = performance.now() + let lastYieldAt = scanStartedAt + const checkScan = () => { + source.signal?.throwIfAborted() + const now = performance.now() + if (now - scanStartedAt >= MAX_SCAN_TIME_MS) { + throw new Error( + 'Search incomplete: scanning exceeded the time budget. Narrow with --scope or --in.' ) } - const scopes = parseScopes(flags) - if (typeof scopes === 'string') return agentCliFail(scopes) - const limit = parseLimit(flags) - if (typeof limit === 'string') return agentCliFail(limit) - const context = parseContext(flags) - if (typeof context === 'string') return agentCliFail(context) - const ignoreCase = flags.i === true - const countOnly = flags.count === true - // `--in tables` reads as "search the tables world", so a world name narrows the scope; - // `--in blocks/table_v2` is the path a match line prints (world, then resource); a bare - // value is a resource id or name inside the searched worlds. - const within = typeof flags.in === 'string' ? flags.in : undefined - const [withinHead, ...withinRest] = within ? within.toLowerCase().split('/') : [] - if (within && withinHead && KNOWLEDGE_SELECTOR_HEADS.has(withinHead)) { - return agentCliFail(knowledgeWithin(within)) - } - const withinScope = SCOPES.find((scope) => scope === withinHead) - const withinResource = withinScope ? withinRest.join('/') : within?.toLowerCase() - const searched: Scope[] = withinScope ? [withinScope] : scopes - const nameFilter = withinResource || undefined - /** - * Refused before any fetch: the model learns the accepted forms without paying for - * a full materialization it would only get an empty result from. - */ - if (within && nameFilter && PREFIX_SELECTOR.test(nameFilter)) { - return agentCliFail(unknownWithin(within)) - } - const matches = compilePattern(pattern, ignoreCase) - const issues: string[] = [] - let issueCount = 0 - const bounded = gatedRuntime({ - ...runtime, - reportIncomplete: (message) => { - issueCount++ - if (issues.length < 10) issues.push(clip(message)) - }, - }) - const finish = (stdout: string) => - issueCount === 0 - ? agentCliOk(stdout) - : { - exitCode: 1, - stdout, - stderr: `Search incomplete (${issueCount} issue${issueCount === 1 ? '' : 's'}); matches and counts cover only the content read.\n${issues.join('\n')}${issueCount > issues.length ? `\n${issueCount - issues.length} further issues omitted.` : ''}`, - } - const candidates = nameFilter - ? await materializeWithin(bounded, searched, nameFilter) - : (await settleAll(searched.map((scope) => materializeScope(bounded, scope)))).flat() - runtime.signal?.throwIfAborted() - /** - * A resource nothing in the searched worlds answers to is a wrong selector, not a - * search with no hits — a silent "No matches" would hide the misspelling. - */ - if (within && nameFilter && candidates.length === 0) { - return issueCount ? finish('') : agentCliFail(unknownWithin(within)) - } - - const out: string[] = [] - let total = 0 - let shownMatches = 0 - const perScope = new Map() - const scanStartedAt = performance.now() - let lastYieldAt = scanStartedAt - const checkScan = () => { - runtime.signal?.throwIfAborted() - const now = performance.now() - if (now - scanStartedAt >= MAX_SCAN_TIME_MS) { - throw new Error( - 'Search incomplete: scanning exceeded the time budget. Narrow with --scope or --in.' - ) - } - return now + return now + } + const egress = source.registry?.forkForPropagatedEntries() + for (const resource of candidates) { + if (resource.text === null) continue + const text = redactForModel(resource.text, egress) + if (text === null) { + reportIncomplete( + `${resource.scope}/${resource.label} (${resource.id}): could not be checked for secret values, so it was not searched.` + ) + continue } - const egress = runtime.resolvedSecretTraceRegistry?.forkForPropagatedEntries() - for (const resource of candidates) { - if (resource.text === null) continue - const text = redactForModel(resource.text, egress) - if (text === null) { - bounded.reportIncomplete( - `${resource.scope}/${resource.label} (${resource.id}): could not be checked for secret values, so it was not searched.` - ) - continue + const lines = text.split('\n') + const selected = new Map() + const remainingLines = limit - out.length + let nextContextLine = 0 + for (let i = 0; i < lines.length; i++) { + if (checkScan() - lastYieldAt >= SCAN_YIELD_INTERVAL_MS) { + await sleep(0) + lastYieldAt = checkScan() } - const lines = text.split('\n') - const selected = new Map() - const remainingLines = limit - out.length - let nextContextLine = 0 - for (let i = 0; i < lines.length; i++) { - if (checkScan() - lastYieldAt >= SCAN_YIELD_INTERVAL_MS) { - await sleep(0) - lastYieldAt = checkScan() - } - if (!matches(lines[i])) continue - total++ - perScope.set(resource.scope, (perScope.get(resource.scope) ?? 0) + 1) - if (countOnly) continue - for ( - let j = Math.max(nextContextLine, i - context.before); - j <= Math.min(lines.length - 1, i + context.after) && selected.size < remainingLines; - j++ - ) { - selected.set(j, false) - nextContextLine = j + 1 - } - if (selected.has(i)) selected.set(i, true) - } - if (countOnly || selected.size === 0 || out.length >= limit) continue - const header = `${resource.scope}/${resource.label}${resource.label === resource.id ? '' : ` (${resource.id})`}` - for (const [i, isMatch] of selected) { - out.push(`${header}:${i + 1}: ${clip(lines[i])}`) - if (isMatch) shownMatches++ + if (!matches(lines[i])) continue + total++ + perScope.set(resource.scope, (perScope.get(resource.scope) ?? 0) + 1) + if (countOnly) continue + for ( + let j = Math.max(nextContextLine, i - context.before); + j <= Math.min(lines.length - 1, i + context.after) && selected.size < remainingLines; + j++ + ) { + selected.set(j, false) + nextContextLine = j + 1 } + if (selected.has(i)) selected.set(i, true) } - checkScan() - - if (countOnly) { - const breakdown = [...perScope.entries()].map(([s, n]) => `${s}=${n}`).join(' ') - return finish(`${total}${breakdown ? ` (${breakdown})` : ''}`) - } - if (out.length === 0) { - return finish( - `No matches for ${JSON.stringify(pattern)} in ${searched.join(', ')}${nameFilter ? ` within "${nameFilter}"` : ''}.` - ) + if (countOnly || selected.size === 0 || out.length >= limit) continue + const header = `${resource.scope}/${resource.label}${resource.label === resource.id ? '' : ` (${resource.id})`}` + for (const [i, isMatch] of selected) { + out.push(`${header}:${i + 1}: ${clip(lines[i])}`) + if (isMatch) shownMatches++ } - const truncated = - total > shownMatches - ? `\n[${shownMatches} of ${total} matching lines shown — narrow with --scope, --in, or a tighter pattern]` - : '' - return finish(out.join('\n') + truncated) - }, + } + checkScan() + + if (countOnly) { + const breakdown = [...perScope.entries()].map(([s, n]) => `${s}=${n}`).join(' ') + return finish(`${total}${breakdown ? ` (${breakdown})` : ''}`) + } + if (out.length === 0) { + return finish( + `No matches for ${JSON.stringify(pattern)} in ${searched.join(', ')}${nameFilter ? ` within "${nameFilter}"` : ''}.` + ) + } + const truncated = + total > shownMatches + ? `\n[${shownMatches} of ${total} matching lines shown — narrow with --scope, --in, or a tighter pattern]` + : '' + return finish(out.join('\n') + truncated) +} + +export const universalGrepCommand: AgentCliEngine = { + execute: (positionals, runtime, flags) => + executeGrep(positionals, flags, { + signal: runtime.signal, + registry: runtime.resolvedSecretTraceRegistry, + async materialize(scopes, nameFilter, reportIncomplete) { + const bounded = gatedRuntime({ ...runtime, reportIncomplete }) + return nameFilter + ? materializeWithin(bounded, scopes, nameFilter) + : (await settleAll(scopes.map((scope) => materializeScope(bounded, scope)))).flat() + }, + }), } diff --git a/apps/sim/lib/mothership/agent-cli/execute.ts b/apps/sim/lib/mothership/agent-cli/execute.ts new file mode 100644 index 00000000000..428b981fa06 --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/execute.ts @@ -0,0 +1,69 @@ +import type { EmbeddedCliIdentity } from 'sim/embed' +import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' +import type { AgentCliExecutionContext } from '@/lib/mothership/agent-cli' +import { runCli } from '@/lib/mothership/agent-cli/run-cli' +import { applySink } from '@/lib/mothership/agent-cli/sink' +import { createWorkbenchFileProvenance } from '@/lib/mothership/agent-cli/workbench-file-provenance' +import type { + AgentCliAugmentationInvocation, + AgentCliRawResult, + AgentCliRequest, +} from '@/lib/mothership/generated/agent-cli' +import type { ResourceChange } from '@/lib/mothership/generated/resources' +import { TraceSpan } from '@/lib/mothership/generated/trace-spans-v1' +import { withCopilotSpan } from '@/lib/mothership/request/otel' +import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key' + +interface PreparedCliInvocation { + identity: EmbeddedCliIdentity + sessionKey: string | null + files?: ReturnType + resources: ResourceChange[] + augment(invocation: AgentCliAugmentationInvocation): Promise + curate?(result: AgentCliRawResult): Promise + projectResources?(resources: ResourceChange[]): ResourceChange[] +} + +/** The in-process endpoint, chat workbench session, file provenance, and effect sink of one invocation. */ +export function prepareCliSession(context: AgentCliExecutionContext) { + const sessionKey = context.chatId ? chatSandboxSessionKey(context.chatId) : null + return { + endpoint: getInternalApiBaseUrl(), + sessionKey, + files: sessionKey ? createWorkbenchFileProvenance({ ...context, sessionKey }) : undefined, + resources: [] as ResourceChange[], + } +} + +/** Owner adapters prepare authority and transport; every invocation shares execution and result delivery. */ +export async function executePreparedCliRequest( + request: AgentCliRequest, + prepared: PreparedCliInvocation +): Promise { + const { identity, sessionKey, files, resources } = prepared + const { invocation, sink } = request + identity.signal?.throwIfAborted() + let result: AgentCliRawResult + if (invocation.kind === 'stdout') { + result = { exitCode: 0, stdout: invocation.stdout, stderr: '' } + } else if (invocation.kind === 'service') { + throw new Error('Service invocation must use the service bridge') + } else { + result = await withCopilotSpan(TraceSpan.CopilotCliInvoke, undefined, () => + invocation.kind === 'augmentation' + ? prepared.augment(invocation) + : runCli(invocation.argv, identity, sessionKey, files) + ) + if (invocation.kind === 'cli' && prepared.curate) result = await prepared.curate(result) + } + if (resources.length) + result = { ...result, resources: [...resources, ...(result.resources ?? [])] } + if (result.resources?.length && prepared.projectResources) { + result = { ...result, resources: prepared.projectResources(result.resources) } + } + return sink + ? withCopilotSpan(TraceSpan.CopilotCliSink, undefined, () => + applySink(sink, sessionKey, result, identity.signal, files?.observeOutput) + ) + : result +} diff --git a/apps/sim/lib/mothership/agent-cli/file-copy-transport.ts b/apps/sim/lib/mothership/agent-cli/file-copy-transport.ts new file mode 100644 index 00000000000..563f383316a --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/file-copy-transport.ts @@ -0,0 +1,82 @@ +import type { ResourceDelegatedPrincipal } from '@sim/auth/principal' +import { NextRequest } from 'next/server' +import { fileCopyInputSchema } from '@/lib/api/contracts/file-copy-input' +import { v2CopyFileItemsContract } from '@/lib/api/contracts/v2/file-copy' +import { markCopilotResourceRequest } from '@/lib/api/server/routes/copilot-request' +import { V2_PARSE_DEFAULTS, v2InvalidBodyResponse } from '@/lib/api/server/routes/v2-json-route' +import { isV2ContractRoute } from '@/lib/api/server/routes/v2-route-pattern' +import { parseRequest } from '@/lib/api/server/validation' +import { requireResourceDelegation } from '@/lib/core/application/resource-delegation' +import { createScopedCliTransport } from '@/lib/mothership/agent-cli/scoped-transport' +import { + type CopilotExecutionContext, + createCopilotResourceApplicationPrincipal, + requireTrustedCopilotResourceExecutionContext, +} from '@/lib/mothership/auth/application-delegation' +import type { CopyFileItemsInput } from '@/lib/workspace-files/application/copy-authorization' +import { + FILE_COPY_DELEGATION_TTL_MS, + fileCopyOperation, +} from '@/lib/workspace-files/application/copy-operation' +import { v2Error } from '@/app/api/v2/lib/response' + +/** Carries one immutable paired grant to the registered compound operation without a wire credential. */ +function markCopilotFileCopyRequest( + request: Request, + context: CopilotExecutionContext, + target: CopyFileItemsInput +): ResourceDelegatedPrincipal { + const principal = createCopilotResourceApplicationPrincipal( + requireTrustedCopilotResourceExecutionContext(context), + { + audience: fileCopyOperation.delegationAudience, + ttlMs: FILE_COPY_DELEGATION_TTL_MS, + createDelegationId: (trusted) => trusted.toolCallId, + scope: { kind: 'file_copy', ...target }, + } + ) + markCopilotResourceRequest(request, (operation, audience) => + operation === fileCopyOperation && audience === fileCopyOperation.delegationAudience + ? principal + : undefined + ) + return principal +} + +/** Binds the native request body to one paired capability; the application rechecks both current owners. */ +export function createFileCopyCliTransport( + endpoint: string, + context: CopilotExecutionContext, + target: CopyFileItemsInput +): typeof fetch { + const bound = fileCopyInputSchema.parse(target) + return createScopedCliTransport(endpoint, { + async admitRequest(request, route) { + if (!isV2ContractRoute(request.method, route, v2CopyFileItemsContract)) { + return v2Error('BAD_REQUEST', 'CLI target is unavailable') + } + const parsed = await parseRequest( + v2CopyFileItemsContract, + new NextRequest(request.clone()), + {}, + { + ...V2_PARSE_DEFAULTS, + invalidJsonResponse: () => v2InvalidBodyResponse(request), + } + ) + if (!parsed.success) return parsed.response + request.signal.throwIfAborted() + try { + const principal = markCopilotFileCopyRequest(request, context, bound) + requireResourceDelegation(principal, { + audience: fileCopyOperation.delegationAudience, + services: fileCopyOperation.delegatedServices, + scope: { kind: 'file_copy', ...parsed.data.body }, + maxTtlMs: FILE_COPY_DELEGATION_TTL_MS, + }) + } catch { + return v2Error('FORBIDDEN', 'File copy authority is unavailable') + } + }, + }) +} diff --git a/apps/sim/lib/mothership/agent-cli/file-copy.ts b/apps/sim/lib/mothership/agent-cli/file-copy.ts new file mode 100644 index 00000000000..8ee839288e7 --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/file-copy.ts @@ -0,0 +1,57 @@ +import type { EmbeddedCliIdentity } from 'sim/embed' +import { fileCopyInputSchema } from '@/lib/api/contracts/file-copy-input' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { AgentCliExecutionContext } from '@/lib/mothership/agent-cli' +import { executePreparedCliRequest, prepareCliSession } from '@/lib/mothership/agent-cli/execute' +import { createFileCopyCliTransport } from '@/lib/mothership/agent-cli/file-copy-transport' +import { createResourceEffectTransport } from '@/lib/mothership/agent-cli/resource-effects' +import { createTracedCliTransport } from '@/lib/mothership/agent-cli/traced-transport' +import { requireTrustedCopilotResourceExecutionContext } from '@/lib/mothership/auth/application-delegation' +import type { AgentCliRawResult, AgentCliRequest } from '@/lib/mothership/generated/agent-cli' + +/** Executes one compound copy with both explicit owners and the original authoring invocation. */ +export async function executeFileCopyCliRequest( + request: AgentCliRequest, + context: AgentCliExecutionContext +): Promise { + const { invocation } = request + if ( + invocation.kind !== 'file-copy' || + request.fileOwner !== undefined || + request.workspaceId !== undefined || + request.curate !== undefined + ) { + throw new OrchestrationError( + 'validation', + 'File copy requires one explicit source and destination' + ) + } + const trustedContext = requireTrustedCopilotResourceExecutionContext(context) + const target = fileCopyInputSchema.parse({ + source: invocation.source, + destination: invocation.destination, + }) + const { endpoint, sessionKey, files, resources } = prepareCliSession(context) + const identity: EmbeddedCliIdentity = { + endpoint, + apiKey: 'mothership-in-process', + transport: createResourceEffectTransport( + endpoint, + createTracedCliTransport( + endpoint, + createFileCopyCliTransport(endpoint, trustedContext, target) + ), + resources + ), + ...(context.signal ? { signal: context.signal } : {}), + } + return executePreparedCliRequest(request, { + identity, + sessionKey, + files, + resources, + async augment() { + throw new OrchestrationError('validation', 'File copy does not support augmentations') + }, + }) +} diff --git a/apps/sim/lib/mothership/agent-cli/file-read-transport.ts b/apps/sim/lib/mothership/agent-cli/file-read-transport.ts index 3a62be5e96a..cb38de7bf8e 100644 --- a/apps/sim/lib/mothership/agent-cli/file-read-transport.ts +++ b/apps/sim/lib/mothership/agent-cli/file-read-transport.ts @@ -1,8 +1,9 @@ import { createLogger } from '@sim/logger' import { NextRequest } from 'next/server' import { v2DownloadFileContract, v2ReadFileTextContract } from '@/lib/api/contracts/v2/files' -import { matchV2Route } from '@/lib/api/server/routes/in-process-transport' +import type { MatchedRoute } from '@/lib/api/server/routes/in-process-transport' import { V2_PARSE_DEFAULTS } from '@/lib/api/server/routes/v2-json-route' +import { tryMatchV2Route, v2RoutePattern } from '@/lib/api/server/routes/v2-route-pattern' import { parseRequest } from '@/lib/api/server/validation' import { observeContentDelivery } from '@/lib/core/application' import { @@ -19,10 +20,8 @@ import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secr const logger = createLogger('MothershipFileReads') -/** Contract paths in the generated route table's `{param}` pattern syntax. */ -const routePattern = (path: string) => path.replace(/\[([^\]]+)\]/g, '{$1}') -const DOWNLOAD_PATTERN = routePattern(v2DownloadFileContract.path) -const TEXT_PATTERN = routePattern(v2ReadFileTextContract.path) +const DOWNLOAD_PATTERN = v2RoutePattern(v2DownloadFileContract) +const TEXT_PATTERN = v2RoutePattern(v2ReadFileTextContract) /** * Answers the two single-file reads Chat needs beyond their public routes — a chat upload by its @@ -46,14 +45,11 @@ export function createFileReadTransport(context: { return async (input, init) => { const url = new URL(input instanceof Request ? input.url : input) const method = (init?.method ?? (input instanceof Request ? input.method : 'GET')).toUpperCase() - let route: ReturnType = null - if (method === 'GET' && url.origin === base.origin && url.pathname.startsWith(`${basePath}/`)) { - try { - route = matchV2Route(url.pathname.slice(basePath.length)) - } catch { - /** A malformed percent encoding names no file; the route answers it. */ - } - } + /** A malformed percent encoding names no file; the route answers it. */ + const route: MatchedRoute | null = + method === 'GET' && url.origin === base.origin && url.pathname.startsWith(`${basePath}/`) + ? tryMatchV2Route(url.pathname.slice(basePath.length)) + : null if (!route || (route.pattern !== DOWNLOAD_PATTERN && route.pattern !== TEXT_PATTERN)) { return context.transport(input, init) } diff --git a/apps/sim/lib/mothership/agent-cli/index.ts b/apps/sim/lib/mothership/agent-cli/index.ts index bfc76115ffb..867223594fa 100644 --- a/apps/sim/lib/mothership/agent-cli/index.ts +++ b/apps/sim/lib/mothership/agent-cli/index.ts @@ -1,42 +1,19 @@ -import { createEmbeddedClient, type EmbeddedCliIdentity } from 'sim/embed' import { countWithheldContent } from '@/lib/core/application' -import { withWorkspaceInvocationScope } from '@/lib/core/application/workspace-invocation-scope' -import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' -import { curateBlockDetail } from '@/lib/mothership/agent-cli/curation' -import { AUGMENTATION_ENGINES, runEngine } from '@/lib/mothership/agent-cli/engines' -import { curateKnowledgeDocuments } from '@/lib/mothership/agent-cli/knowledge-curation' -import { isReadOnlyCliRequest, readOnlyCliTransport } from '@/lib/mothership/agent-cli/read-only' -import { runCli } from '@/lib/mothership/agent-cli/run-cli' +import { executeFileCopyCliRequest } from '@/lib/mothership/agent-cli/file-copy' +import { isReadOnlyCliRequest } from '@/lib/mothership/agent-cli/read-only' import { executeAgentCliService } from '@/lib/mothership/agent-cli/services' -import { applySink } from '@/lib/mothership/agent-cli/sink' -import { createAgentCliTransport } from '@/lib/mothership/agent-cli/transport' -import { createWorkbenchFileProvenance } from '@/lib/mothership/agent-cli/workbench-file-provenance' -import { resolveInvocationWorkspace } from '@/lib/mothership/application/workspace-target' -import { - COPILOT_APPLICATION_DELEGATION_TTL_MS, - createCopilotChatPrincipal, - createTrustedOrganizationCopilotPrincipal, -} from '@/lib/mothership/auth/application-delegation' +import { executeWorkspaceCliRequest } from '@/lib/mothership/agent-cli/workspace' import { describeWithheldContent } from '@/lib/mothership/content-delivery/telemetry' +import { getCopilotFileOwnerAdapter } from '@/lib/mothership/file-owners' import type { AgentCliRawResult, AgentCliRequest } from '@/lib/mothership/generated/agent-cli' -import type { ResourceChange } from '@/lib/mothership/generated/resources' -import { TraceSpan } from '@/lib/mothership/generated/trace-spans-v1' -import { withCopilotSpan } from '@/lib/mothership/request/otel' -import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key' import type { ServerToolContext } from '@/lib/mothership/tools/server/base-tool' -import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' export interface AgentCliExecutionContext extends Omit { - signal?: AbortSignal readOnly?: boolean + signal?: AbortSignal } -/** - * Executes one typed request from the worker: mint the caller's delegated identity, run - * the real CLI, named engine or service, and land the pre-parsed sink. All - * lanes preserve the server-minted identity so "the agent is the user" holds without any - * credential crossing to the worker. Success here means only "the invocation ran". - */ +/** Dispatches explicit file owners through registered policy adapters; non-file commands retain their scope. */ export async function executeAgentCliRequest( request: AgentCliRequest, context: AgentCliExecutionContext @@ -61,140 +38,27 @@ async function executeUncountedAgentCliRequest( stderr: 'This operation is unavailable during benchmark reference generation.', } } - if ( - request.invocation.kind === 'service' || - (request.invocation.kind === 'stdout' && - request.workspaceId === undefined && - (context.chatOrganizationId || context.organizationId)) - ) + if (request.invocation.kind === 'file-copy') { + return executeFileCopyCliRequest(request, context) + } + /** Services validate their own targets, including refusing a file owner, so they dispatch first. */ + if (request.invocation.kind === 'service') { return executeAgentCliService(request, context) - const target = await resolveInvocationWorkspace(context, request.workspaceId) - return withWorkspaceInvocationScope( - { - workspaceId: target.workspaceId, - organizationId: context.chatOrganizationId ?? context.organizationId, - }, - () => - executeBoundAgentCliRequest(request, { - ...context, - ...target, - chatOrganizationId: context.chatOrganizationId ?? context.organizationId, - }) - ) -} - -async function executeBoundAgentCliRequest( - request: AgentCliRequest, - context: AgentCliExecutionContext & { workspaceId: string } -): Promise { - /** The embedded client's required credential is opaque and never valid on the public API. */ - const apiKey = 'mothership-in-process' - const invocationIdentity = { - userId: context.userId, - workspaceId: context.workspaceId, - chatId: context.chatId, } - const endpoint = getInternalApiBaseUrl() - const sessionKey = context.chatId ? chatSandboxSessionKey(context.chatId) : null - const files = sessionKey ? createWorkbenchFileProvenance({ ...context, sessionKey }) : undefined - const resources: ResourceChange[] = [] - const identity = { - endpoint, - apiKey, - workspaceId: context.workspaceId, - transport: createAgentCliTransport({ - endpoint, - invocation: invocationIdentity, - registry: context.resolvedSecretTraceRegistry, - ...(files ? { files } : {}), - resources, - observeReads: - request.invocation.kind === 'cli' || - (request.invocation.kind === 'augmentation' && - AUGMENTATION_ENGINES[request.invocation.name]?.openReadResources === true), - }), - ...(context.signal ? { signal: context.signal } : {}), - } satisfies EmbeddedCliIdentity - if (context.readOnly) identity.transport = readOnlyCliTransport(identity.transport) - - const { invocation, sink } = request - let result: AgentCliRawResult - context.signal?.throwIfAborted() - if (invocation.kind === 'stdout') { - // Text the worker already holds (sliced, or worker-answered): only the sink applies. - result = { exitCode: 0, stdout: invocation.stdout, stderr: '' } - } else if (invocation.kind === 'augmentation') { - result = await withCopilotSpan(TraceSpan.CopilotCliInvoke, undefined, async () => - runEngine( - invocation.name, - invocation.positionals, - { - client: createEmbeddedClient(identity), - workspaceId: context.workspaceId, - userId: context.userId, - principal: createCopilotChatPrincipal( - invocationIdentity, - WORKSPACE_FILES_DELEGATION_AUDIENCE - ), - invocation: invocationIdentity, - ...(context.chatOrganizationId && context.chatId - ? { - chatOrganizationId: context.chatOrganizationId, - chatPrincipal: createTrustedOrganizationCopilotPrincipal( - { - userId: context.userId, - organizationId: context.chatOrganizationId, - chatId: context.chatId, - delegationId: `scratch:${context.chatId}`, - }, - { - audience: WORKSPACE_FILES_DELEGATION_AUDIENCE, - ttlMs: COPILOT_APPLICATION_DELEGATION_TTL_MS, - } - ), - } - : {}), - ...(context.chatId !== undefined ? { chatId: context.chatId } : {}), - resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, - signal: context.signal, - }, - invocation.flags - ) - ) - } else if (invocation.kind === 'cli') { - const { argv } = invocation - result = await withCopilotSpan(TraceSpan.CopilotCliInvoke, undefined, () => - runCli(argv, identity, sessionKey, files) + if (request.fileOwner) { + const owner = Object.freeze({ ...request.fileOwner }) + return getCopilotFileOwnerAdapter(owner).executeCli( + { ...request, fileOwner: owner }, + context, + owner ) - if (request.curate === 'knowledge-documents') result = curateKnowledgeDocuments(result) - if (result.exitCode === 0 && request.curate === 'block') { - result = await withCopilotSpan(TraceSpan.CopilotCliCurate, undefined, () => - curateBlockDetail(result, context) - ) - } - } else throw new Error('Service invocation must use the service bridge') - if (resources.length) - result = { ...result, resources: [...resources, ...(result.resources ?? [])] } - // Only organization chats address a workspace; a workspace chat's resources leave it implicit. - if (context.chatOrganizationId && result.resources?.length) - result = { - ...result, - resources: result.resources.map((effect) => { - switch (effect.op) { - case 'upsert': - return { ...effect, resource: { ...effect.resource, workspaceId: context.workspaceId } } - case 'remove': - return { ...effect, resource: { ...effect.resource, workspaceId: context.workspaceId } } - case 'refresh': - return { ...effect, resource: { ...effect.resource, workspaceId: context.workspaceId } } - case 'clear_view': - return { ...effect, resource: { ...effect.resource, workspaceId: context.workspaceId } } - } - }), - } - return sink - ? withCopilotSpan(TraceSpan.CopilotCliSink, undefined, () => - applySink(sink, sessionKey, result, context.signal, files?.observeOutput) - ) - : result + } + if ( + request.invocation.kind === 'stdout' && + request.workspaceId === undefined && + (context.chatOrganizationId || context.organizationId) + ) { + return executeAgentCliService(request, context) + } + return executeWorkspaceCliRequest(request, context) } diff --git a/apps/sim/lib/mothership/agent-cli/project-file-grep.test.ts b/apps/sim/lib/mothership/agent-cli/project-file-grep.test.ts new file mode 100644 index 00000000000..9af8f96bfdb --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-file-grep.test.ts @@ -0,0 +1,161 @@ +/** @vitest-environment node */ + +import { flushMacrotask } from '@sim/testing/helpers/async' +import { createDeferred } from '@sim/testing/helpers/deferred' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { list, textRead } = vi.hoisted(() => ({ list: vi.fn(), textRead: vi.fn() })) +vi.mock('@/lib/projects/files/application', () => ({ + listProjectFiles: { execute: list }, + readProjectFileText: { execute: textRead }, +})) +vi.mock('@/lib/mothership/application/execute-project-file-use-case', () => ({ + executeCopilotProjectFileUseCase: ( + _context: unknown, + useCase: { execute(input: unknown): Promise }, + input: unknown + ) => useCase.execute(input), +})) + +import { MAX_GREP_FILES } from '@/lib/mothership/agent-cli/engines/universal-grep' +import { grepProjectFiles } from '@/lib/mothership/agent-cli/project-file-grep' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +const invocation = { + kind: 'augmentation' as const, + name: 'grep', + positionals: ['needle'], + flags: { scope: 'files', in: 'files/notes' }, +} +const context = () => ({ + userId: 'reader', + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry([], { userId: 'reader' }), +}) +beforeEach(() => { + list.mockResolvedValue({ + files: [{ id: 'file', name: 'notes.txt', folderPath: '/' }], + nextKeys: null, + }) + textRead.mockResolvedValue({ + file: { id: 'file', name: 'notes.txt' }, + contentType: 'text/plain', + text: 'before\nneedle\nafter', + truncated: false, + degraded: false, + degradedReason: null, + secretProvenance: { status: 'exact', entries: [] }, + }) +}) + +describe('Project grep corpus', () => { + it('selects a file on a later inventory page and preserves matching line numbers', async () => { + list.mockImplementation(async ({ after }: { after?: { value: string }[] }) => + after?.[0]?.value === 'cursor' + ? { files: [{ id: 'file', name: 'notes.txt', folderPath: '/' }], nextKeys: null } + : { + files: [{ id: 'other', name: 'unrelated.txt' }], + nextKeys: [{ value: 'cursor', direction: 'asc' }], + } + ) + const result = await grepProjectFiles(invocation, context(), 'project') + expect(result).toEqual({ exitCode: 0, stdout: 'files/notes.txt (file):2: needle', stderr: '' }) + }) + it.each>([ + { scope: 'files,workflows' }, + { scope: 'files', in: 'workflows' }, + { scope: '' }, + ])('refuses incompatible Project scope before reading a corpus: %j', async (flags) => { + const result = await grepProjectFiles({ ...invocation, flags }, context(), 'project') + expect(result.exitCode).toBe(1) + expect(list).not.toHaveBeenCalled() + }) + it('withholds one file without exact source evidence and keeps searching the rest', async () => { + list.mockResolvedValue({ + files: [ + { id: 'file', name: 'notes.txt', folderPath: '/' }, + { id: 'other', name: 'notes-2.txt', folderPath: '/' }, + ], + nextKeys: null, + }) + textRead.mockResolvedValueOnce({ + file: { id: 'file', name: 'notes.txt' }, + contentType: 'text/plain', + text: 'needle', + truncated: false, + degraded: false, + degradedReason: null, + secretProvenance: { status: 'unknown' }, + }) + const ctx = context() + const result = await grepProjectFiles(invocation, ctx, 'project') + expect(result.stdout).toBe('files/notes-2.txt (other):2: needle') + expect(result.stderr).toContain('files/notes.txt (file): text withheld without its secret') + expect(ctx.resolvedSecretTraceRegistry.isPermanentlyIncomplete()).toBe(false) + }) + it('stops walking the inventory once the file budget is full', async () => { + list.mockImplementation(async ({ after }: { after?: { value: string }[] }) => ({ + files: Array.from({ length: MAX_GREP_FILES + 1 }, (_, index) => ({ + id: `${after ? 'late' : 'early'}-${index}`, + name: `notes-${index}.txt`, + folderPath: '/', + })), + nextKeys: after ? null : [{ value: 'cursor', direction: 'asc' }], + })) + const result = await grepProjectFiles(invocation, context(), 'project') + expect(list).toHaveBeenCalledTimes(1) + expect(result.stderr).toContain(`searched the first ${MAX_GREP_FILES} selected files`) + }) + it('marks truncated inventory incomplete instead of claiming a missing selector has no matches', async () => { + list.mockResolvedValue({ files: [], nextKeys: [{ value: 'cursor', direction: 'asc' }] }) + const result = await grepProjectFiles(invocation, context(), 'project') + expect(result.exitCode).toBe(1) + expect(result.stderr).toContain('listing stopped after 50 pages') + expect(result.stdout).toBe('') + }) +}) + +it('reads five Project files concurrently while preserving inventory result order', async () => { + const gates = Array.from({ length: 6 }, () => createDeferred()) + let active = 0 + let peak = 0 + list.mockResolvedValue({ + files: Array.from({ length: 6 }, (_, index) => ({ + id: `file-${index}`, + name: `notes-${index}.txt`, + folderPath: '/', + })), + nextKeys: null, + }) + textRead.mockImplementation(async ({ fileId }: { fileId: string }) => { + active++ + peak = Math.max(peak, active) + await gates[Number(fileId.slice(5))].promise + active-- + return { + file: { id: fileId, name: `${fileId}.txt` }, + contentType: 'text/plain', + text: 'needle', + truncated: false, + degraded: false, + degradedReason: null, + secretProvenance: { status: 'exact', entries: [] }, + } + }) + const pending = grepProjectFiles(invocation, context(), 'project') + try { + await flushMacrotask() + expect(active).toBe(5) + for (const index of [4, 3, 2, 1, 0, 5]) { + gates[index].resolve() + await flushMacrotask() + } + } finally { + for (const gate of gates) gate.resolve() + await pending + } + const result = await pending + expect(peak).toBe(5) + expect(result.stdout.split('\n')).toEqual( + Array.from({ length: 6 }, (_, index) => `files/notes-${index}.txt (file-${index}):1: needle`) + ) +}) diff --git a/apps/sim/lib/mothership/agent-cli/project-file-grep.ts b/apps/sim/lib/mothership/agent-cli/project-file-grep.ts new file mode 100644 index 00000000000..6667059df6c --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-file-grep.ts @@ -0,0 +1,119 @@ +import type { CursorKey } from '@/lib/api/list-query' +import type { AgentCliExecutionContext } from '@/lib/mothership/agent-cli' +import { + executeGrep, + type GrepMaterialized, + grepListingStoppedMessage, + MAX_GREP_BYTES_PER_FILE, + MAX_GREP_FILES, + MAX_GREP_LIST_PAGES, + mapGrepFileReads, + withGrepReadSlot, +} from '@/lib/mothership/agent-cli/engines/universal-grep' +import { admitProjectFileProvenance } from '@/lib/mothership/agent-cli/project-file-provenance' +import { executeCopilotProjectFileUseCase } from '@/lib/mothership/application/execute-project-file-use-case' +import type { AgentCliAugmentationInvocation } from '@/lib/mothership/generated/agent-cli' +import { listProjectFiles, readProjectFileText } from '@/lib/projects/files/application' + +/** Project grep materializes only freshly authorized files and imports evidence before matching. */ +export function grepProjectFiles( + invocation: AgentCliAugmentationInvocation, + context: AgentCliExecutionContext, + projectId: string +) { + /** Files are a Project's only grep scope, so an omitted scope means files. */ + const flags = + invocation.flags.scope === undefined + ? { ...invocation.flags, scope: 'files' } + : invocation.flags + return executeGrep(invocation.positionals, flags, { + signal: context.signal, + registry: context.resolvedSecretTraceRegistry, + scopeLimit: { + scopes: ['files'], + refusal: 'Project grep supports only --scope files. Search workspace resources separately.', + }, + async materialize(_scopes, nameFilter, reportIncomplete) { + const selectedFiles: { + item: Awaited>['files'][number] + label: string + }[] = [] + let after: CursorKey[] | undefined + /** Selection runs per page so a full file budget stops the inventory walk. */ + inventory: for (let pageNumber = 0; pageNumber < MAX_GREP_LIST_PAGES; pageNumber++) { + context.signal?.throwIfAborted() + const page = await withGrepReadSlot( + () => + executeCopilotProjectFileUseCase( + context, + listProjectFiles, + { projectId, sortBy: 'name', sortOrder: 'asc', limit: 100, after }, + { projectId } + ), + context.signal + ) + for (const item of page.files) { + const folder = (item.folderPath ?? '').replace(/^\/+|\/+$/g, '') + const label = folder ? `${folder}/${item.name}` : item.name + if ( + nameFilter && + item.id.toLowerCase() !== nameFilter && + !label.toLowerCase().includes(nameFilter) + ) + continue + if (selectedFiles.length >= MAX_GREP_FILES) { + reportIncomplete( + `files: searched the first ${MAX_GREP_FILES} selected files; narrow with --in files/.` + ) + break inventory + } + selectedFiles.push({ item, label }) + } + if (!page.nextKeys) break + after = page.nextKeys + if (pageNumber === MAX_GREP_LIST_PAGES - 1) + reportIncomplete(grepListingStoppedMessage('files')) + } + return mapGrepFileReads(selectedFiles, async ({ item, label }): Promise => { + context.signal?.throwIfAborted() + const target = { projectId, fileId: item.id } + let read: Awaited> + try { + read = await withGrepReadSlot( + () => + executeCopilotProjectFileUseCase( + context, + readProjectFileText, + { ...target, maxBytes: MAX_GREP_BYTES_PER_FILE, skipFolderPath: true }, + target + ), + context.signal + ) + } catch { + context.signal?.throwIfAborted() + reportIncomplete( + `files/${label} (${item.id}): could not read text; use files read to inspect the error.` + ) + return { scope: 'files', id: item.id, label, text: null } + } + if ( + !(await admitProjectFileProvenance( + context.resolvedSecretTraceRegistry, + read.secretProvenance + )) + ) { + reportIncomplete( + `files/${label} (${item.id}): text withheld without its secret provenance; use files read to inspect it.` + ) + return { scope: 'files', id: item.id, label, text: null } + } + if (read.degraded || read.truncated) { + reportIncomplete( + `files/${label} (${item.id}): text extraction is ${read.degraded ? 'degraded' : 'truncated'}; use files read or files get to inspect it.` + ) + } + return { scope: 'files', id: item.id, label, text: read.degraded ? null : read.text } + }) + }, + }) +} diff --git a/apps/sim/lib/mothership/agent-cli/project-file-provenance.ts b/apps/sim/lib/mothership/agent-cli/project-file-provenance.ts new file mode 100644 index 00000000000..00c0e19916f --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-file-provenance.ts @@ -0,0 +1,52 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' +import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +/** + * Import the bound source entries before any Project file bytes reach an agent consumer. A caller + * without a registry is refused outright: its egress could not redact and would withhold the + * result anyway. With one, content recording no secret passes as it does for a workspace file + * (`unrecorded`, or an exact empty set), while `unknown` or unimportable secrets are refused. + */ +export async function importProjectFileProvenance( + registry: ResolvedSecretTraceRegistry | undefined, + source: WorkspaceFileSecretProvenance | undefined +) { + if ( + registry && + !registry.isPermanentlyIncomplete() && + (source?.status === 'unrecorded' || (source?.status === 'exact' && source.entries.length === 0)) + ) + return + if ( + !registry || + source?.status !== 'exact' || + !(await importDurableSecretProvenance(registry, source)) || + registry.isPermanentlyIncomplete() + ) { + registry?.markIncomplete('project-file-provenance-unavailable') + throw new OrchestrationError( + 'forbidden', + 'File content cannot be delivered without its secret provenance' + ) + } +} + +/** + * Imports one search candidate's bound entries without latching the registry. False means this + * file's text must stay out of the result; the search continues and reports it incomplete, so an + * unknown sidecar on a file the agent never sees does not refuse the rest of the turn. + */ +export async function admitProjectFileProvenance( + registry: ResolvedSecretTraceRegistry | undefined, + source: WorkspaceFileSecretProvenance | undefined +): Promise { + if (!registry || registry.isPermanentlyIncomplete()) return false + if (source?.status === 'unrecorded') return true + if (source?.status !== 'exact') return false + return ( + source.entries.length === 0 || + (await importDurableSecretProvenance(registry, source, undefined, { atomic: true })) + ) +} diff --git a/apps/sim/lib/mothership/agent-cli/project-file-read.integration.ts b/apps/sim/lib/mothership/agent-cli/project-file-read.integration.ts new file mode 100644 index 00000000000..f03804d7cdb --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-file-read.integration.ts @@ -0,0 +1,751 @@ +import { mkdtempSync } from 'node:fs' +import { mkdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { db } from '@sim/db' +import { + permissions, + user, + userStats, + workspace, + workspaceFileSearchRevision, + workspaceFileSecretProvenance, + workspaceFiles, +} from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { setUploadDirServer, uploadsSetupMock } from '@sim/testing/mocks/uploads-setup.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { Document, Packer, Paragraph } from 'docx' +import { eq } from 'drizzle-orm' +import sharp from 'sharp' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { redisUrl } = await vi.hoisted(async () => { + const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') + const redisUrl = readTestRedisUrl() + if (redisUrl) process.env.REDIS_URL = redisUrl + return { redisUrl } +}) + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) + +vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock) + +import { observeContentDelivery } from '@/lib/core/application/content-delivery' +import { closeRedisConnection } from '@/lib/core/config/redis' +import { encryptSecret } from '@/lib/core/security/encryption' +import { type AgentCliExecutionContext, executeAgentCliRequest } from '@/lib/mothership/agent-cli' +import { createCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' +import { createRegistryContentGate } from '@/lib/mothership/content-delivery' +import type { AgentCliAugmentationInvocation } from '@/lib/mothership/generated/agent-cli' +import { projectToolResultForCopilot } from '@/lib/mothership/request/tools/resolved-secret-result' +import { createProjectFile } from '@/lib/projects/files/application' +import { storeCompiledDoc } from '@/lib/uploads/documents/compiled-store' +import { fileDocumentInputIdentity } from '@/lib/uploads/documents/input-identity' +import { createWorkspaceFile } from '@/lib/workspace-files/application/create-workspace-file' +import { indexWorkspaceFileForSearch } from '@/lib/workspace-files/search/indexing' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +const storageRoot = mkdtempSync(join(tmpdir(), 'sim-project-cli-reads-')) +setUploadDirServer(storageRoot) +const fixtures: { userId: string; workspaceId: string; projectId: string }[] = [] +const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = + [] + +beforeEach(() => { + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation(async (flag) => flag === 'projects') + vi.stubEnv('PROJECTS', 'true') + vi.stubEnv('FREE_STORAGE_LIMIT_GB', '') +}) + +function check(name: string, run: () => Promise, requiresRedis = false) { + it.skipIf(requiresRedis && !redisUrl)(name, async () => { + const started = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} + +async function fixture() { + const userId = generateId() + const workspaceId = generateId() + await db.insert(user).values({ + id: userId, + email: `${userId}@project-read.invalid`, + name: 'File reader', + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + }) + await db.insert(userStats).values({ id: generateId(), userId }) + await insertWorkspaceFixture(db, { + id: workspaceId, + ownerId: userId, + billedAccountUserId: userId, + name: 'Read environment', + }) + await db.insert(permissions).values({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: 'admin', + }) + const [binding] = await db + .select({ projectId: workspace.projectId }) + .from(workspace) + .where(eq(workspace.id, workspaceId)) + if (!binding?.projectId) throw new Error('Project fixture missing') + const ids = { userId, workspaceId, projectId: binding.projectId } + fixtures.push(ids) + const context: AgentCliExecutionContext = { + userId, + workspaceId, + requestMode: 'agent', + copilotToolExecution: true, + toolCallId: generateId(), + copilotResourceAdmission: createCopilotResourceAdmission({ + userId, + invocation: { kind: 'workspace', workspaceId }, + }), + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry([], { userId, workspaceId }), + } + const principal = createSessionPrincipal({ userId }) + return { ...ids, context, principal } +} + +/** Chat runs every CLI call under the tool executor's registry gate; reads here do the same. */ +function runCli( + request: Parameters[0], + context: AgentCliExecutionContext +) { + const gate = createRegistryContentGate({ + registry: context.resolvedSecretTraceRegistry, + workspaceId: context.workspaceId, + actorUserId: context.userId, + }) + return observeContentDelivery(gate, () => executeAgentCliRequest(request, context)) +} + +function read( + projectId: string, + context: AgentCliExecutionContext, + reference: string, + flags: AgentCliAugmentationInvocation['flags'] = {}, + name = 'files read' +) { + return runCli( + { + fileOwner: { entityType: 'project', entityId: projectId }, + invocation: { kind: 'augmentation', name, positionals: [reference], flags }, + }, + context + ) +} + +async function imageContent() { + return ( + await sharp(Buffer.from([25, 80, 150, 255]), { + raw: { width: 1, height: 1, channels: 4 }, + }) + .png() + .toBuffer() + ).toString('base64') +} + +check( + 'Project files without recorded provenance stay readable, as workspace files do', + async () => { + const f = await fixture() + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'legacy-notes.txt', + contentType: 'text/plain', + content: 'legacy-line', + encoding: 'utf-8', + }, + }) + // As stored when an unrecorded workspace file is copied into a Project. + await db + .update(workspaceFileSecretProvenance) + .set({ status: 'unrecorded', entries: [] }) + .where(eq(workspaceFileSecretProvenance.fileId, file.id)) + const result = await read(f.projectId, f.context, file.id) + expect(result.exitCode, result.stderr).toBe(0) + expect(JSON.parse(result.stdout).text).toContain('legacy-line') + const grep = await read(f.projectId, f.context, 'legacy-line', {}, 'grep') + expect(grep.exitCode, grep.stderr).toBe(0) + expect(grep.stdout).toContain(':1: legacy-line') + } +) + +check( + 'a Project file with no text decoder reads as binary metadata, as in a workspace', + async () => { + const f = await fixture() + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'archive.bin', + contentType: 'application/octet-stream', + content: Buffer.from([0, 1, 2, 3, 255]).toString('base64'), + encoding: 'base64', + }, + }) + const result = await read(f.projectId, f.context, file.id) + expect(result.exitCode, result.stderr).toBe(0) + expect(JSON.parse(result.stdout)).toMatchObject({ + representation: 'binary', + contentAvailable: false, + }) + } +) + +check('Project grep searches files when no scope is given', async () => { + const f = await fixture() + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'unscoped-notes.txt', + contentType: 'text/plain', + content: 'alpha\nunscoped-needle\nomega', + encoding: 'utf-8', + }, + }) + const result = await read(f.projectId, f.context, 'unscoped-needle', {}, 'grep') + expect(result.exitCode, result.stderr).toBe(0) + expect(result.stdout).toContain(`files/unscoped-notes.txt (${file.id}):2:`) +}) + +check('benchmark reference generation cannot create Project files', async () => { + const f = await fixture() + const name = `read-only-${generateId()}.txt` + const result = await runCli( + { + fileOwner: { entityType: 'project', entityId: f.projectId }, + invocation: { + kind: 'cli', + argv: ['projects', 'files', 'create', f.projectId, '--name', name, '--content', 'x'], + }, + }, + { ...f.context, readOnly: true } + ) + expect(result.exitCode).not.toBe(0) + expect(result.stdout + result.stderr).toContain('can only read') + const stored = () => + db + .select({ id: workspaceFiles.id }) + .from(workspaceFiles) + .where(eq(workspaceFiles.originalName, name)) + expect(await stored()).toEqual([]) + // The same command creates the file outside reference generation. + const created = await runCli( + { + fileOwner: { entityType: 'project', entityId: f.projectId }, + invocation: { + kind: 'cli', + argv: ['projects', 'files', 'create', f.projectId, '--name', name, '--content', 'x'], + }, + }, + f.context + ) + expect(created.exitCode, created.stderr).toBe(0) + expect(await stored()).toHaveLength(1) +}) + +check('Project grep matches public bytes while excluding classified secret text', async () => { + const f = await fixture() + const secret = 'synthetic-project-grep-canary' + const { encrypted } = await encryptSecret(secret) + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'grep-notes.txt', + contentType: 'text/plain', + content: `before\n${secret}\nafter`, + encoding: 'utf-8', + secretProvenance: { + status: 'exact', + entries: [ + { + encryptedValue: encrypted, + sourceUserId: f.userId, + sourceWorkspaceId: f.workspaceId, + name: 'CANARY', + }, + ], + }, + }, + }) + const result = await read( + f.projectId, + f.context, + 'canary', + { scope: 'files', in: `files/${file.id}` }, + 'grep' + ) + expect(result.exitCode, result.stderr).toBe(0) + expect(result.stdout).toContain('No matches') + expect(result.stdout).not.toContain(secret) + const visible = await read( + f.projectId, + f.context, + 'before', + { scope: 'files', in: `files/${file.id}` }, + 'grep' + ) + expect(visible.exitCode, visible.stderr).toBe(0) + expect(visible.stdout).toContain(`files/grep-notes.txt (${file.id}):1:`) + expect(visible.stdout).not.toContain(secret) + + // Text reads match workspace files: admission imports the provenance, so the model egress + // masks the secret while every other line stays readable. A line window is redacted before it + // is cut, so it never holds a fragment the egress could miss. + const egress = (output: unknown) => + JSON.stringify( + projectToolResultForCopilot({ success: true, output }, f.context.resolvedSecretTraceRegistry) + ) + const whole = await read(f.projectId, f.context, file.id) + expect(whole.exitCode, whole.stderr).toBe(0) + expect(JSON.parse(whole.stdout)).toMatchObject({ representation: 'text' }) + const wholeEgress = egress(JSON.parse(whole.stdout)) + expect(wholeEgress).toContain('before') + expect(wholeEgress).toContain('after') + expect(wholeEgress).not.toContain(secret) + const ranged = await read(f.projectId, f.context, file.id, { offset: '2', limit: '1' }) + expect(ranged.exitCode, ranged.stderr).toBe(0) + expect(JSON.parse(ranged.stdout).lineRange).toMatchObject({ offset: 2, lineCount: 1 }) + expect(ranged.stdout).not.toContain(secret) + + // Byte reads stay opaque: no redaction reaches bytes, so a secret still refuses them. + const bytes = await read(f.projectId, f.context, file.id, {}, 'files view') + expect(bytes.exitCode).not.toBe(0) + expect(bytes.stdout + bytes.stderr).not.toContain(secret) +}) + +check('raw Project source downloads refuse known secrets outside a workbench session', async () => { + const f = await fixture() + const secret = 'synthetic-project-source-canary' + const { encrypted } = await encryptSecret(secret) + const source = (fileId: string) => + runCli( + { + fileOwner: { entityType: 'project', entityId: f.projectId }, + invocation: { kind: 'cli', argv: ['projects', 'files', 'source', f.projectId, fileId] }, + }, + f.context + ) + const { file: secretFile } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'source-secret.txt', + contentType: 'text/plain', + content: `before\n${secret}\nafter`, + encoding: 'utf-8', + secretProvenance: { + status: 'exact', + entries: [ + { + encryptedValue: encrypted, + sourceUserId: f.userId, + sourceWorkspaceId: f.workspaceId, + name: 'CANARY', + }, + ], + }, + }, + }) + const refused = await source(secretFile.id) + expect(refused.exitCode).not.toBe(0) + expect(refused.stderr).toContain( + 'File cannot be sent to a model because its secret provenance is unavailable' + ) + expect(refused.stdout + refused.stderr).not.toContain(secret) + + const { file: publicFile } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'source-public.txt', + contentType: 'text/plain', + content: 'public-source-bytes', + encoding: 'utf-8', + secretProvenance: { status: 'exact', entries: [] }, + }, + }) + const delivered = await source(publicFile.id) + expect(delivered.exitCode, delivered.stderr).toBe(0) + expect(delivered.stdout).toContain('public-source-bytes') +}) + +describe('Project CLI representations against PostgreSQL and stored bytes', () => { + check( + 'native workspace content search reaches its collection operation instead of a file named search', + async () => { + const f = await fixture() + const result = await runCli( + { + fileOwner: { entityType: 'workspace', entityId: f.workspaceId }, + invocation: { + kind: 'cli', + argv: ['files', 'search', '--query', 'absent-needle', '--mode', 'exact'], + }, + }, + f.context + ) + expect(result.exitCode, result.stderr).toBe(0) + expect(JSON.parse(result.stdout)).toMatchObject({ count: 0, results: [], complete: true }) + } + ) + + check( + 'native indexed workspace delivery withholds secret-bearing and untracked excerpts', + async () => { + const f = await fixture() + const owner = { entityType: 'workspace' as const, entityId: f.workspaceId } + const { encrypted: rootSecret } = await encryptSecret('root-native-secret') + const { encrypted: inputSecret } = await encryptSecret('input-native-secret') + const create = async (name: string, content: string, encryptedValue: string) => + createWorkspaceFile.execute({ + principal: f.principal, + input: { + workspaceId: f.workspaceId, + name, + contentType: 'text/plain', + content, + encoding: 'utf-8', + exactName: true, + secretProvenance: { + status: 'exact', + entries: [ + { encryptedValue, sourceUserId: f.userId, sourceWorkspaceId: f.workspaceId }, + ], + }, + }, + }) + const { file: dependency } = await create('input.txt', 'input-native-secret', inputSecret) + const source = `getFileBase64('${dependency.id}')` + const { file } = await create('generated.docx', source, rootSecret) + const [inputRow] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.id, dependency.id)) + const artifact = await Packer.toBuffer( + new Document({ + sections: [ + { + children: [ + new Paragraph('nativeprovenanceneedle root-native-secret input-native-secret'), + ], + }, + ], + }) + ) + await storeCompiledDoc( + owner, + source, + 'docx', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + artifact, + fileDocumentInputIdentity(owner, [inputRow]) + ) + const token = new Date() + const [revision] = await db + .update(workspaceFileSearchRevision) + .set({ dispatchedAt: token }) + .where(eq(workspaceFileSearchRevision.fileId, file.id)) + .returning() + await indexWorkspaceFileForSearch( + { + owner, + fileId: file.id, + sourceContentUpdatedAt: revision.sourceContentUpdatedAt.toISOString(), + dispatchToken: token.toISOString(), + }, + new AbortController().signal + ) + const invocation = { + kind: 'cli' as const, + argv: ['files', 'search', '--query', 'nativeprovenanceneedle', '--mode', 'exact'], + } + const result = await runCli({ fileOwner: owner, invocation }, f.context) + expect(result.exitCode, result.stderr).toBe(0) + expect(JSON.parse(result.stdout).results[0].fileId).toBe(file.id) + expect(JSON.parse(result.stdout).results[0].text).toContain('Preview withheld') + expect(result.stdout).not.toContain('root-native-secret') + expect(result.stdout).not.toContain('input-native-secret') + await db + .delete(workspaceFileSecretProvenance) + .where(eq(workspaceFileSecretProvenance.fileId, dependency.id)) + const rejected = await runCli( + { fileOwner: owner, invocation }, + { + ...f.context, + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry([], { + userId: f.userId, + workspaceId: f.workspaceId, + }), + } + ) + expect(rejected.exitCode, rejected.stderr).toBe(0) + expect(JSON.parse(rejected.stdout).results[0].text).toContain('Preview withheld') + expect(rejected.stdout).not.toContain('input-native-secret') + }, + true + ) + + check( + 'explicit workspace file owners preserve native reads and reject conflicting invocation scope', + async () => { + const f = await fixture() + const { file } = await createWorkspaceFile.execute({ + principal: f.principal, + input: { + workspaceId: f.workspaceId, + name: 'workspace-owner.md', + contentType: 'text/markdown', + content: 'Canonical workspace documentation', + encoding: 'utf-8', + exactName: true, + }, + }) + const owner = { entityType: 'workspace' as const, entityId: f.workspaceId } + const invocation = { kind: 'cli' as const, argv: ['files', 'describe', file.id] } + const explicit = await runCli({ fileOwner: owner, invocation }, f.context) + expect(explicit.exitCode, explicit.stderr).toBe(0) + expect(explicit.stdout).toContain(file.id) + expect(explicit.resources).toContainEqual({ + op: 'upsert', + readOnly: true, + resource: expect.objectContaining({ type: 'file', id: file.id, owner }), + }) + const legacy = await runCli({ workspaceId: f.workspaceId, invocation }, f.context) + expect(legacy.exitCode, legacy.stderr).toBe(0) + expect(legacy.stdout).toBe(explicit.stdout) + await expect( + runCli({ fileOwner: owner, workspaceId: generateId(), invocation }, f.context) + ).rejects.toMatchObject({ code: 'validation' }) + const other = await fixture() + await expect( + runCli({ fileOwner: { ...owner, entityId: other.workspaceId }, invocation }, f.context) + ).rejects.toMatchObject({ code: 'not_found' }) + } + ) + + check( + 'generated Project pages reach the model as rendered documents with canonical ownership', + async () => { + const f = await fixture() + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'design.html', + contentType: 'text/x-sim-page', + content: + '---\ntitle: Project architecture\n---\n\n# Shared design\n\nThe durable Project context.', + encoding: 'utf-8', + }, + }) + const result = await read(f.projectId, f.context, file.id) + expect(result.exitCode, result.stderr).toBe(0) + const output = JSON.parse(result.stdout) + expect(output.representation).toBe('text') + expect(output.type).toBe('text/html') + expect(output.text).toContain('The durable Project context.') + expect(output.path).toBe(`projects/${f.projectId}/files/design`) + expect(result.resources?.[0]).toMatchObject({ + resource: { id: file.id, owner: { entityType: 'project', entityId: f.projectId } }, + }) + } + ) + + check( + 'Project references read the selected owner and preserve text line selection and resource ownership', + async () => { + const f = await fixture() + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'architecture.md', + contentType: 'text/markdown', + content: 'Heading\nProject line\nEnd', + encoding: 'utf-8', + }, + }) + for (const reference of [file.id, `projects/${f.projectId}/files/architecture.md`]) { + const result = await read(f.projectId, f.context, reference, { offset: '2', limit: '1' }) + expect(result.exitCode, result.stderr).toBe(0) + expect(JSON.parse(result.stdout)).toMatchObject({ + fileId: file.id, + path: `projects/${f.projectId}/files/architecture.md`, + representation: 'text', + text: 'Project line', + }) + expect(result.resources).toContainEqual({ + op: 'upsert', + readOnly: true, + resource: { + type: 'file', + id: file.id, + title: file.name, + owner: { entityType: 'project', entityId: f.projectId }, + }, + }) + } + const invalidFlags: AgentCliAugmentationInvocation['flags'][] = [ + { offset: '1', render: true }, + { limit: true }, + { 'max-bytes': '0' }, + { pages: '' }, + ] + for (const flags of invalidFlags) { + const result = await read(f.projectId, f.context, file.id, flags) + expect(result.exitCode).not.toBe(0) + expect(result.stdout).toBe('') + } + const foreign = await fixture() + const result = await read( + f.projectId, + f.context, + `projects/${foreign.projectId}/files/architecture.md` + ) + expect(result.exitCode).not.toBe(0) + expect(result.stdout).toBe('') + } + ) + + check( + 'visual Project reads expose the authorized image bytes and owner without a workspace file route', + async () => { + const f = await fixture() + const content = await imageContent() + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'diagram.png', + contentType: 'image/png', + content, + encoding: 'base64', + }, + }) + for (const name of ['files read', 'files view']) { + const result = await read(f.projectId, f.context, file.id, {}, name) + expect(result.exitCode, result.stderr).toBe(0) + expect(JSON.parse(result.stdout)).toMatchObject({ + fileId: file.id, + representation: 'visual', + path: `projects/${f.projectId}/files/diagram.png`, + }) + expect(result.observations).toMatchObject([{ resourceId: file.id, mediaType: 'image/png' }]) + const observation = result.observations?.[0] + if (!observation) throw new Error('Image observation missing') + const pixels = await sharp(Buffer.from(observation.data, 'base64')) + .ensureAlpha() + .raw() + .toBuffer() + expect([...pixels]).toEqual([25, 80, 150, 255]) + } + } + ) + + check( + 'missing file tracking or a missing caller registry prevents content delivery after a successful metadata lookup', + async () => { + const f = await fixture() + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'restricted.md', + contentType: 'text/markdown', + content: 'must not be emitted', + encoding: 'utf-8', + }, + }) + const missingRegistry = await read( + f.projectId, + { ...f.context, resolvedSecretTraceRegistry: undefined }, + file.id + ) + expect(missingRegistry.exitCode).not.toBe(0) + expect(missingRegistry.stdout).toBe('') + expect(missingRegistry.observations).toBeUndefined() + await db + .delete(workspaceFileSecretProvenance) + .where(eq(workspaceFileSecretProvenance.fileId, file.id)) + const missingTracking = await read(f.projectId, f.context, file.id) + expect(missingTracking.exitCode).not.toBe(0) + expect(missingTracking.stdout).toBe('') + expect(missingTracking.observations).toBeUndefined() + } + ) + + check( + 'visual content with known secret provenance is not emitted as an opaque model observation', + async () => { + const f = await fixture() + const { encrypted } = await encryptSecret('secret-visible-in-image') + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'restricted.png', + contentType: 'image/png', + content: await imageContent(), + encoding: 'base64', + secretProvenance: { + status: 'exact', + entries: [ + { + encryptedValue: encrypted, + sourceUserId: f.userId, + sourceWorkspaceId: f.workspaceId, + name: 'TOKEN', + }, + ], + }, + }, + }) + const result = await read(f.projectId, f.context, file.id) + expect(result.exitCode).not.toBe(0) + expect(result.stdout).toBe('') + expect(result.observations).toBeUndefined() + } + ) +}) + +afterAll(async () => { + for (const f of fixtures) { + await db.delete(workspaceFiles).where(eq(workspaceFiles.projectId, f.projectId)) + await deleteWorkspaceFixture(db, eq(workspace.id, f.workspaceId)) + await db.delete(user).where(eq(user.id, f.userId)) + } + await rm(storageRoot, { recursive: true, force: true }) + const report = process.env.PROJECT_FILE_READ_REPORT_PATH ?? 'test-results/project-file-read.json' + await mkdir(dirname(report), { recursive: true }) + await writeFile(report, JSON.stringify({ checks }, null, 2)) + await closeRedisConnection() + await db.$client.end() +}) diff --git a/apps/sim/lib/mothership/agent-cli/project-file-read.ts b/apps/sim/lib/mothership/agent-cli/project-file-read.ts new file mode 100644 index 00000000000..fb80fc8220f --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-file-read.ts @@ -0,0 +1,183 @@ +import { v2ReadFileTextQuerySchema } from '@/lib/api/contracts/v2/files' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { AgentCliExecutionContext } from '@/lib/mothership/agent-cli' +import { fileReadFailure } from '@/lib/mothership/agent-cli/engines/file-view' +import { + isDefaultVisualMediaType, + observePrivateFile, + presentObservedText, +} from '@/lib/mothership/agent-cli/engines/observe-private-file' +import { importProjectFileProvenance } from '@/lib/mothership/agent-cli/project-file-provenance' +import { agentCliFail } from '@/lib/mothership/agent-cli/types' +import { executeCopilotProjectFileUseCase } from '@/lib/mothership/application/execute-project-file-use-case' +import type { + AgentCliAugmentationInvocation, + AgentCliRawResult, +} from '@/lib/mothership/generated/agent-cli' +import { + readProjectFileArtifact, + readProjectFileText, + resolveProjectFileReference, +} from '@/lib/projects/files/application' +import { workspaceFileVfsPath } from '@/lib/uploads/contexts/workspace' +import { + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { + MAX_TEXT_EXTRACTION_BYTES, + observedMediaType, + resolveEffectiveMimeType, +} from '@/lib/uploads/utils/file' +import { normalizeMimeType } from '@/lib/uploads/utils/mime' +import { fileOwnerVfsPath } from '@/lib/workspace-files/owner-paths' +import { workspaceFileTextFormat } from '@/lib/workspace-files/text-format' + +const projectReadQuerySchema = v2ReadFileTextQuerySchema.omit({ workspaceId: true }) +const READ_FLAGS = new Set(['max-bytes', 'offset', 'limit', 'render', 'pages']) + +/** Project observation reuses file decoders after canonical reference and exact-file authorization. */ +export async function readProjectFileForAgent( + invocation: AgentCliAugmentationInvocation, + context: AgentCliExecutionContext, + projectId: string +): Promise { + if (invocation.name !== 'files read' && invocation.name !== 'files view') + return agentCliFail('This Project file operation is not supported.') + const { flags, positionals } = invocation + const reference = positionals[0] + if (!reference || positionals.length !== 1) + return agentCliFail('Project file reads require one file ID or Project file path.') + for (const flag of Object.keys(flags)) { + if (!READ_FLAGS.has(flag)) return agentCliFail(`Unsupported Project file read flag: --${flag}`) + } + for (const flag of ['max-bytes', 'offset', 'limit']) { + if (flags[flag] === true) return agentCliFail(`--${flag} requires a value.`) + } + if (flags.render !== undefined && flags.render !== true) + return agentCliFail('--render is a flag without a value.') + if (flags.pages !== undefined && (typeof flags.pages !== 'string' || !flags.pages.trim())) + return agentCliFail('--pages requires a page number or range.') + const textRange = flags.offset !== undefined || flags.limit !== undefined + const explicitVisual = + invocation.name === 'files view' || flags.render === true || flags.pages !== undefined + if (textRange && explicitVisual) + return agentCliFail('Use text line ranges or visual page options in one read, not both.') + const query = projectReadQuerySchema.safeParse({ + ...(flags['max-bytes'] !== undefined ? { maxBytes: flags['max-bytes'] } : {}), + ...(flags.offset !== undefined ? { offset: flags.offset } : {}), + ...(flags.limit !== undefined ? { limit: flags.limit } : {}), + }) + if (!query.success) + return agentCliFail(query.error.issues.map((issue) => issue.message).join('; ')) + try { + context.signal?.throwIfAborted() + const resolved = await executeCopilotProjectFileUseCase( + context, + resolveProjectFileReference, + { projectId, fileReference: reference }, + { projectId } + ) + const target = { projectId, fileId: resolved.file.id } + const maxBytes = query.data.maxBytes ?? MAX_TEXT_EXTRACTION_BYTES + /** A generated document is chosen by the type it renders to, as the workspace reader does. */ + const resolvedType = observedMediaType(resolved.file.type, resolved.file.name) + /** + * Text reads admit the file as `complete`, so the registry masks its secrets as it does for a + * workspace file; only visual and undecodable reads take its stored bytes, which no redaction + * reaches. Images and PDFs default to visual unless a line range asks for their text. + */ + const readsText = + !explicitVisual && + Boolean(workspaceFileTextFormat({ name: resolved.file.name, type: resolvedType })) && + (textRange || !isDefaultVisualMediaType(resolvedType)) + let file: typeof resolved.file + let result: Awaited> + if (readsText) { + const read = await executeCopilotProjectFileUseCase( + context, + readProjectFileText, + { ...target, maxBytes, offset: query.data.offset, limit: query.data.limit }, + target + ) + await importProjectFileProvenance(context.resolvedSecretTraceRegistry, read.secretProvenance) + context.signal?.throwIfAborted() + file = read.file + result = presentObservedText( + { + name: file.name, + path: fileOwnerVfsPath(file.owner, workspaceFileVfsPath(file)), + type: normalizeMimeType(resolveEffectiveMimeType(read.contentType, file.name)), + source: 'project', + fileId: file.id, + }, + read + ) + } else { + const artifact = await executeCopilotProjectFileUseCase( + context, + readProjectFileArtifact, + { ...target, maxBytes, forModel: explicitVisual }, + target + ) + await importProjectFileProvenance( + context.resolvedSecretTraceRegistry, + artifact.secretProvenance + ) + context.signal?.throwIfAborted() + file = artifact.file + // Default visual selection uses the authorized artifact's MIME, not earlier discovery metadata. + const type = normalizeMimeType(resolveEffectiveMimeType(artifact.contentType, file.name)) + const visual = explicitVisual || (!textRange && isDefaultVisualMediaType(type)) + if (visual && hasPossibleSecrets(artifact.secretProvenance)) + throw new OrchestrationError('forbidden', MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE) + result = await observePrivateFile( + { + buffer: artifact.buffer, + name: file.name, + contentType: type, + fileId: file.id, + path: fileOwnerVfsPath(file.owner, workspaceFileVfsPath(file)), + source: 'project', + }, + flags, + { ...query.data, visual }, + context.signal + ) + } + const resource = { + type: 'file' as const, + id: file.id, + title: file.name, + owner: { entityType: 'project' as const, entityId: projectId }, + } + return { + ...result, + ...(result.observations + ? { + observations: result.observations.map((observation) => ({ + ...observation, + resource, + })), + } + : {}), + resources: [ + { + op: 'upsert', + readOnly: true, + resource, + }, + ], + } + } catch (error) { + return fileReadFailure(error) + } +} + +/** A rendered image would bypass text redaction, so only provenance known to hold no secret passes. */ +function hasPossibleSecrets(provenance: WorkspaceFileSecretProvenance) { + return ( + provenance.status === 'unknown' || + (provenance.status === 'exact' && provenance.entries.length > 0) + ) +} diff --git a/apps/sim/lib/mothership/agent-cli/project-file-transport.ts b/apps/sim/lib/mothership/agent-cli/project-file-transport.ts new file mode 100644 index 00000000000..52955d37137 --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-file-transport.ts @@ -0,0 +1,74 @@ +import type { Logger } from '@sim/logger' +import { v2ListProjectFilesContract } from '@/lib/api/contracts/v2/project-files' +import type { MatchedRoute } from '@/lib/api/server/routes/in-process-transport' +import { v2OrchestrationErrorPolicy } from '@/lib/api/server/routes/v2-json-route' +import { tryMatchV2Route, v2RoutePattern } from '@/lib/api/server/routes/v2-route-pattern' +import { createScopedCliTransport } from '@/lib/mothership/agent-cli/scoped-transport' +import { markCopilotProjectFileRequest } from '@/lib/mothership/application/execute-project-file-use-case' +import type { CopilotExecutionContext } from '@/lib/mothership/auth/application-delegation' +import type { ProjectFileTarget } from '@/lib/projects/files/application/authorization' +import { v2Error } from '@/app/api/v2/lib/response' + +const PROJECT_FILES_ROUTE = v2RoutePattern(v2ListProjectFilesContract) + +/** Project transport binds owner-specific routes to exact, freshly authorized resource grants. */ +export function createProjectFileCliTransport( + endpoint: string, + context: CopilotExecutionContext, + target: ProjectFileTarget +): typeof fetch { + const bound = Object.freeze({ ...target }) + return createScopedCliTransport(endpoint, { + admitRequest(request, route) { + if ( + (route.pattern !== PROJECT_FILES_ROUTE && + !route.pattern.startsWith(`${PROJECT_FILES_ROUTE}/`)) || + route.params.projectId !== bound.projectId || + (bound.fileId !== undefined && route.params.fileId !== bound.fileId) + ) { + return Response.json({ error: 'CLI target is unavailable' }, { status: 400 }) + } + try { + markCopilotProjectFileRequest(request, context, { + projectId: bound.projectId, + ...(route.params.fileId !== undefined ? { fileId: route.params.fileId } : {}), + }) + } catch { + return Response.json({ error: 'Project file authority is unavailable' }, { status: 403 }) + } + }, + }) +} + +/** The route of a request addressed to this Project on the CLI's own origin, or its refusal. */ +export function matchProjectCliRoute( + request: Request, + origin: string, + projectId: string +): MatchedRoute | Response { + const url = new URL(request.url) + const route = url.origin === origin ? tryMatchV2Route(url.pathname) : null + if (!route || route.params.projectId !== projectId) { + return v2Error('BAD_REQUEST', 'CLI target is unavailable') + } + return route +} + +/** Stops a private Project handler once its invocation or its request is cancelled. */ +export function throwIfCancelled(...signals: (AbortSignal | undefined)[]): void { + for (const signal of signals) signal?.throwIfAborted() +} + +/** Renders a private Project handler failure as its v2 envelope; cancellation is rethrown first. */ +export function renderProjectCliFailure( + error: unknown, + signals: (AbortSignal | undefined)[], + logger: Logger, + failure: { log: string; message: string } +): Response { + throwIfCancelled(...signals) + const response = v2OrchestrationErrorPolicy.render(error) + if (response) return response + logger.error(failure.log, { error }) + return v2Error('INTERNAL_ERROR', failure.message) +} diff --git a/apps/sim/lib/mothership/agent-cli/project-file-upload-transport.ts b/apps/sim/lib/mothership/agent-cli/project-file-upload-transport.ts new file mode 100644 index 00000000000..b17d6221096 --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-file-upload-transport.ts @@ -0,0 +1,149 @@ +import { createLogger } from '@sim/logger' +import { NextRequest } from 'next/server' +import { + v2CompleteProjectFileUploadContract, + v2CreateProjectFileUploadContract, +} from '@/lib/api/contracts/v2/project-file-uploads' +import { V2_PARSE_DEFAULTS, v2InvalidBodyResponse } from '@/lib/api/server/routes/v2-json-route' +import { isV2ContractRoute } from '@/lib/api/server/routes/v2-route-pattern' +import { parseRequest } from '@/lib/api/server/validation' +import { + matchProjectCliRoute, + renderProjectCliFailure, + throwIfCancelled, +} from '@/lib/mothership/agent-cli/project-file-transport' +import { + type ProjectFileProvenanceOptions, + requireCleanProjectFileMetadata, +} from '@/lib/mothership/agent-cli/project-file-write-provenance' +import { executeCopilotProjectFileUseCase } from '@/lib/mothership/application/execute-project-file-use-case' +import { requireTrustedCopilotResourceExecutionContext } from '@/lib/mothership/auth/application-delegation' +import { toV2ProjectFileUpload } from '@/lib/projects/files/api/upload-presenter' +import { + completeProjectFileUploadSession, + createProjectFileUploadSession, +} from '@/lib/projects/files/application' +import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { v2Data, v2Error } from '@/app/api/v2/lib/response' + +const logger = createLogger('ProjectFileUploadTransport') + +/** Private control seals a streamed receipt only after the workbench observer has consumed its actual bytes. */ +export function createProjectFileUploadTransport( + options: ProjectFileProvenanceOptions & { + endpoint: string + projectId: string + fallback: typeof fetch + uploadProvenance?: () => WorkspaceFileSecretProvenance | Promise + uploadBinding?: { + record(uploadId: string): Promise + contains(uploadId: string): Promise + } + } +): typeof fetch { + const { context, projectId, fallback, uploadProvenance } = options + const origin = new URL(options.endpoint).origin + const created = new Set() + return async (input, init) => { + const originalRequest = new Request(input, init) + const route = matchProjectCliRoute(originalRequest, origin, projectId) + if (route instanceof Response) return route + const creating = isV2ContractRoute( + originalRequest.method, + route, + v2CreateProjectFileUploadContract + ) + const completing = isV2ContractRoute( + originalRequest.method, + route, + v2CompleteProjectFileUploadContract + ) + if (!creating && !completing) return fallback(originalRequest) + const request = new NextRequest(originalRequest) + try { + requireTrustedCopilotResourceExecutionContext(context) + } catch { + return v2Error('FORBIDDEN', 'Project file authority is unavailable') + } + try { + throwIfCancelled(context.signal, request.signal) + if (creating) { + const parsed = await parseRequest( + v2CreateProjectFileUploadContract, + request, + { params: route.params }, + { ...V2_PARSE_DEFAULTS, invalidJsonResponse: () => v2InvalidBodyResponse(request) } + ) + if (!parsed.success) return parsed.response + const { body } = parsed.data + await requireCleanProjectFileMetadata(body, options) + throwIfCancelled(context.signal, request.signal) + const session = await executeCopilotProjectFileUseCase( + context, + createProjectFileUploadSession, + { + projectId, + fileName: body.name, + contentType: body.contentType, + fileSize: body.size, + folderId: body.folderId, + folderPath: body.folderPath, + localOrigin: origin, + secretProvenance: 'pending', + }, + { projectId } + ) + created.add(session.id) + await options.uploadBinding?.record(session.id) + return v2Data( + { + session: toV2ProjectFileUpload(session, null), + uploadToken: session.uploadToken, + transfer: session.transfer, + }, + { status: 201 } + ) + } + const parsed = await parseRequest( + v2CompleteProjectFileUploadContract, + request, + { params: route.params }, + V2_PARSE_DEFAULTS + ) + if (!parsed.success) return parsed.response + const bound = options.uploadBinding + ? await options.uploadBinding.contains(parsed.data.params.uploadId) + : created.has(parsed.data.params.uploadId) + if (!bound) { + return v2Error('FORBIDDEN', 'Upload completion is not bound to this invocation') + } + let secretProvenance: WorkspaceFileSecretProvenance | undefined + try { + secretProvenance = await uploadProvenance?.() + } catch { + throwIfCancelled(context.signal, request.signal) + } + if (!secretProvenance) { + return v2Error('SERVICE_UNAVAILABLE', 'Workbench upload source has not finished streaming') + } + throwIfCancelled(context.signal, request.signal) + const result = await executeCopilotProjectFileUseCase( + context, + completeProjectFileUploadSession, + { + projectId, + uploadId: parsed.data.params.uploadId, + uploadToken: parsed.data.headers['upload-token'], + secretProvenance, + }, + { projectId } + ) + return v2Data(toV2ProjectFileUpload(result.session, result.value.file)) + } catch (error) { + return renderProjectCliFailure(error, [context.signal, request.signal], logger, { + log: 'Project upload control failed', + message: 'Project upload control could not be completed', + }) + } + } +} diff --git a/apps/sim/lib/mothership/agent-cli/project-file-write-provenance.ts b/apps/sim/lib/mothership/agent-cli/project-file-write-provenance.ts new file mode 100644 index 00000000000..442132e8001 --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-file-write-provenance.ts @@ -0,0 +1,27 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { AgentCliExecutionContext } from '@/lib/mothership/agent-cli' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +/** The invocation a Project write checks, and the registry holding its source's committed provenance. */ +export interface ProjectFileProvenanceOptions { + context: AgentCliExecutionContext + resolveSecretTraceRegistry?: () => Promise +} + +/** Shared metadata is public even when the file's bytes retain protected provenance. */ +export async function requireCleanProjectFileMetadata( + value: unknown, + options: ProjectFileProvenanceOptions +): Promise { + const { context } = options + context.signal?.throwIfAborted() + const registry = options.resolveSecretTraceRegistry + ? await options.resolveSecretTraceRegistry() + : context.resolvedSecretTraceRegistry + const provenance = registry?.exportCommittedProvenanceForValue(value, { anonymous: true }) + if (!provenance?.complete || provenance.entries.length !== 0) + throw new OrchestrationError( + 'forbidden', + 'Project file metadata or content cannot be published without complete, empty secret provenance' + ) +} diff --git a/apps/sim/lib/mothership/agent-cli/project-file-write-transport.integration.ts b/apps/sim/lib/mothership/agent-cli/project-file-write-transport.integration.ts new file mode 100644 index 00000000000..24fd0a67ac5 --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-file-write-transport.integration.ts @@ -0,0 +1,804 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync } from 'node:fs' +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { db } from '@sim/db' +import { + copilotAsyncToolCalls, + copilotChats, + copilotRuns, + folder, + outboxEvent, + permissions, + uploadSession, + user, + userStats, + workspace, + workspaceFileSecretProvenance, + workspaceFiles, +} from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { sha256Hex } from '@sim/security/hash' +import { createDeferred } from '@sim/testing/helpers/deferred' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { + remoteSandboxProviderMock, + remoteSandboxProviderMockFns, +} from '@sim/testing/mocks/remote-sandbox-provider.mock' +import { setUploadDirServer, uploadsSetupMock } from '@sim/testing/mocks/uploads-setup.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { eq, sql } from 'drizzle-orm' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { redisUrl } = await vi.hoisted(async () => { + const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') + const redisUrl = readTestRedisUrl() + if (redisUrl) process.env.REDIS_URL = redisUrl + return { redisUrl } +}) +vi.mock('@/lib/execution/remote-sandbox/provider', () => remoteSandboxProviderMock) + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) + +vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock) + +import { + v2CompleteProjectFileUploadContract, + v2CreateProjectFileUploadContract, +} from '@/lib/api/contracts/v2/project-file-uploads' +import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' +import { encryptSecret } from '@/lib/core/security/encryption' +import { + initializeSessionFileProvenance, + readSessionSecretProvenance, + recordSessionFileInput, +} from '@/lib/execution/remote-sandbox/session-file-provenance' +import type { AgentCliExecutionContext } from '@/lib/mothership/agent-cli' +import { createProjectFileUploadTransport } from '@/lib/mothership/agent-cli/project-file-upload-transport' +import { createProjectFileWriteTransport } from '@/lib/mothership/agent-cli/project-file-write-transport' +import { executeProjectFileCliRequest } from '@/lib/mothership/agent-cli/project-files' +import { createCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' +import { proxySandboxResourceRequest } from '@/lib/mothership/tools/sandbox-resource-transport' +import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key' +import { writeLocalPutObject } from '@/lib/uploads/upload-session/provider' +import { + uploadSessionObjectMetadata, + verifyUploadSessionToken, +} from '@/lib/uploads/upload-session/service' +import { workspaceFileRevision } from '@/lib/workspace-files/application/file-revision' +import { MAX_WORKSPACE_FILE_INLINE_BODY_BYTES } from '@/lib/workspace-files/orchestration' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +const callbackRedisKeys: string[] = [] +const storageRoot = mkdtempSync(join(tmpdir(), 'sim-project-cli-writes-')) +setUploadDirServer(storageRoot) +const fixtures: { + userId: string + workspaceId: string + projectId: string + originWorkspaceId?: string +}[] = [] +const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = + [] + +beforeEach(() => { + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation(async (flag) => flag === 'projects') + vi.stubEnv('PROJECTS', 'true') + vi.stubEnv('FREE_STORAGE_LIMIT_GB', '') +}) + +function check(name: string, run: () => Promise) { + it(name, async () => { + const started = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} + +async function fixture(crossProject = false) { + const userId = generateId() + const workspaceId = generateId() + await db.insert(user).values({ + id: userId, + email: `${userId}@private-project.invalid`, + name: 'Project writer', + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + }) + await db.insert(userStats).values({ id: generateId(), userId }) + await insertWorkspaceFixture(db, { + id: workspaceId, + ownerId: userId, + billedAccountUserId: userId, + name: 'Authoring environment', + }) + await db.insert(permissions).values({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: 'admin', + }) + const [binding] = await db + .select({ projectId: workspace.projectId }) + .from(workspace) + .where(eq(workspace.id, workspaceId)) + if (!binding?.projectId) throw new Error('Project fixture missing') + const originWorkspaceId = crossProject ? generateId() : workspaceId + if (crossProject) { + await insertWorkspaceFixture(db, { + id: originWorkspaceId, + ownerId: userId, + billedAccountUserId: userId, + name: 'Different origin Project', + }) + await db.insert(permissions).values({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: originWorkspaceId, + permissionType: 'read', + }) + } + const ids = { userId, workspaceId, projectId: binding.projectId, originWorkspaceId } + fixtures.push(ids) + const context: AgentCliExecutionContext = { + userId, + workspaceId: originWorkspaceId, + requestMode: 'agent', + copilotToolExecution: true, + toolCallId: generateId(), + copilotResourceAdmission: createCopilotResourceAdmission({ + userId, + invocation: { kind: 'workspace', workspaceId: originWorkspaceId }, + }), + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry([], { + userId, + workspaceId: originWorkspaceId, + }), + } + return { ...ids, context } +} + +function invoke(projectId: string, context: AgentCliExecutionContext, args: string[]) { + return executeProjectFileCliRequest( + { + fileOwner: { entityType: 'project', entityId: projectId }, + invocation: { kind: 'cli', argv: ['projects', 'files', ...args] }, + }, + context, + projectId + ) +} + +function create( + projectId: string, + context: AgentCliExecutionContext, + content = 'Architecture notes', + encoding = 'utf-8' +) { + return invoke(projectId, context, [ + 'create', + projectId, + '--name', + 'architecture.md', + '--content', + content, + '--encoding', + encoding, + ]) +} + +async function rows(projectId: string) { + return db.select().from(workspaceFiles).where(eq(workspaceFiles.projectId, projectId)) +} + +describe('private native Project writes against PostgreSQL and local storage', () => { + for (const [operation, signalOwner] of [ + ['create', 'context'], + ['update', 'context'], + ['create', 'request'], + ['update', 'request'], + ] as const) { + check( + `${signalOwner} cancellation during provenance lookup prevents ${operation}`, + async () => { + const f = await fixture() + if (operation === 'update') expect((await create(f.projectId, f.context)).exitCode).toBe(0) + const before = await rows(f.projectId) + const controller = new AbortController() + const entered = createDeferred() + const registry = createDeferred() + const endpoint = 'http://localhost:3200' + const transport = createProjectFileWriteTransport({ + endpoint, + projectId: f.projectId, + context: { + ...f.context, + signal: signalOwner === 'context' ? controller.signal : undefined, + }, + resolveSecretTraceRegistry: () => { + entered.resolve() + return registry.promise + }, + fallback: async () => { + throw new Error('Unexpected fallback') + }, + }) + const result = transport( + `${endpoint}/api/v2/projects/${f.projectId}/files${operation === 'update' ? `/${before[0].id}/content` : ''}`, + { + method: operation === 'update' ? 'PUT' : 'POST', + signal: signalOwner === 'request' ? controller.signal : undefined, + headers: { 'content-type': 'application/json' }, + body: JSON.stringify( + operation === 'update' + ? { + content: 'cancelled content', + expectedRevision: workspaceFileRevision(before[0]), + } + : { name: 'cancelled.txt', content: 'cancelled content' } + ), + } + ) + const rejected = expect(result).rejects.toThrow('cancelled during provenance lookup') + await entered.promise + controller.abort(new Error('cancelled during provenance lookup')) + registry.resolve(new ResolvedSecretTraceRegistry([], { userId: f.userId })) + await rejected + expect(await rows(f.projectId)).toEqual(before) + if (operation === 'update') + expect(await readFile(join(storageRoot, before[0].key), 'utf8')).toBe( + 'Architecture notes' + ) + } + ) + } + + check('request bodies and headers cannot mint provenance or Copilot authority', async () => { + const f = await fixture() + const endpoint = 'http://localhost:3200' + const fallback: typeof fetch = async () => { + throw new Error('Write unexpectedly fell through to public transport') + } + const send = (context: AgentCliExecutionContext, extra: Record = {}) => + createProjectFileWriteTransport({ endpoint, projectId: f.projectId, context, fallback })( + `${endpoint}/api/v2/projects/${f.projectId}/files`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Secret-Provenance': 'exact', + 'X-Copilot-User-Id': f.userId, + }, + body: JSON.stringify({ name: 'spoof.md', content: 'plain', ...extra }), + } + ) + expect( + (await send(f.context, { secretProvenance: { status: 'exact', entries: [] } })).status + ).toBe(400) + expect((await send({ ...f.context, resolvedSecretTraceRegistry: undefined })).status).toBe(403) + expect((await send({ ...f.context, copilotResourceAdmission: undefined })).status).toBe(403) + expect(await rows(f.projectId)).toEqual([]) + }) + + check( + 'cross-Project inline writes preserve provenance and revision conflicts do not overwrite bytes', + async () => { + const f = await fixture(true) + const created = await create(f.projectId, f.context) + expect(created.exitCode, created.stderr).toBe(0) + const [file] = await rows(f.projectId) + if (!file) throw new Error('Created file missing') + const [sidecar] = await db + .select() + .from(workspaceFileSecretProvenance) + .where(eq(workspaceFileSecretProvenance.fileId, file.id)) + expect(sidecar).toMatchObject({ status: 'exact', entries: [] }) + expect(file.userId).toBe(f.userId) + expect(await readFile(join(storageRoot, file.key), 'utf8')).toBe('Architecture notes') + const revision = workspaceFileRevision(file) + if (!revision) throw new Error('Created file has no revision') + const args = [ + 'set-content', + f.projectId, + file.id, + '--content', + 'updated', + '--expected-revision', + revision, + ] + expect(await invoke(f.projectId, f.context, args)).toMatchObject({ exitCode: 0 }) + expect( + (await invoke(f.projectId, f.context, [...args.slice(0, 4), 'stale', ...args.slice(5)])) + .exitCode + ).not.toBe(0) + const [updated] = await rows(f.projectId) + if (!updated) throw new Error('Updated file missing') + expect(await readFile(join(storageRoot, updated.key), 'utf8')).toBe('updated') + } + ) + + check( + 'missing or incomplete registries and base64 inputs cannot publish unclassified content', + async () => { + const f = await fixture() + const incomplete = new ResolvedSecretTraceRegistry([], { + userId: f.userId, + workspaceId: f.workspaceId, + }) + incomplete.markIncomplete('entry-decrypt-failed') + for (const registry of [undefined, incomplete]) { + const denied = await create(f.projectId, { + ...f.context, + resolvedSecretTraceRegistry: registry, + }) + expect(denied.exitCode, denied.stderr).not.toBe(0) + expect(denied.stderr).toContain('provenance') + expect(await rows(f.projectId)).toEqual([]) + } + expect((await create(f.projectId, f.context, 'cGxhaW4=', 'base64')).exitCode).not.toBe(0) + expect(await rows(f.projectId)).toEqual([]) + } + ) + + check( + 'known secrets including workspace redaction exemptions cannot be reclassified for a Project', + async () => { + const f = await fixture() + const secret = 'private-project-test-secret-value' + const { encrypted } = await encryptSecret(secret) + for (const unredacted of [false, true]) { + const registry = new ResolvedSecretTraceRegistry( + [ + { + name: 'TOKEN', + plaintext: secret, + encryptedValue: encrypted, + scope: 'workspace', + ...(unredacted ? { unredacted: true as const } : {}), + }, + ], + { userId: f.userId, workspaceId: f.workspaceId } + ) + expect(registry.recordResolved('TOKEN', secret)).toBe(true) + const denied = await create( + f.projectId, + { ...f.context, resolvedSecretTraceRegistry: registry }, + `notes ${secret}` + ) + expect(denied.exitCode, denied.stderr).not.toBe(0) + expect(denied.stderr).toContain('provenance') + expect(await rows(f.projectId)).toEqual([]) + } + } + ) + + for (const upload of [false, true]) { + for (const field of ['name', 'folderPath', 'contentType'] as const) { + check( + `a secret in Project ${upload ? 'upload' : 'create'} ${field} cannot escape as public metadata`, + async () => { + const f = await fixture() + const secret = 'synthetic-metadata-canary' + if (field === 'folderPath') + await db.insert(folder).values({ + id: generateId(), + projectId: f.projectId, + resourceType: 'file', + name: secret, + userId: f.userId, + }) + const registry = new ResolvedSecretTraceRegistry( + [ + { + name: 'TOKEN', + plaintext: secret, + encryptedValue: (await encryptSecret(secret)).encrypted, + scope: 'workspace', + }, + ], + { userId: f.userId, workspaceId: f.workspaceId } + ) + expect(registry.recordResolved('TOKEN', secret)).toBe(true) + const transport = ( + upload ? createProjectFileUploadTransport : createProjectFileWriteTransport + )({ + endpoint: 'http://localhost:3200', + projectId: f.projectId, + context: { ...f.context, resolvedSecretTraceRegistry: registry }, + fallback: async () => { + throw new Error('Unexpected fallback') + }, + }) + const response = await transport( + `http://localhost:3200/api/v2/projects/${f.projectId}/files${upload ? '/uploads' : ''}`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + name: 'safe.txt', + ...(upload ? { size: 9, contentType: 'text/plain' } : { content: 'safe body' }), + [field]: + field === 'folderPath' + ? `/${secret}` + : field === 'contentType' + ? `text/${secret}` + : `${secret}.txt`, + }), + } + ) + expect(response.status, await response.clone().text()).toBe(403) + expect(await rows(f.projectId)).toEqual([]) + expect( + await db.select().from(uploadSession).where(eq(uploadSession.userId, f.userId)) + ).toEqual([]) + } + ) + } + } + + for (const [method, path, body] of [ + ['PATCH', 'files/file-id', (name: string) => ({ name })], + ['POST', 'files/folders', (name: string) => ({ name })], + ['PATCH', 'files/folders/folder-id', (name: string) => ({ name })], + ] as const) { + check(`a secret cannot become a shared name through ${method} ${path}`, async () => { + const f = await fixture() + const secret = 'synthetic-rename-canary' + const registry = new ResolvedSecretTraceRegistry( + [ + { + name: 'TOKEN', + plaintext: secret, + encryptedValue: (await encryptSecret(secret)).encrypted, + scope: 'workspace', + }, + ], + { userId: f.userId, workspaceId: f.workspaceId } + ) + expect(registry.recordResolved('TOKEN', secret)).toBe(true) + const forwarded: string[] = [] + const transport = createProjectFileWriteTransport({ + endpoint: 'http://localhost:3200', + projectId: f.projectId, + context: { ...f.context, resolvedSecretTraceRegistry: registry }, + fallback: async (input, init) => { + forwarded.push(await new Request(input, init).text()) + return Response.json({ data: {} }) + }, + }) + const send = (name: string) => + transport(`http://localhost:3200/api/v2/projects/${f.projectId}/${path}`, { + method, + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body(name)), + }) + const denied = await send(`${secret}.txt`) + expect(denied.status, await denied.clone().text()).toBe(403) + expect(forwarded).toEqual([]) + expect((await send('renamed.txt')).status).toBe(200) + expect(forwarded).toEqual([JSON.stringify(body('renamed.txt'))]) + }) + } + + check('an oversized name write is refused before it is buffered or forwarded', async () => { + const f = await fixture() + const forwarded: string[] = [] + const transport = createProjectFileWriteTransport({ + endpoint: 'http://localhost:3200', + projectId: f.projectId, + context: f.context, + fallback: async (input, init) => { + forwarded.push(new Request(input, init).url) + return Response.json({ data: {} }) + }, + }) + const response = await transport( + `http://localhost:3200/api/v2/projects/${f.projectId}/files/folders`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name: 'x'.repeat(MAX_WORKSPACE_FILE_INLINE_BODY_BYTES + 1) }), + } + ) + expect(response.status).toBe(413) + expect(forwarded).toEqual([]) + }) + + check('a cancellation during the name check never forwards the write', async () => { + const f = await fixture() + const controller = new AbortController() + const forwarded: string[] = [] + const transport = createProjectFileWriteTransport({ + endpoint: 'http://localhost:3200', + projectId: f.projectId, + context: { ...f.context, signal: controller.signal }, + resolveSecretTraceRegistry: async () => { + controller.abort() + return new ResolvedSecretTraceRegistry([], { userId: f.userId, workspaceId: f.workspaceId }) + }, + fallback: async (input, init) => { + forwarded.push(new Request(input, init).url) + return Response.json({ data: {} }) + }, + }) + await expect( + transport(`http://localhost:3200/api/v2/projects/${f.projectId}/files/folders`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name: 'Reports' }), + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(forwarded).toEqual([]) + }) + + check('a private target cannot write a different Project', async () => { + const f = await fixture() + const other = await fixture() + const result = await invoke(f.projectId, f.context, [ + 'create', + other.projectId, + '--name', + 'bad.md', + '--content', + 'plain', + ]) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toContain('CLI target is unavailable') + expect(await rows(other.projectId)).toEqual([]) + }) +}) + +describe.skipIf(!redisUrl)('Project workbench callback across requests', () => { + check( + 'enforces live callback leases, binds uploads across requests and persists machine source history', + async () => { + const f = await fixture() + const chatId = generateId() + await db + .insert(copilotChats) + .values({ id: chatId, userId: f.userId, workspaceId: f.workspaceId, type: 'mothership' }) + const token = generateId() + const otherToken = generateId() + const apiKey = generateId() + const sessionKey = chatSandboxSessionKey(chatId) + const machine = { providerId: 'e2b' as const, sandboxId: generateId() } + remoteSandboxProviderMockFns.mockResolveProvider.mockReturnValue({ + id: machine.providerId, + findSessionSandbox: async () => ({ sandboxId: machine.sandboxId }), + }) + const scope = { + toolCallId: generateId(), + runId: generateId(), + userId: f.userId, + ownerToken: generateId(), + chatId, + workspaceId: f.workspaceId, + apiKeyHash: sha256Hex(apiKey), + fileOwnerProtocolVersion: 1 as const, + } + await db.insert(copilotRuns).values({ + id: scope.runId, + executionId: generateId(), + streamId: generateId(), + chatId, + userId: f.userId, + workspaceId: f.workspaceId, + }) + await db.insert(copilotAsyncToolCalls).values({ + runId: scope.runId, + toolCallId: scope.toolCallId, + toolName: 'run_code', + executionOwnerToken: scope.ownerToken, + executionStartedAt: new Date(), + executionLeaseExpiresAt: new Date(Date.now() + 60_000), + }) + const redis = getRedisClient() + if (!redis) throw new Error('Expected integration Redis') + for (const leaseToken of [token, otherToken]) { + const prefix = `mothership:sandbox-resources:${leaseToken}` + callbackRedisKeys.push(`${prefix}:context`, `${prefix}:inbox`, `${prefix}:seen`) + await redis.set(`${prefix}:context`, JSON.stringify(scope), 'EX', 60) + } + const historyKey = `mothership:workbench-provenance:v2:${createHash('sha256') + .update(JSON.stringify([sessionKey, machine.providerId, machine.sandboxId])) + .digest('hex')}` + callbackRedisKeys.push( + historyKey, + `mothership:sandbox-resources:${token}:project-uploads:${f.projectId}` + ) + await initializeSessionFileProvenance(sessionKey, machine) + const callback = (suffix: string, init?: RequestInit, callbackToken = token) => { + const path = `/api/mothership/sandbox/${callbackToken}/api/v2/projects/${f.projectId}/files${suffix}` + const headers = new Headers(init?.headers) + if (!headers.has('x-api-key')) headers.set('x-api-key', apiKey) + headers.set( + 'x-mothership-file-owner', + JSON.stringify({ entityType: 'project', entityId: f.projectId }) + ) + return proxySandboxResourceRequest( + new Request(`http://localhost:3000${path}`, { ...init, headers }), + callbackToken + ) + } + expect((await callback('', { headers: { 'x-api-key': generateId() } })).status).toBe(403) + const canary = 'synthetic-workbench-callback-canary' + const evidence = { + status: 'exact' as const, + entries: [ + { + encryptedValue: (await encryptSecret(canary)).encrypted, + sourceUserId: f.userId, + sourceWorkspaceId: f.workspaceId, + }, + ], + } + await recordSessionFileInput(sessionKey, machine, evidence) + const rejected = await callback('', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name: 'unsafe.txt', content: canary, encoding: 'utf-8' }), + }) + expect(rejected.status).toBe(403) + const safe = await callback('', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name: 'safe.txt', content: 'safe text', encoding: 'utf-8' }), + }) + expect(safe.status, await safe.clone().text()).toBe(201) + await db.insert(folder).values({ + id: generateId(), + projectId: f.projectId, + resourceType: 'file', + name: canary, + userId: f.userId, + }) + for (const metadata of [ + { name: `${canary}.txt` }, + { contentType: `text/${canary}` }, + { folderPath: `/${canary}` }, + ]) { + const denied = await callback('/uploads', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + name: 'safe.txt', + contentType: 'text/plain', + size: 9, + ...metadata, + }), + }) + expect(denied.status, await denied.clone().text()).toBe(403) + expect( + await db.select().from(uploadSession).where(eq(uploadSession.userId, f.userId)) + ).toEqual([]) + } + const bytes = Buffer.from('safe upload') + const createdResponse = await callback('/uploads', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name: 'upload.txt', contentType: 'text/plain', size: bytes.length }), + }) + expect(createdResponse.status, await createdResponse.clone().text()).toBe(201) + const created = v2CreateProjectFileUploadContract.response.schema.parse( + await createdResponse.json() + ).data + const upload = await verifyUploadSessionToken(created.uploadToken) + await writeLocalPutObject({ + uploadId: upload.id, + key: upload.finalKey, + body: new ReadableStream({ + start(controller) { + controller.enqueue(bytes) + controller.close() + }, + }), + expectedSize: bytes.length, + contentType: upload.contentType, + metadata: uploadSessionObjectMetadata(upload), + }) + const complete = { method: 'POST', headers: { 'upload-token': created.uploadToken } } + expect((await callback(`/uploads/${upload.id}/complete`, complete, otherToken)).status).toBe( + 403 + ) + const beforeUnattributedUpload = await rows(f.projectId) + await recordSessionFileInput(sessionKey, machine, { + status: 'exact', + entries: [ + { encryptedValue: (await encryptSecret('unattributed synthetic value')).encrypted }, + ], + }) + expect((await callback(`/uploads/${upload.id}/complete`, complete)).status).toBe(503) + expect(await rows(f.projectId)).toEqual(beforeUnattributedUpload) + await redis.del(historyKey) + await initializeSessionFileProvenance(sessionKey, machine) + await recordSessionFileInput(sessionKey, machine, evidence) + const response = await callback(`/uploads/${upload.id}/complete`, complete) + expect(response.status, await response.clone().text()).toBe(200) + const completed = v2CompleteProjectFileUploadContract.response.schema.parse( + await response.json() + ).data + if (!completed.file) throw new Error('Expected completed upload') + const [stored] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.id, completed.file.id)) + const [provenance] = await db + .select() + .from(workspaceFileSecretProvenance) + .where(eq(workspaceFileSecretProvenance.fileId, completed.file.id)) + expect(stored).toMatchObject({ projectId: f.projectId, workspaceId: null, userId: f.userId }) + expect(provenance).toMatchObject(evidence) + const freshMachine = { ...machine, sandboxId: generateId() } + remoteSandboxProviderMockFns.mockResolveProvider.mockReturnValue({ + id: freshMachine.providerId, + findSessionSandbox: async () => ({ sandboxId: freshMachine.sandboxId }), + }) + callbackRedisKeys.push( + `mothership:workbench-provenance:v2:${createHash('sha256') + .update(JSON.stringify([sessionKey, freshMachine.providerId, freshMachine.sandboxId])) + .digest('hex')}` + ) + await initializeSessionFileProvenance(sessionKey, freshMachine) + expect(await readSessionSecretProvenance(sessionKey, freshMachine)).toEqual({ + status: 'exact', + entries: [], + }) + const read = await callback(`/${completed.file.id}/content`, undefined, otherToken) + expect(read.status).toBe(200) + expect(await read.text()).toBe('safe upload') + expect(await readSessionSecretProvenance(sessionKey, freshMachine)).toEqual(evidence) + const effects = ( + await redis.lrange(`mothership:sandbox-resources:${token}:inbox`, 0, -1) + ).map((entry) => JSON.parse(entry)) + expect(effects.length).toBeGreaterThan(0) + for (const effect of effects) { + expect(effect.resource.owner).toEqual({ entityType: 'project', entityId: f.projectId }) + expect(effect.resource.workspaceId).toBeUndefined() + } + await db + .update(copilotAsyncToolCalls) + .set({ executionRevokedAt: new Date() }) + .where(eq(copilotAsyncToolCalls.toolCallId, scope.toolCallId)) + expect((await callback(`/${completed.file.id}/content`)).status).toBe(403) + } + ) +}) + +afterAll(async () => { + if (callbackRedisKeys.length) await getRedisClient()?.del(...callbackRedisKeys) + for (const f of fixtures) { + await db.delete(uploadSession).where(eq(uploadSession.userId, f.userId)) + await db + .delete(outboxEvent) + .where(sql`${outboxEvent.payload}::jsonb ->> 'key' LIKE ${`project/${f.projectId}/%`}`) + await db.delete(workspaceFiles).where(eq(workspaceFiles.projectId, f.projectId)) + await db.delete(folder).where(eq(folder.projectId, f.projectId)) + await deleteWorkspaceFixture(db, eq(workspace.id, f.workspaceId)) + if (f.originWorkspaceId && f.originWorkspaceId !== f.workspaceId) + await deleteWorkspaceFixture(db, eq(workspace.id, f.originWorkspaceId)) + await db.delete(user).where(eq(user.id, f.userId)) + } + await rm(storageRoot, { recursive: true, force: true }) + const report = + process.env.PROJECT_FILE_WRITE_TRANSPORT_REPORT_PATH ?? + 'test-results/project-file-write-transport.json' + await mkdir(dirname(report), { recursive: true }) + await writeFile(report, JSON.stringify({ checks }, null, 2)) + await closeRedisConnection() + await db.$client.end() +}) diff --git a/apps/sim/lib/mothership/agent-cli/project-file-write-transport.ts b/apps/sim/lib/mothership/agent-cli/project-file-write-transport.ts new file mode 100644 index 00000000000..3e66a34817c --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-file-write-transport.ts @@ -0,0 +1,193 @@ +import { createLogger } from '@sim/logger' +import { NextRequest } from 'next/server' +import { + v2CreateProjectFileFolderContract, + v2UpdateProjectFileFolderContract, +} from '@/lib/api/contracts/v2/project-file-folders' +import { v2RenameProjectFileContract } from '@/lib/api/contracts/v2/project-file-lifecycle' +import { + v2CreateProjectFileContract, + v2UpdateProjectFileContentContract, +} from '@/lib/api/contracts/v2/project-files' +import type { MatchedRoute } from '@/lib/api/server/routes/in-process-transport' +import { + V2_PARSE_DEFAULTS, + v2InvalidBodyResponse, + v2OrchestrationErrorPolicy, +} from '@/lib/api/server/routes/v2-json-route' +import { isV2ContractRoute } from '@/lib/api/server/routes/v2-route-pattern' +import { parseRequest } from '@/lib/api/server/validation' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + matchProjectCliRoute, + renderProjectCliFailure, + throwIfCancelled, +} from '@/lib/mothership/agent-cli/project-file-transport' +import { + type ProjectFileProvenanceOptions, + requireCleanProjectFileMetadata, +} from '@/lib/mothership/agent-cli/project-file-write-provenance' +import { executeCopilotProjectFileUseCase } from '@/lib/mothership/application/execute-project-file-use-case' +import { requireTrustedCopilotResourceExecutionContext } from '@/lib/mothership/auth/application-delegation' +import { toCreateProjectFileInput } from '@/lib/projects/files/api/create-input' +import { toV2ProjectFile } from '@/lib/projects/files/api/presenters' +import { createProjectFile, updateProjectFileContent } from '@/lib/projects/files/application' +import { EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE } from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { MAX_WORKSPACE_FILE_INLINE_BODY_BYTES } from '@/lib/workspace-files/orchestration' +import { v2Data, v2Error } from '@/app/api/v2/lib/response' + +const logger = createLogger('ProjectFileWriteTransport') + +/** Forwarded writes that publish a caller-chosen name, so their metadata is checked first. */ +const NAME_WRITE_CONTRACTS = [ + v2RenameProjectFileContract, + v2CreateProjectFileFolderContract, + v2UpdateProjectFileFolderContract, +] as const + +/** + * Parses a write that publishes a caller-chosen name under its route's size limit, or returns + * undefined for forwarded writes that reference existing items only. + */ +function parseNameWrite(original: Request, route: MatchedRoute) { + const contract = NAME_WRITE_CONTRACTS.find((candidate) => + isV2ContractRoute(original.method, route, candidate) + ) + if (!contract) return undefined + /** Only a name write is copied, so forwarded upload bodies are never teed. */ + const request = new NextRequest(original.clone()) + return parseRequest( + contract, + request, + { params: route.params }, + { + ...V2_PARSE_DEFAULTS, + maxBodyBytes: MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, + invalidJsonResponse: () => v2InvalidBodyResponse(request), + } + ) +} + +/** Private inline writes carry only a proven empty classification; original source identities are never reconstructed. */ +export function createProjectFileWriteTransport( + options: ProjectFileProvenanceOptions & { + endpoint: string + projectId: string + fallback: typeof fetch + } +): typeof fetch { + const origin = new URL(options.endpoint).origin + const projectId = options.projectId + return async (input, init) => { + const originalRequest = new Request(input, init) + const matched = matchProjectCliRoute(originalRequest, origin, projectId) + if (matched instanceof Response) return matched + const create = isV2ContractRoute(originalRequest.method, matched, v2CreateProjectFileContract) + const update = isV2ContractRoute( + originalRequest.method, + matched, + v2UpdateProjectFileContentContract + ) + if (!create && !update) { + const nameWrite = parseNameWrite(originalRequest, matched) + if (!nameWrite) return options.fallback(originalRequest) + try { + const parsed = await nameWrite + if (!parsed.success) return parsed.response + await requireCleanProjectFileMetadata(parsed.data.body, options) + throwIfCancelled(options.context.signal, originalRequest.signal) + } catch (error) { + throwIfCancelled(options.context.signal, originalRequest.signal) + const response = v2OrchestrationErrorPolicy.render(error) + if (response) return response + throw error + } + return options.fallback(originalRequest) + } + const request = new NextRequest(originalRequest) + try { + requireTrustedCopilotResourceExecutionContext(options.context) + } catch { + return v2Error('FORBIDDEN', 'Project file authority is unavailable') + } + const parseOptions = { + ...V2_PARSE_DEFAULTS, + maxBodyBytes: MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, + invalidJsonResponse: () => v2InvalidBodyResponse(request), + } + try { + throwIfCancelled(options.context.signal, request.signal) + if (create) { + const parsed = await parseRequest( + v2CreateProjectFileContract, + request, + { params: matched.params }, + parseOptions + ) + if (!parsed.success) return parsed.response + const { params, body } = parsed.data + await requireCleanInlineText(body, options) + throwIfCancelled(options.context.signal, request.signal) + const result = await executeCopilotProjectFileUseCase( + options.context, + createProjectFile, + { + ...toCreateProjectFileInput(params.projectId, body), + secretProvenance: EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, + }, + { projectId } + ) + const response = v2CreateProjectFileContract.response.schema.parse({ + data: toV2ProjectFile(result.file), + }) + return v2Data(response.data, { status: v2CreateProjectFileContract.response.status }) + } + const parsed = await parseRequest( + v2UpdateProjectFileContentContract, + request, + { params: matched.params }, + parseOptions + ) + if (!parsed.success) return parsed.response + const { params, body } = parsed.data + await requireCleanInlineText(body, options) + throwIfCancelled(options.context.signal, request.signal) + const target = { projectId, fileId: params.fileId } + const result = await executeCopilotProjectFileUseCase( + options.context, + updateProjectFileContent, + { + ...target, + content: body.content, + encoding: body.encoding, + expectedRevision: body.expectedRevision, + secretProvenance: EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, + }, + target + ) + const response = v2UpdateProjectFileContentContract.response.schema.parse({ + data: toV2ProjectFile(result.file), + }) + return v2Data(response.data) + } catch (error) { + return renderProjectCliFailure(error, [options.context.signal, request.signal], logger, { + log: 'Project file write failed', + message: 'Project file write could not be completed', + }) + } + } +} + +async function requireCleanInlineText( + body: { content: string; encoding: 'utf-8' | 'base64' }, + options: ProjectFileProvenanceOptions +): Promise { + const { context } = options + context.signal?.throwIfAborted() + if (body.encoding !== 'utf-8') + throw new OrchestrationError( + 'forbidden', + 'Project CLI writes currently require UTF-8 text with verified secret provenance' + ) + await requireCleanProjectFileMetadata(body, options) +} diff --git a/apps/sim/lib/mothership/agent-cli/project-files.ts b/apps/sim/lib/mothership/agent-cli/project-files.ts new file mode 100644 index 00000000000..b7938bef735 --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/project-files.ts @@ -0,0 +1,98 @@ +import type { EmbeddedCliIdentity } from 'sim/embed' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { AgentCliExecutionContext } from '@/lib/mothership/agent-cli' +import { executePreparedCliRequest, prepareCliSession } from '@/lib/mothership/agent-cli/execute' +import { grepProjectFiles } from '@/lib/mothership/agent-cli/project-file-grep' +import { importProjectFileProvenance } from '@/lib/mothership/agent-cli/project-file-provenance' +import { readProjectFileForAgent } from '@/lib/mothership/agent-cli/project-file-read' +import { createProjectFileCliTransport } from '@/lib/mothership/agent-cli/project-file-transport' +import { createProjectFileUploadTransport } from '@/lib/mothership/agent-cli/project-file-upload-transport' +import { createProjectFileWriteTransport } from '@/lib/mothership/agent-cli/project-file-write-transport' +import { readOnlyCliTransport } from '@/lib/mothership/agent-cli/read-only' +import { createResourceEffectTransport } from '@/lib/mothership/agent-cli/resource-effects' +import { createTracedCliTransport } from '@/lib/mothership/agent-cli/traced-transport' +import { executeCopilotProjectFileUseCase } from '@/lib/mothership/application/execute-project-file-use-case' +import type { AgentCliRawResult, AgentCliRequest } from '@/lib/mothership/generated/agent-cli' +import { getProjectFileCapabilities } from '@/lib/projects/files/application' +import { + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { observeWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' +import { v2Error } from '@/app/api/v2/lib/response' + +/** Executes the native command tree with Project-only transport and no ambient workspace target. */ +export async function executeProjectFileCliRequest( + request: AgentCliRequest, + context: AgentCliExecutionContext, + projectId: string +): Promise { + if (request.curate) + throw new OrchestrationError( + 'validation', + 'Project file commands do not support this projection' + ) + const target = { projectId } + await executeCopilotProjectFileUseCase(context, getProjectFileCapabilities, target, target) + const { endpoint, sessionKey, files, resources } = prepareCliSession(context) + const scoped = createProjectFileUploadTransport({ + endpoint, + projectId, + context, + uploadProvenance: files?.uploadProvenance, + fallback: createProjectFileWriteTransport({ + endpoint, + projectId, + context, + fallback: createProjectFileCliTransport(endpoint, context, target), + }), + }) + const observed: typeof fetch = async (input, init) => { + let provenance: WorkspaceFileSecretProvenance | undefined + const response = await observeWorkspaceFileDelivery( + async (source) => { + provenance = source + await importProjectFileProvenance(context.resolvedSecretTraceRegistry, source) + }, + () => scoped(input, init) + ) + if (response.ok && response.headers.has('Content-Disposition') && !provenance) { + await response.body?.cancel() + return v2Error('SERVICE_UNAVAILABLE', 'File read provenance is unavailable. Retry the read.') + } + /** Raw bytes reach the model unredacted unless a workbench tracker keeps them private. */ + if ( + !files && + response.headers.has('Content-Disposition') && + provenance?.status === 'exact' && + provenance.entries.length > 0 + ) { + await response.body?.cancel() + return v2Error('FORBIDDEN', MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE) + } + if (response.body && provenance) files?.trackDownload(response.body, provenance) + return response + } + const transport = createResourceEffectTransport( + endpoint, + createTracedCliTransport(endpoint, observed), + resources + ) + const identity: EmbeddedCliIdentity = { + endpoint, + apiKey: 'mothership-in-process', + /** Mutations are refused before the Project write and upload interceptors, as for workspaces. */ + transport: context.readOnly ? readOnlyCliTransport(transport) : transport, + ...(context.signal ? { signal: context.signal } : {}), + } + return executePreparedCliRequest(request, { + identity, + sessionKey, + files, + resources, + augment: (invocation) => + invocation.name === 'grep' + ? grepProjectFiles(invocation, context, projectId) + : readProjectFileForAgent(invocation, context, projectId), + }) +} diff --git a/apps/sim/lib/mothership/agent-cli/resource-effects.ts b/apps/sim/lib/mothership/agent-cli/resource-effects.ts index 4d76da4ecd4..648c2257189 100644 --- a/apps/sim/lib/mothership/agent-cli/resource-effects.ts +++ b/apps/sim/lib/mothership/agent-cli/resource-effects.ts @@ -3,6 +3,7 @@ import type { z } from 'zod' import type { AnyApiRouteContract } from '@/lib/api/contracts/types' import * as credentials from '@/lib/api/contracts/v2/credentials' import * as customTools from '@/lib/api/contracts/v2/custom-tools' +import { v2CopyFileItemsContract } from '@/lib/api/contracts/v2/file-copy' import * as files from '@/lib/api/contracts/v2/files' import * as knowledge from '@/lib/api/contracts/v2/knowledge' import * as chunks from '@/lib/api/contracts/v2/knowledge-chunks' @@ -10,15 +11,20 @@ import * as tags from '@/lib/api/contracts/v2/knowledge-tags' import * as logs from '@/lib/api/contracts/v2/logs' import * as logStats from '@/lib/api/contracts/v2/logs-stats' import * as mcpServers from '@/lib/api/contracts/v2/mcp-servers' +import { v2UnzipProjectFileContract } from '@/lib/api/contracts/v2/project-file-extraction' +import { v2CompleteProjectFileUploadContract } from '@/lib/api/contracts/v2/project-file-uploads' +import * as projectFiles from '@/lib/api/contracts/v2/project-files' import * as sandboxes from '@/lib/api/contracts/v2/sandboxes' import * as secrets from '@/lib/api/contracts/v2/secrets' import * as tables from '@/lib/api/contracts/v2/tables' import * as workflows from '@/lib/api/contracts/v2/workflows' -import { parseFolderPath } from '@/lib/folders/paths' +import { buildFolderPath, parseFolderPath } from '@/lib/folders/paths' import { isCommittedWriteResponse } from '@/lib/mothership/content-delivery' import type { ResourceAddress, ResourceChange } from '@/lib/mothership/generated/resources' import { findNonTabFileIds } from '@/lib/mothership/resources/file-tabs' import { encodeVfsPathSegments } from '@/lib/vfs/path' +import { parseWorkspaceFileFolderDisplayPath } from '@/lib/workspace-files/folder-display-path' +import { fileOwnerVfsPath } from '@/lib/workspace-files/owner-paths' const logger = createLogger('ResourceEffects') @@ -43,9 +49,22 @@ function refresh(type: ResourceKind, id?: string): ResourceChange[] { return [{ op: 'refresh', resource: { type, ...(id ? { id } : {}) } }] } +function ownedFilePath(owner: ResourceAddress['owner'], folderPath: string, name: string): string { + const path = `files/${encodeVfsPathSegments([...parseFolderPath(folderPath), name])}` + return owner ? fileOwnerVfsPath(owner, path) : path +} + +/** Refreshes an owner's file and folder listings after a change too broad to open as one tab. */ +function refreshOwnerFiles(owner: NonNullable): ResourceChange[] { + return [ + { op: 'refresh', resource: { type: 'file', owner } }, + { op: 'refresh', resource: { type: 'filefolder', owner } }, + ] +} + function upsert( type: ResourceKind, - value: { id: string; name?: string; folderPath?: string }, + value: { id: string; name?: string; folderPath?: string; owner?: ResourceAddress['owner'] }, readOnly = false ): ResourceChange[] { return [ @@ -55,11 +74,10 @@ function upsert( resource: { type, id: value.id, + ...(value.owner ? { owner: value.owner } : {}), ...(value.name ? { title: value.name } : {}), ...(type === 'file' && value.name && value.folderPath !== undefined - ? { - path: `files/${encodeVfsPathSegments([...parseFolderPath(value.folderPath), value.name])}`, - } + ? { path: ownedFilePath(value.owner, value.folderPath, value.name) } : {}), }, }, @@ -152,6 +170,33 @@ function settingsMutations( } const EFFECT_ROUTES: EffectRoute[] = [ + after(v2UnzipProjectFileContract, (_result, { params }) => + refreshOwnerFiles({ entityType: 'project', entityId: params.projectId }) + ), + after(v2CompleteProjectFileUploadContract, ({ data }) => + data.file ? upsert('file', data.file) : [] + ), + /** A single copied file opens as a tab; a bulk or folder copy refreshes the destination owner. */ + after(v2CopyFileItemsContract, ({ data }) => { + const [file] = data.files + if (file && data.files.length === 1 && data.folders.length === 0) + return upsert('file', { + ...file, + folderPath: buildFolderPath( + file.folderPath ? parseWorkspaceFileFolderDisplayPath(file.folderPath) : [] + ), + }) + const owner = file?.owner ?? data.folders[0]?.owner + return owner ? refreshOwnerFiles(owner) : [] + }), + { + ...after(projectFiles.v2GetProjectFileMetadataContract, ({ data }) => + upsert('file', data, true) + ), + readOnly: true, + }, + after(projectFiles.v2CreateProjectFileContract, ({ data }) => upsert('file', data)), + after(projectFiles.v2UpdateProjectFileContentContract, ({ data }) => upsert('file', data)), ...settingsMutations('secrets', [secrets.v2SetSecretContract, secrets.v2DeleteSecretContract]), ...settingsMutations('credentials', [ credentials.v2CreateCredentialConnectionContract, @@ -348,7 +393,7 @@ function matchParams(pattern: string, pathname: string): Record } /** - * Only workspace files and chat uploads open as file tabs; other rows open as their owner. The + * Workspace/Project files and chat uploads open as file tabs; other rows open as their owner. The * request already committed, so a failed lookup opens no file tab rather than failing it. */ async function onlyFileTabs(changes: ResourceChange[]): Promise { diff --git a/apps/sim/lib/mothership/agent-cli/scope-routing.test.ts b/apps/sim/lib/mothership/agent-cli/scope-routing.test.ts index 8e7f47aa128..9a54be32cfe 100644 --- a/apps/sim/lib/mothership/agent-cli/scope-routing.test.ts +++ b/apps/sim/lib/mothership/agent-cli/scope-routing.test.ts @@ -149,4 +149,26 @@ describe('worker-local CLI authorization through the real handler and scope reso expect(boundary.target).not.toHaveBeenCalled() expect(boundary.route).not.toHaveBeenCalled() }) + + it.each([ + { entityType: 'workspace' as const, entityId: selected }, + { entityType: 'project' as const, entityId: 'project' }, + ])( + 'rejects a file owner on a service before any owner adapter runs: $entityType', + async (owner) => { + expect( + await executeSimCli( + { + request: { + fileOwner: owner, + invocation: { kind: 'service', name: 'dashboards', input: {} }, + }, + }, + workspace + ) + ).toEqual({ success: false, error: 'Service invocations do not take a file owner' }) + expect(boundary.target).not.toHaveBeenCalled() + expect(boundary.route).not.toHaveBeenCalled() + } + ) }) diff --git a/apps/sim/lib/mothership/agent-cli/scoped-transport.ts b/apps/sim/lib/mothership/agent-cli/scoped-transport.ts index 70af6276439..2ed54c2a981 100644 --- a/apps/sim/lib/mothership/agent-cli/scoped-transport.ts +++ b/apps/sim/lib/mothership/agent-cli/scoped-transport.ts @@ -1,21 +1,40 @@ import { NextRequest } from 'next/server' import { markCopilotRequest } from '@/lib/api/server/routes/copilot-request' -import { dispatchInProcessV2Request } from '@/lib/api/server/routes/in-process-transport' +import { + dispatchInProcessV2Request, + type MatchedRoute, +} from '@/lib/api/server/routes/in-process-transport' +import { tryMatchV2Route } from '@/lib/api/server/routes/v2-route-pattern' import type { CopilotChatDelegationContext } from '@/lib/mothership/auth/application-delegation' +interface ScopedCliInvocation { + admitRequest( + request: NextRequest, + route: MatchedRoute + ): Response | undefined | Promise +} + /** Private admission uses only this validated in-process identity; public requests remain API-key authenticated. */ export function createScopedCliTransport( endpoint: string, - invocation: CopilotChatDelegationContext + invocation: CopilotChatDelegationContext | ScopedCliInvocation ): typeof fetch { const origin = new URL(endpoint).origin + const scope = Object.freeze({ ...invocation }) + const unavailable = () => Response.json({ error: 'CLI target is unavailable' }, { status: 400 }) return async (input, init) => { const request = new Request(input, init) const url = new URL(request.url) - if (url.origin !== origin || !url.pathname.startsWith('/api/v2/')) - return Response.json({ error: 'CLI target is unavailable' }, { status: 400 }) + if (url.origin !== origin || !url.pathname.startsWith('/api/v2/')) return unavailable() const forwarded = new NextRequest(request) - markCopilotRequest(forwarded, invocation) + if ('admitRequest' in scope) { + const matched = tryMatchV2Route(url.pathname) + if (!matched) return unavailable() + const denied = await scope.admitRequest(forwarded, matched) + if (denied) return denied + } else { + markCopilotRequest(forwarded, scope) + } return ( (await dispatchInProcessV2Request(forwarded)) ?? Response.json({ error: 'API route or method unavailable' }, { status: 404 }) diff --git a/apps/sim/lib/mothership/agent-cli/services.ts b/apps/sim/lib/mothership/agent-cli/services.ts index ec55375c37d..60f2210fd01 100644 --- a/apps/sim/lib/mothership/agent-cli/services.ts +++ b/apps/sim/lib/mothership/agent-cli/services.ts @@ -36,15 +36,19 @@ export async function executeAgentCliService( const invocation = request.invocation if (invocation.kind !== 'service' && invocation.kind !== 'stdout') throw new OrchestrationError('validation', 'Invalid service invocation') + if (request.fileOwner) + throw new OrchestrationError('validation', 'Service invocations do not take a file owner') const scope = - invocation.kind === 'service' && invocation.name === 'settings' - ? invocation.input.scope - : invocation.kind === 'service' && - (invocation.name === 'dashboards' || - invocation.name === 'tests' || - invocation.name === 'changelog') - ? 'workspace' - : 'organization' + invocation.kind === 'service' && invocation.name === 'list_user_projects' + ? 'account' + : invocation.kind === 'service' && invocation.name === 'settings' + ? invocation.input.scope + : invocation.kind === 'service' && + (invocation.name === 'dashboards' || + invocation.name === 'tests' || + invocation.name === 'changelog') + ? 'workspace' + : 'organization' if ( invocation.kind === 'service' && invocation.name !== 'list_workspaces' && diff --git a/apps/sim/lib/mothership/agent-cli/workspace.ts b/apps/sim/lib/mothership/agent-cli/workspace.ts new file mode 100644 index 00000000000..e69b7f58f9e --- /dev/null +++ b/apps/sim/lib/mothership/agent-cli/workspace.ts @@ -0,0 +1,150 @@ +import { createEmbeddedClient, type EmbeddedCliIdentity } from 'sim/embed' +import { withWorkspaceInvocationScope } from '@/lib/core/application/workspace-invocation-scope' +import type { AgentCliExecutionContext } from '@/lib/mothership/agent-cli' +import { curateBlockDetail } from '@/lib/mothership/agent-cli/curation' +import { AUGMENTATION_ENGINES, runEngine } from '@/lib/mothership/agent-cli/engines' +import { executePreparedCliRequest, prepareCliSession } from '@/lib/mothership/agent-cli/execute' +import { curateKnowledgeDocuments } from '@/lib/mothership/agent-cli/knowledge-curation' +import { readOnlyCliTransport } from '@/lib/mothership/agent-cli/read-only' +import { createAgentCliTransport } from '@/lib/mothership/agent-cli/transport' +import { resolveInvocationWorkspace } from '@/lib/mothership/application/workspace-target' +import { + COPILOT_APPLICATION_DELEGATION_TTL_MS, + createCopilotChatPrincipal, + createTrustedOrganizationCopilotPrincipal, +} from '@/lib/mothership/auth/application-delegation' +import type { AgentCliRawResult, AgentCliRequest } from '@/lib/mothership/generated/agent-cli' +import type { ResourceChange } from '@/lib/mothership/generated/resources' +import { TraceSpan } from '@/lib/mothership/generated/trace-spans-v1' +import { withCopilotSpan } from '@/lib/mothership/request/otel' +import { isFileResourceType } from '@/lib/mothership/resources/types' +import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' + +/** Resolves workspace policy before preparing the existing workspace CLI transport. */ +export async function executeWorkspaceCliRequest( + request: AgentCliRequest, + context: AgentCliExecutionContext, + workspaceId = request.workspaceId +): Promise { + const target = await resolveInvocationWorkspace(context, workspaceId) + return withWorkspaceInvocationScope( + { + workspaceId: target.workspaceId, + organizationId: context.chatOrganizationId ?? context.organizationId, + }, + () => + executeBoundAgentCliRequest(request, { + ...context, + ...target, + chatOrganizationId: context.chatOrganizationId ?? context.organizationId, + }) + ) +} + +async function executeBoundAgentCliRequest( + request: AgentCliRequest, + context: AgentCliExecutionContext & { workspaceId: string } +): Promise { + /** The embedded client's required credential is opaque and never valid on the public API. */ + const apiKey = 'mothership-in-process' + const invocationIdentity = { + userId: context.userId, + workspaceId: context.workspaceId, + chatId: context.chatId, + } + const { endpoint, sessionKey, files, resources } = prepareCliSession(context) + const identity = { + endpoint, + apiKey, + workspaceId: context.workspaceId, + transport: createAgentCliTransport({ + endpoint, + invocation: invocationIdentity, + registry: context.resolvedSecretTraceRegistry, + ...(files ? { files } : {}), + resources, + observeReads: + request.invocation.kind === 'cli' || + (request.invocation.kind === 'augmentation' && + AUGMENTATION_ENGINES[request.invocation.name]?.openReadResources === true), + }), + ...(context.signal ? { signal: context.signal } : {}), + } satisfies EmbeddedCliIdentity + + if (context.readOnly) identity.transport = readOnlyCliTransport(identity.transport) + + return executePreparedCliRequest(request, { + identity, + sessionKey, + files, + resources, + augment: (invocation) => + runEngine( + invocation.name, + invocation.positionals, + { + client: createEmbeddedClient(identity), + workspaceId: context.workspaceId, + userId: context.userId, + principal: createCopilotChatPrincipal( + invocationIdentity, + WORKSPACE_FILES_DELEGATION_AUDIENCE + ), + invocation: invocationIdentity, + ...(context.chatOrganizationId && context.chatId + ? { + chatOrganizationId: context.chatOrganizationId, + chatPrincipal: createTrustedOrganizationCopilotPrincipal( + { + userId: context.userId, + organizationId: context.chatOrganizationId, + chatId: context.chatId, + delegationId: `scratch:${context.chatId}`, + }, + { + audience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + ttlMs: COPILOT_APPLICATION_DELEGATION_TTL_MS, + } + ), + } + : {}), + ...(context.chatId !== undefined ? { chatId: context.chatId } : {}), + resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, + signal: context.signal, + }, + invocation.flags + ), + async curate(result) { + if (request.curate === 'knowledge-documents') return curateKnowledgeDocuments(result) + if (result.exitCode === 0 && request.curate === 'block') { + return withCopilotSpan(TraceSpan.CopilotCliCurate, undefined, () => + curateBlockDetail(result, context) + ) + } + return result + }, + projectResources: (changes) => { + function scopeResource(resource: R): R { + return { + ...resource, + ...(request.fileOwner && isFileResourceType(resource.type) + ? { owner: request.fileOwner } + : {}), + ...(context.chatOrganizationId ? { workspaceId: context.workspaceId } : {}), + } + } + return changes.map((effect) => { + switch (effect.op) { + case 'upsert': + return { ...effect, resource: scopeResource(effect.resource) } + case 'remove': + return { ...effect, resource: scopeResource(effect.resource) } + case 'refresh': + return { ...effect, resource: scopeResource(effect.resource) } + case 'clear_view': + return { ...effect, resource: scopeResource(effect.resource) } + } + }) + }, + }) +} diff --git a/apps/sim/lib/mothership/application/adapter.ts b/apps/sim/lib/mothership/application/adapter.ts index e582517dc32..15e0d683336 100644 --- a/apps/sim/lib/mothership/application/adapter.ts +++ b/apps/sim/lib/mothership/application/adapter.ts @@ -1,5 +1,11 @@ -import type { DelegatedPrincipal } from '@sim/auth/principal' +import type { + DelegatedPrincipal, + Principal, + ResourceDelegatedPrincipal, + ResourceDelegationScope, +} from '@sim/auth/principal' import { + type ApplicationOperation, type OperationUseCase, requireAllowedWorkspacePrincipal, type WorkspaceOperation, @@ -7,10 +13,14 @@ import { import { type CopilotDelegationConfiguration, type CopilotExecutionContext, + type CopilotResourceDelegationConfiguration, type CopilotResourceScope, createCopilotApplicationPrincipal, + createCopilotResourceApplicationPrincipal, requireTrustedCopilotExecutionContext, + requireTrustedCopilotResourceExecutionContext, type TrustedCopilotExecutionContext, + type TrustedCopilotResourceExecutionContext, } from '@/lib/mothership/auth/application-delegation' type CopilotApplicationPrincipalFactory = (args: { @@ -19,6 +29,7 @@ type CopilotApplicationPrincipalFactory = (args: { }) => DelegatedPrincipal interface CopilotApplicationAdapterOptions { + mode?: 'workspace' domain: string delegation: CopilotDelegationConfiguration operations: Readonly> @@ -29,8 +40,44 @@ interface CopilotApplicationAdapterOptions = { + mode: 'resource' + domain: string + delegation: CopilotResourceDelegationConfiguration + operations: Readonly> +} & ( + | { + resourceScope: ResourceDelegationScope + projectResourceScope?: never + } + | { + resourceScope?: never + projectResourceScope( + input: ScopeInput, + context: TrustedCopilotResourceExecutionContext + ): ResourceDelegationScope + } +) + type ScopeArguments = [ScopeInput] extends [undefined] ? [] : [scope: ScopeInput] +type CopilotApplicationExecutor = < + Selected extends O, + I, + R, +>( + context: CopilotExecutionContext | undefined, + useCase: OperationUseCase, + input: I, + ...scopeArguments: ScopeArguments +) => Promise + const RESOURCE_SCOPE_KEYS = ['fileId', 'tableId', 'chatId', 'executionId'] as const function requireValidProjectedResourceScope(resourceScope: CopilotResourceScope): void { @@ -90,11 +137,11 @@ function requireMatchingPrincipal( } } -/** Adapts trusted Copilot calls to a domain's existing application use cases. */ -export function createCopilotApplicationAdapter< - O extends WorkspaceOperation, - ScopeInput = undefined, ->(options: CopilotApplicationAdapterOptions) { +function registeredCopilotOperations(options: { + domain: string + delegation: { audience: string; ttlMs: number } + operations: Readonly> +}): Set { if (!options.domain.trim()) throw new Error('Copilot application adapter requires a domain') if (!options.delegation.audience.trim()) { throw new Error('Copilot application adapter requires a delegation audience') @@ -117,7 +164,13 @@ export function createCopilotApplicationAdapter< } operationIds.add(operation.id) } - const registeredOperations = new Set(operations) + return new Set(operations) +} + +function createWorkspaceApplicationAdapter( + options: CopilotApplicationAdapterOptions +): CopilotApplicationExecutor { + const registeredOperations = registeredCopilotOperations(options) return function executeCopilotApplicationUseCase( context: CopilotExecutionContext | undefined, @@ -153,3 +206,73 @@ export function createCopilotApplicationAdapter< return useCase.execute({ principal, input }) } } + +/** Whether an operation accepts a Copilot resource grant at all; its audience is checked separately. */ +export function admitsCopilotResourceDelegation(operation: CopilotResourceOperation): boolean { + return ( + operation.principalKinds.includes('resource_delegated') && + operation.delegatedServices?.includes('copilot') === true + ) +} + +function createResourceApplicationAdapter( + options: CopilotResourceAdapterOptions +): CopilotApplicationExecutor { + const registeredOperations = registeredCopilotOperations(options) + return function executeCopilotResourceUseCase( + context: CopilotExecutionContext | undefined, + useCase: OperationUseCase, + input: I, + ...scopeArguments: ScopeArguments + ): Promise { + if (!registeredOperations.has(useCase.operation)) { + throw new Error(`Unregistered Copilot ${options.domain} operation: ${useCase.operation.id}`) + } + const operation = useCase.operation + if (!admitsCopilotResourceDelegation(operation)) { + throw new Error(`Operation ${operation.id} does not admit Copilot resource delegation`) + } + if (operation.delegationAudience !== options.delegation.audience) { + throw new Error(`Operation ${operation.id} has a different delegation audience`) + } + const trustedContext = requireTrustedCopilotResourceExecutionContext(context) + let scope: ResourceDelegationScope + if (options.projectResourceScope) { + if (scopeArguments.length !== 1) { + throw new Error(`Copilot ${options.domain} execution requires trusted scope input`) + } + scope = options.projectResourceScope(scopeArguments[0], trustedContext) + } else { + if (scopeArguments.length !== 0) { + throw new Error(`Copilot ${options.domain} execution does not accept resource scope input`) + } + scope = options.resourceScope + } + const principal = createCopilotResourceApplicationPrincipal(trustedContext, { + ...options.delegation, + scope, + }) + return useCase.execute({ principal, input }) + } +} + +/** Adapts admitted Copilot calls to registered operations without changing their owner policy. */ +export function createCopilotApplicationAdapter< + O extends CopilotResourceOperation, + ScopeInput = undefined, +>(options: CopilotResourceAdapterOptions): CopilotApplicationExecutor +export function createCopilotApplicationAdapter< + O extends WorkspaceOperation, + ScopeInput = undefined, +>( + options: CopilotApplicationAdapterOptions +): CopilotApplicationExecutor +export function createCopilotApplicationAdapter( + options: + | CopilotApplicationAdapterOptions + | CopilotResourceAdapterOptions +) { + return options.mode === 'resource' + ? createResourceApplicationAdapter(options) + : createWorkspaceApplicationAdapter(options) +} diff --git a/apps/sim/lib/mothership/application/execute-project-file-use-case.ts b/apps/sim/lib/mothership/application/execute-project-file-use-case.ts new file mode 100644 index 00000000000..84e23cf5e09 --- /dev/null +++ b/apps/sim/lib/mothership/application/execute-project-file-use-case.ts @@ -0,0 +1,65 @@ +import { markCopilotResourceRequest } from '@/lib/api/server/routes/copilot-request' +import type { ApplicationOperation } from '@/lib/core/application/operation' +import { + admitsCopilotResourceDelegation, + createCopilotApplicationAdapter, +} from '@/lib/mothership/application/adapter' +import { + type CopilotExecutionContext, + type CopilotResourceDelegationConfiguration, + createCopilotResourceApplicationPrincipal, + requireTrustedCopilotResourceExecutionContext, +} from '@/lib/mothership/auth/application-delegation' +import { projectFileOperations } from '@/lib/projects/files/application' +import type { ProjectFileTarget } from '@/lib/projects/files/application/authorization' +import { + PROJECT_FILE_DELEGATION_TTL_MS, + type ProjectFileOperation, + projectFileDelegationScope, +} from '@/lib/projects/files/application/operations' + +const PROJECT_FILE_DELEGATION: CopilotResourceDelegationConfiguration = { + audience: projectFileOperations.list.delegationAudience, + ttlMs: PROJECT_FILE_DELEGATION_TTL_MS, + createDelegationId: (context) => context.toolCallId, +} + +const REGISTERED_OPERATIONS = new Map( + Object.values(projectFileOperations).map((operation) => [operation, operation]) +) + +export const executeCopilotProjectFileUseCase = createCopilotApplicationAdapter({ + mode: 'resource', + domain: 'project files', + operations: projectFileOperations, + projectResourceScope: projectFileDelegationScope, + delegation: PROJECT_FILE_DELEGATION, +}) + +/** Binds one precise Project request to opaque authoring admission; the use case reauthorizes current access. */ +export function markCopilotProjectFileRequest( + request: Request, + context: CopilotExecutionContext, + target: ProjectFileTarget +): void { + const trustedContext = requireTrustedCopilotResourceExecutionContext(context) + if (!target.projectId.trim() || (target.fileId !== undefined && !target.fileId.trim())) { + throw new Error('Copilot Project file requests require a valid resource target') + } + const bound = Object.freeze({ ...target }) + markCopilotResourceRequest(request, (operation, audience) => { + const registered = REGISTERED_OPERATIONS.get(operation) + if ( + !registered || + !admitsCopilotResourceDelegation(registered) || + registered.delegationAudience !== audience || + (registered.target === 'file') !== (bound.fileId !== undefined) + ) { + return undefined + } + return createCopilotResourceApplicationPrincipal(trustedContext, { + ...PROJECT_FILE_DELEGATION, + scope: projectFileDelegationScope(bound), + }) + }) +} diff --git a/apps/sim/lib/mothership/application/execute-project-use-case.ts b/apps/sim/lib/mothership/application/execute-project-use-case.ts new file mode 100644 index 00000000000..be9a9592dc2 --- /dev/null +++ b/apps/sim/lib/mothership/application/execute-project-use-case.ts @@ -0,0 +1,15 @@ +import { createCopilotApplicationAdapter } from '@/lib/mothership/application/adapter' +import { projectOperations } from '@/lib/projects/application' +import { PROJECT_DISCOVERY_DELEGATION_TTL_MS } from '@/lib/projects/application/operations' + +export const executeCopilotProjectDiscovery = createCopilotApplicationAdapter({ + mode: 'resource', + domain: 'project discovery', + operations: { list: projectOperations.list }, + resourceScope: { kind: 'project_discovery' }, + delegation: { + audience: projectOperations.list.delegationAudience, + ttlMs: PROJECT_DISCOVERY_DELEGATION_TTL_MS, + createDelegationId: (context) => context.toolCallId, + }, +}) diff --git a/apps/sim/lib/mothership/application/file-owner-context.ts b/apps/sim/lib/mothership/application/file-owner-context.ts new file mode 100644 index 00000000000..8d4dd2dff6b --- /dev/null +++ b/apps/sim/lib/mothership/application/file-owner-context.ts @@ -0,0 +1,74 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { omit } from '@sim/utils/object' +import { asOrchestrationError } from '@/lib/core/orchestration/types' +import { executeCopilotProjectDiscovery } from '@/lib/mothership/application/execute-project-use-case' +import { + type CopilotExecutionContext, + hasCopilotResourceAdmission, +} from '@/lib/mothership/auth/application-delegation' +import { getCopilotFileOwnerAdapter } from '@/lib/mothership/file-owners' +import type { FileOwnerContext } from '@/lib/mothership/generated/file-owner' +import { supportsFileOwnerProtocol } from '@/lib/mothership/request/lifecycle/file-owner-protocol' +import { getCurrentWorkspaceProject } from '@/lib/projects/application' +import { isProjectsEnabled } from '@/lib/projects/rollout.server' + +const logger = createLogger('CopilotFileOwnerContext') + +/** + * Adds current owner hints to admitted authoring turns using the worker's generated contract. + * Hints are best-effort: a failed Project lookup sends the turn with no Project rather than failing + * it. Support is probed on the selected worker URL, which may front several worker releases, and a + * worker that predates the hint keys rejects them; rolling the worker back therefore requires + * turning the `projects` flag off first. + */ +export async function withFileOwnerContext( + payload: Record, + context: CopilotExecutionContext, + baseURL: string, + route: string +): Promise> { + const clean = omit(payload, ['project', 'fileOwnerCapabilities', 'fileOwnerProtocolVersion']) + if ( + !['/api/copilot', '/api/mothership'].includes(route) || + context.requestMode === 'assistant' || + !hasCopilotResourceAdmission(context) || + !(await isProjectsEnabled()) || + !(await supportsFileOwnerProtocol(baseURL)) + ) + return clean + + const hints: FileOwnerContext = { + fileOwnerProtocolVersion: 1, + project: null, + fileOwnerCapabilities: [], + } + if (context.workspaceId) { + const discoveryContext = { ...context, toolCallId: `file-context:${generateId()}` } + try { + const result = await executeCopilotProjectDiscovery( + discoveryContext, + getCurrentWorkspaceProject, + {} + ) + const project = result.project + if (project) { + const owner = { entityType: 'project' as const, entityId: project.id } + const capability = await getCopilotFileOwnerAdapter(owner).capabilities?.( + discoveryContext, + owner + ) + hints.project = { id: project.id, name: project.name } + hints.fileOwnerCapabilities = capability ? [capability] : [] + } + } catch (error) { + const code = asOrchestrationError(error)?.code + if (code !== 'forbidden' && code !== 'not_found') + logger.warn('Sending turn without Project file owner hints', { + error: getErrorMessage(error), + }) + } + } + return { ...clean, ...hints } +} diff --git a/apps/sim/lib/mothership/application/resource-adapter.test.ts b/apps/sim/lib/mothership/application/resource-adapter.test.ts new file mode 100644 index 00000000000..c2f036bf041 --- /dev/null +++ b/apps/sim/lib/mothership/application/resource-adapter.test.ts @@ -0,0 +1,240 @@ +import type { Principal } from '@sim/auth/principal' +import { describe, expect, it } from 'vitest' +import { requireResourceDelegation } from '@/lib/core/application/resource-delegation' +import { createCopilotApplicationAdapter } from '@/lib/mothership/application/adapter' +import { + type CopilotExecutionContext, + createCopilotResourceAdmission, + hasCopilotResourceAdmission, +} from '@/lib/mothership/auth/application-delegation' + +const operation = Object.freeze({ + id: 'projects.list', + capability: 'none', + principalKinds: Object.freeze(['resource_delegated'] as const), + delegatedServices: Object.freeze(['copilot'] as const), + delegationAudience: 'sim:projects:discovery', +} as const) +const delegation = { + audience: operation.delegationAudience, + ttlMs: 60_000, + createDelegationId: (context: { toolCallId: string }) => context.toolCallId, +} +const useCase = { + operation, + async execute({ principal }: { principal: Principal; input: unknown }) { + return principal + }, +} +const adapter = () => + createCopilotApplicationAdapter({ + mode: 'resource', + domain: 'projects', + delegation, + operations: { list: operation }, + resourceScope: { kind: 'project_discovery' }, + }) + +function admittedContext(): CopilotExecutionContext { + return { + userId: 'real-user', + chatId: 'real-chat', + toolCallId: 'real-tool-call', + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: 'real-user', + invocation: { kind: 'chat', chatId: 'real-chat' }, + }), + } +} + +describe('Copilot resource authoring admission', () => { + it('recognizes turn admission before a tool call exists', () => { + expect(hasCopilotResourceAdmission({ ...admittedContext(), toolCallId: undefined })).toBe(true) + }) + + it('binds the actual user and invocation without a selected workspace', async () => { + const result = await adapter()(admittedContext(), useCase, { + userId: 'forged-user', + chatId: 'forged-chat', + scope: { kind: 'entity', entityType: 'organization', entityId: 'other' }, + }) + expect(result).toMatchObject({ + kind: 'resource_delegated', + subjectUserId: 'real-user', + serviceId: 'copilot', + delegationId: 'real-tool-call', + invocation: { kind: 'chat', chatId: 'real-chat' }, + scope: { kind: 'project_discovery' }, + }) + }) + + it('retains editor workflow and Copilot run identity without treating them as workflow authority', async () => { + const result = await adapter()( + { ...admittedContext(), workflowId: 'edited-workflow', executionId: 'copilot-run' }, + useCase, + {} + ) + expect(result).toMatchObject({ + kind: 'resource_delegated', + invocation: { kind: 'chat', chatId: 'real-chat' }, + scope: { kind: 'project_discovery' }, + }) + expect(result).not.toHaveProperty('workflowId') + }) + + it('accepts an explicitly admitted workspace authoring run without a chat', async () => { + const result = await adapter()( + { + userId: 'real-user', + workspaceId: 'workspace', + toolCallId: 'tool', + copilotToolExecution: true, + copilotInteractionMode: 'headless', + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: 'real-user', + invocation: { kind: 'workspace', workspaceId: 'workspace' }, + }), + }, + useCase, + {} + ) + expect(result).toMatchObject({ invocation: { kind: 'workspace', workspaceId: 'workspace' } }) + }) + + it.each([ + ['another subject', { userId: 'other' }], + ['another chat', { chatId: 'other' }], + ['missing tool identity', { toolCallId: undefined }], + ['missing admission', { copilotResourceAdmission: undefined }], + ['executor origin', { executorDelegationOrigin: { workflowId: 'workflow' } }], + ['Mothership block', { mcpBlockId: 'block' }], + ['workflow execution', { boundWorkflowExecutionId: 'workflow-execution' }], + ])('rejects %s even when old origin flags say interactive', (_name, changes) => { + const context = { ...admittedContext(), copilotInteractionMode: 'interactive', ...changes } + expect(() => adapter()(context as never, useCase, {})).toThrow() + }) + + it('does not accept serialized or copied admission objects', () => { + const context = admittedContext() + for (const admission of [ + { userId: 'real-user', invocation: { kind: 'chat', chatId: 'real-chat' } }, + { ...context.copilotResourceAdmission }, + ]) { + expect(() => + adapter()({ ...context, copilotResourceAdmission: admission } as never, useCase, {}) + ).toThrow('authoring admission') + } + }) + + it('refuses same-ID operation substitution', () => { + expect(() => + adapter()(admittedContext(), { ...useCase, operation: { ...operation } }, {}) + ).toThrow('Unregistered Copilot projects operation') + }) + + it('refuses a registered operation that does not admit Copilot resource delegation', () => { + const refusedOperation = Object.freeze({ + ...operation, + delegatedServices: Object.freeze(['realtime'] as const), + } as const) + const execute = createCopilotApplicationAdapter({ + mode: 'resource', + domain: 'projects', + delegation, + operations: { list: refusedOperation }, + resourceScope: { kind: 'project_discovery' }, + }) + expect(() => + execute(admittedContext(), { ...useCase, operation: refusedOperation }, {}) + ).toThrow('does not admit Copilot resource delegation') + }) + + it('refuses the wrong domain audience even with a registered operation', () => { + const execute = createCopilotApplicationAdapter({ + mode: 'resource', + domain: 'projects', + delegation: { ...delegation, audience: 'sim:files' }, + operations: { list: operation }, + resourceScope: { kind: 'project_discovery' }, + }) + expect(() => execute(admittedContext(), useCase, {})).toThrow('delegation audience') + }) + + it('projects a requested file bound only from the adapter scope argument', async () => { + const execute = createCopilotApplicationAdapter({ + mode: 'resource', + domain: 'projects', + delegation, + operations: { list: operation }, + projectResourceScope: ({ fileId }) => ({ + kind: 'entity', + entityType: 'project', + entityId: 'project', + fileId, + }), + }) + const result = await execute( + admittedContext(), + useCase, + { fileId: 'forged' }, + { fileId: 'file' } + ) + expect(result).toMatchObject({ + scope: { kind: 'entity', entityType: 'project', entityId: 'project', fileId: 'file' }, + }) + }) + + it('copy delegation cannot be widened by mutating nested requested targets after admission', async () => { + const copyOperation = Object.freeze({ + ...operation, + id: 'files.copy', + delegationAudience: 'sim:files:copy', + }) + const scope = { + kind: 'file_copy' as const, + source: { + owner: { entityType: 'workspace' as const, entityId: 'source' }, + fileIds: ['one'], + folderIds: [] as string[], + }, + destination: { + owner: { entityType: 'project' as const, entityId: 'destination' }, + folderId: null as string | null, + }, + } + const required = structuredClone(scope) + const execute = createCopilotApplicationAdapter({ + mode: 'resource', + domain: 'file-copy', + delegation: { ...delegation, audience: copyOperation.delegationAudience }, + operations: { copy: copyOperation }, + resourceScope: scope, + }) + const result = await execute(admittedContext(), { ...useCase, operation: copyOperation }, {}) + scope.source.owner.entityId = 'another-owner' + scope.source.fileIds.push('extra-file') + scope.destination.folderId = 'another-folder' + expect( + requireResourceDelegation(result, { + audience: copyOperation.delegationAudience, + services: ['copilot'], + maxTtlMs: 60_000, + scope: required, + }) + ).toBe('real-user') + expect(() => + requireResourceDelegation(result, { + audience: copyOperation.delegationAudience, + services: ['copilot'], + maxTtlMs: 60_000, + scope, + }) + ).toThrow('Resource delegation is no longer valid') + if (result.kind !== 'resource_delegated' || result.scope.kind !== 'file_copy') + throw new Error('Missing copy grant') + expect(Reflect.set(result.scope.source.owner, 'entityId', 'forged')).toBe(false) + expect(Reflect.set(result.scope.source.fileIds, '1', 'injected')).toBe(false) + expect(Reflect.set(result.scope.destination, 'folderId', 'forged')).toBe(false) + }) +}) diff --git a/apps/sim/lib/mothership/auth/application-delegation.ts b/apps/sim/lib/mothership/auth/application-delegation.ts index 62993efc220..1471787c42a 100644 --- a/apps/sim/lib/mothership/auth/application-delegation.ts +++ b/apps/sim/lib/mothership/auth/application-delegation.ts @@ -1,4 +1,10 @@ -import type { DelegatedPrincipal, OrganizationDelegatedPrincipal } from '@sim/auth/principal' +import type { + DelegatedPrincipal, + OrganizationDelegatedPrincipal, + ResourceDelegatedPrincipal, + ResourceDelegationScope, +} from '@sim/auth/principal' +import { requireResourceDelegation } from '@/lib/core/application/resource-delegation' import { TOOL_WATCHDOG_LONG_RUNNING_MS } from '@/lib/mothership/constants' /** @@ -17,8 +23,160 @@ export interface CopilotExecutionContext { chatId?: string executionId?: string toolCallId?: string + mcpBlockId?: string + executorDelegationOrigin?: unknown + boundWorkflowExecutionId?: string copilotToolExecution?: boolean copilotInteractionMode?: 'interactive' | 'headless' + copilotResourceAdmission?: CopilotResourceAdmission +} + +type CopilotResourcePrincipal = Extract + +const RESOURCE_ADMISSION = Symbol('copilot-resource-admission') + +/** Opaque admission evidence carried only inside Sim; serialized copies grant no authority. */ +export interface CopilotResourceAdmission { + readonly [RESOURCE_ADMISSION]: true +} + +const RESOURCE_ADMISSIONS = new WeakMap< + CopilotResourceAdmission, + Readonly<{ userId: string; invocation: CopilotResourcePrincipal['invocation'] }> +>() + +/** A context proven admitted by {@link requireTrustedCopilotResourceExecutionContext}; only that function constructs one. */ +export interface TrustedCopilotResourceExecutionContext extends CopilotExecutionContext { + userId: string + toolCallId: string + copilotToolExecution: true + copilotResourceAdmission: CopilotResourceAdmission + invocation: CopilotResourcePrincipal['invocation'] +} + +export interface CopilotResourceDelegationConfiguration { + audience: string + ttlMs: number + createDelegationId(context: TrustedCopilotResourceExecutionContext): string +} + +/** The invocation an authoring turn acts for: its chat when it has one, else its workspace. */ +export function authoringInvocation(scope: { + chatId?: string + workspaceId?: string +}): CopilotResourcePrincipal['invocation'] | undefined { + if (scope.chatId) return { kind: 'chat', chatId: scope.chatId } + if (scope.workspaceId) return { kind: 'workspace', workspaceId: scope.workspaceId } + return undefined +} + +/** Called only after authoring ingress authenticates its user and invocation, including resumed runs. */ +export function createCopilotResourceAdmission(input: { + userId: string + invocation: CopilotResourcePrincipal['invocation'] +}): CopilotResourceAdmission { + requireNonEmpty(input.userId, 'an authenticated authoring user ID') + if (input.invocation.kind === 'chat') { + requireNonEmpty(input.invocation.chatId, 'an authoring chat ID') + } else if (input.invocation.kind === 'workspace') { + requireNonEmpty(input.invocation.workspaceId, 'an authoring workspace ID') + } else { + throw new Error('Copilot resource admission requires an authoring invocation') + } + const admission = Object.freeze({ [RESOURCE_ADMISSION]: true as const }) + RESOURCE_ADMISSIONS.set( + admission, + Object.freeze({ userId: input.userId, invocation: Object.freeze({ ...input.invocation }) }) + ) + return admission +} + +function resourceAdmissionForContext(context: CopilotExecutionContext | undefined) { + const admission = context?.copilotResourceAdmission + ? RESOURCE_ADMISSIONS.get(context.copilotResourceAdmission) + : undefined + if (!context || !admission) return undefined + if ( + context.copilotToolExecution !== true || + context.userId !== admission.userId || + context.executorDelegationOrigin !== undefined || + context.mcpBlockId !== undefined || + context.boundWorkflowExecutionId !== undefined || + (admission.invocation.kind === 'chat' + ? context.chatId !== admission.invocation.chatId + : context.workspaceId !== admission.invocation.workspaceId || context.chatId !== undefined) + ) { + return undefined + } + return admission +} + +/** Recognizes admitted authoring turns before a tool call exists, without serializing their evidence. */ +export function hasCopilotResourceAdmission(context: CopilotExecutionContext | undefined): boolean { + return resourceAdmissionForContext(context) !== undefined +} + +/** Narrows positive server admission; tool flags and model parameters cannot establish it. */ +export function requireTrustedCopilotResourceExecutionContext( + context: CopilotExecutionContext | undefined +): TrustedCopilotResourceExecutionContext { + const admission = resourceAdmissionForContext(context) + if (!context || !admission || !context.copilotResourceAdmission) { + throw new Error('Copilot resource operations require current authoring admission') + } + requireNonEmpty(context.toolCallId, 'a tool call ID') + return Object.freeze({ + userId: admission.userId, + toolCallId: context.toolCallId, + copilotToolExecution: true, + copilotResourceAdmission: context.copilotResourceAdmission, + invocation: admission.invocation, + ...(context.chatId ? { chatId: context.chatId } : {}), + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(context.organizationId ? { organizationId: context.organizationId } : {}), + ...(context.workflowId ? { workflowId: context.workflowId } : {}), + ...(context.executionId ? { executionId: context.executionId } : {}), + ...(context.requestMode ? { requestMode: context.requestMode } : {}), + ...(context.copilotInteractionMode + ? { copilotInteractionMode: context.copilotInteractionMode } + : {}), + }) +} + +/** Creates per-operation resource authority while preserving the admitted human and invocation. */ +export function createCopilotResourceApplicationPrincipal( + context: TrustedCopilotResourceExecutionContext, + options: CopilotResourceDelegationConfiguration & { scope: ResourceDelegationScope } +): CopilotResourcePrincipal { + const trustedContext = requireTrustedCopilotResourceExecutionContext(context) + const issuedAt = new Date() + const scope = structuredClone(options.scope) + if (scope.kind === 'file_copy') { + Object.freeze(scope.source.owner) + Object.freeze(scope.source.fileIds) + Object.freeze(scope.source.folderIds) + Object.freeze(scope.source) + Object.freeze(scope.destination.owner) + Object.freeze(scope.destination) + } + const principal: CopilotResourcePrincipal = Object.freeze({ + kind: 'resource_delegated', + serviceId: 'copilot', + subjectUserId: trustedContext.userId, + delegationId: options.createDelegationId(trustedContext), + audience: options.audience, + issuedAt, + expiresAt: new Date(issuedAt.getTime() + options.ttlMs), + invocation: Object.freeze({ ...trustedContext.invocation }), + scope: Object.freeze(scope), + }) + requireResourceDelegation(principal, { + audience: options.audience, + services: ['copilot'], + scope: options.scope, + maxTtlMs: options.ttlMs, + }) + return principal } export interface TrustedCopilotExecutionContext extends CopilotExecutionContext { diff --git a/apps/sim/lib/mothership/chat/application/change-resources.ts b/apps/sim/lib/mothership/chat/application/change-resources.ts index 0d6dcfb2976..2ff9af1b30c 100644 --- a/apps/sim/lib/mothership/chat/application/change-resources.ts +++ b/apps/sim/lib/mothership/chat/application/change-resources.ts @@ -9,6 +9,7 @@ import { type ChatResourceChange, changeStoredChatResources, } from '@/lib/mothership/resources/store' +import { requireProjectApiEnabled } from '@/lib/projects/rollout.server' interface ChangeChatResourcesInput { chatId: string @@ -43,6 +44,13 @@ export const changeChatResources = defineAuthorizedChatUseCase({ }, authorizationOptions: {}, async execute({ context, input }) { + /** A new Project tab waits for the rollout; removing or reordering stored tabs stays possible. */ + if ( + input.change.kind === 'upsert' && + input.change.resources.some((resource) => resource.owner?.entityType === 'project') + ) { + await requireProjectApiEnabled() + } if (input.change.kind === 'upsert' || input.change.kind === 'reorder') { for (const resource of input.change.resources) { if ( diff --git a/apps/sim/lib/mothership/chat/file-attachment.ts b/apps/sim/lib/mothership/chat/file-attachment.ts new file mode 100644 index 00000000000..1f907f0ab29 --- /dev/null +++ b/apps/sim/lib/mothership/chat/file-attachment.ts @@ -0,0 +1,79 @@ +import type { z } from 'zod' +import { + type mothershipResourceAttachmentSchema, + mothershipResourceSchema, +} from '@/lib/api/contracts/mothership-resources' +import { withWorkspaceInvocationScope } from '@/lib/core/application/workspace-invocation-scope' +import { resolveInvocationWorkspace } from '@/lib/mothership/application/workspace-target' +import { resolveOwnedFileContext } from '@/lib/mothership/chat/process-contents' +import { isOwnerScopedFile } from '@/lib/mothership/file-owners' +import type { ChatContextItem } from '@/lib/mothership/generated/protocol' + +/** + * Resolves an open file panel against its owner's current authority. The returned tab keeps the + * panel's own identity for workspace files, so a legacy address is not rewritten into a new key. + */ +export async function resolveFileAttachment( + resource: z.infer, + scope: { + userId: string + workspaceId?: string + organizationId?: string + chatId?: string + requestMode?: string + } +): Promise<{ context: ChatContextItem; tab?: z.infer }> { + const { userId, workspaceId, organizationId, chatId, requestMode } = scope + if (resource.viewId || resource.currentView) throw new Error('Invalid file resource target') + const usesWorkspace = !resource.owner || !isOwnerScopedFile(resource.owner) + const assertedWorkspaceId = usesWorkspace + ? (resource.owner?.entityId ?? resource.workspaceId) + : undefined + if (resource.workspaceId && (!usesWorkspace || resource.workspaceId !== assertedWorkspaceId)) + throw new Error('File owner conflicts with the selected workspace') + const target = usesWorkspace + ? organizationId || assertedWorkspaceId + ? await resolveInvocationWorkspace( + { userId, workspaceId, organizationId, chatId }, + assertedWorkspaceId + ) + : workspaceId + ? { workspaceId } + : null + : null + const owner = + resource.owner ?? + (target ? { entityType: 'workspace' as const, entityId: target.workspaceId } : null) + if (!owner || (usesWorkspace && !target)) throw new Error('File resource requires an owner') + const readContext = () => + resolveOwnedFileContext( + { owner, fileId: resource.id }, + { userId, workspaceId: target?.workspaceId ?? workspaceId, chatId, requestMode } + ) + const ctx = target + ? await withWorkspaceInvocationScope( + { workspaceId: target.workspaceId, organizationId }, + readContext + ) + : await readContext() + const tab = ctx.resource + ? mothershipResourceSchema.parse({ + ...resource, + ...ctx.resource, + ...(usesWorkspace ? { owner: resource.owner } : {}), + ...(organizationId && target + ? { workspaceId: target.workspaceId } + : { workspaceId: undefined, workspaceName: undefined }), + }) + : undefined + return { + context: { + ...ctx, + ...(organizationId && target + ? { content: `Workspace ${target.workspaceId}:\n${ctx.content}` } + : {}), + tag: resource.active ? '@active_tab' : '@open_tab', + }, + ...(tab ? { tab } : {}), + } +} diff --git a/apps/sim/lib/mothership/chat/persisted-message.ts b/apps/sim/lib/mothership/chat/persisted-message.ts index cd1a5f34528..e96b053061a 100644 --- a/apps/sim/lib/mothership/chat/persisted-message.ts +++ b/apps/sim/lib/mothership/chat/persisted-message.ts @@ -14,6 +14,7 @@ import { redactSensitiveContent, redactToolCallResult, } from '@/lib/mothership/chat/sim-key-redaction' +import type { FileOperationOwner } from '@/lib/mothership/generated/file-owner' import { MothershipStreamV1CompletionStatus, MothershipStreamV1EventType, @@ -41,6 +42,7 @@ export interface PersistedFileAttachment { } export interface PersistedMessageContext { + owner?: FileOperationOwner kind: string label: string workflowId?: string @@ -94,6 +96,7 @@ export function copyPersistedMessageContext(c: PersistedMessageContext): Persist return { kind: c.kind, label: c.label, + ...(c.owner ? { owner: { ...c.owner } } : {}), ...(c.workflowId ? { workflowId: c.workflowId } : {}), ...(c.knowledgeId ? { knowledgeId: c.knowledgeId } : {}), ...(c.tableId ? { tableId: c.tableId } : {}), diff --git a/apps/sim/lib/mothership/chat/post.test.ts b/apps/sim/lib/mothership/chat/post.test.ts index 25da285f890..3ec651efe99 100644 --- a/apps/sim/lib/mothership/chat/post.test.ts +++ b/apps/sim/lib/mothership/chat/post.test.ts @@ -1114,7 +1114,8 @@ describe('handleUnifiedChatPost', () => { 'ws-1', 'chat-1', expect.any(ResolvedSecretTraceRegistry), - undefined + undefined, + 'agent' ) expect(resolveActiveResourceContext).toHaveBeenCalledWith( 'table', @@ -1230,7 +1231,8 @@ describe('handleUnifiedChatPost', () => { 'ws-1', 'chat-1', expect.any(ResolvedSecretTraceRegistry), - undefined + undefined, + 'agent' ) }) diff --git a/apps/sim/lib/mothership/chat/post.ts b/apps/sim/lib/mothership/chat/post.ts index dfb4d04b14d..44f15d17f3f 100644 --- a/apps/sim/lib/mothership/chat/post.ts +++ b/apps/sim/lib/mothership/chat/post.ts @@ -20,7 +20,6 @@ import { resolveBillingAttribution, resolveOrganizationBillingAttribution, } from '@/lib/billing/core/attribution' -import { withWorkspaceInvocationScope } from '@/lib/core/application/workspace-invocation-scope' import { type AtomicClaimResult, chatSendIdempotency } from '@/lib/core/idempotency' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { listPersonalCredentials } from '@/lib/credentials/application/personal-credentials' @@ -30,7 +29,10 @@ import { } from '@/lib/knowledge/access/availability' import { loadCopilotSearchIntegrations } from '@/lib/mothership/application/load-search-integrations' import { chatOperations } from '@/lib/mothership/application/operations' -import { resolveInvocationWorkspace } from '@/lib/mothership/application/workspace-target' +import { + authoringInvocation, + createCopilotResourceAdmission, +} from '@/lib/mothership/auth/application-delegation' import { admitChatTurn } from '@/lib/mothership/chat/application/admit-turn' import { ChatSendSupersededError } from '@/lib/mothership/chat/application/send-superseded' import { @@ -54,10 +56,8 @@ import { } from '@/lib/mothership/chat/lifecycle' import { authorizeOrganizationChat } from '@/lib/mothership/chat/organization-chats' import { buildCopilotRequestPayload } from '@/lib/mothership/chat/payload' -import { - processContextsServer, - resolveActiveResourceContext, -} from '@/lib/mothership/chat/process-contents' +import { processContextsServer } from '@/lib/mothership/chat/process-contents' +import { resolveResourceAttachmentContexts } from '@/lib/mothership/chat/resource-attachments' import { MAX_FILE_SELECTION_TEXT_LENGTH, MAX_TABLE_SELECTION_COLUMNS, @@ -72,7 +72,9 @@ import { import { prepareCopilotEnvironmentContext } from '@/lib/mothership/environment-context' import { isMothershipModelSelectorEnabled, isPlanModeEnabled } from '@/lib/mothership/feature-flags' import { AssistantSearchLevel } from '@/lib/mothership/generated/assistant' +import { FileOperationOwner } from '@/lib/mothership/generated/file-owner' import { + type ChatContextItem, type ChatRequest, type ModelSelection, ModelSelectionSchema, @@ -83,6 +85,7 @@ import { TraceAttr } from '@/lib/mothership/generated/trace-attributes-v1' import { TraceSpan } from '@/lib/mothership/generated/trace-spans-v1' import { resolveMothershipModelSettings } from '@/lib/mothership/model-options' import { createBadRequestResponse, createUnauthorizedResponse } from '@/lib/mothership/request/http' +import { durableResourceAuthoringVersion } from '@/lib/mothership/request/lifecycle/recovery-config' import { createSSEStream, SSE_RESPONSE_HEADERS } from '@/lib/mothership/request/lifecycle/start' import { startCopilotOtelRoot, withCopilotSpan } from '@/lib/mothership/request/otel' import { @@ -92,13 +95,7 @@ import { } from '@/lib/mothership/request/session' import { getLocalChatStreamLease } from '@/lib/mothership/request/session/abort' import type { ExecutionContext } from '@/lib/mothership/request/types' -import { persistChatResources } from '@/lib/mothership/resources/persistence' -import { searchResourceMatchesOwner } from '@/lib/mothership/resources/search' -import { - hasAddressableId, - isEphemeralResource, - sanitizeChatResources, -} from '@/lib/mothership/resources/types' +import { hasAddressableId } from '@/lib/mothership/resources/types' import { prepareExecutionContext } from '@/lib/mothership/tools/handlers/context' import { isWorkspaceCapabilityWithheld } from '@/lib/permission-groups/capability-assertions' import { capabilityRefusalResponse } from '@/lib/permission-groups/capability-response' @@ -127,38 +124,6 @@ const FileAttachmentSchema = z.object({ path: z.string().optional(), }) -const GENERIC_RESOURCE_TITLE: Record['type'], string> = { - search: 'Search results', - sources: 'Sources', - workflow: 'Workflow', - table: 'Table', - integration: 'Integration', - file: 'File', - dashboard: 'Dashboard', - test: 'Test', - knowledgebase: 'Knowledge Base', - folder: 'Folder', - filefolder: 'File Folder', - task: 'Task', - log: 'Log', - generic: 'Resource', - browser: 'Browser', - terminal: 'Terminal', -} - -/** - * Synthetic client-side panels are context-only: never persisted to the chat. - * Browser tabs are among them — the desktop app restores its own pages — so - * their page title and URL remain request context only. - */ -function isPersistableAttachment(resource: z.infer): boolean { - return !isEphemeralResource({ - type: resource.type, - id: resource.id, - title: resource.title ?? '', - }) -} - /** * Drops open tabs the client cannot address, so one unusable tab does not fail * the whole message — clients on a stale bundle still send them. A non-string @@ -230,6 +195,7 @@ const ChatContextSchema = z 'browser_tab', 'terminal_tab', 'workspace', + 'project', ]), label: z.string().max(MAX_CHAT_CONTEXT_LABEL_LENGTH), chatId: z.string().optional(), @@ -243,11 +209,13 @@ const ChatContextSchema = z viewId: mothershipResourceSchema.shape.viewId, currentView: mothershipTableViewContextSchema.optional(), fileId: z.string().optional(), + owner: FileOperationOwner.optional(), dashboardId: z.string().optional(), folderId: z.string().optional(), fileFolderId: z.string().optional(), skillId: z.string().optional(), workspaceId: z.string().min(1).max(200).optional(), + projectId: z.string().min(1).max(200).optional(), serverId: z.string().optional(), scheduleId: z.string().optional(), tabId: z.string().optional(), @@ -261,7 +229,28 @@ const ChatContextSchema = z columnIds: z.array(z.string()).max(MAX_TABLE_SELECTION_COLUMNS).optional(), selection: z.union([BrowserTextSelectionSchema, TerminalTextSelectionSchema]).optional(), }) - .superRefine(({ kind, selection, workspaceId }, refinementContext) => { + .superRefine(({ kind, selection, workspaceId, projectId, owner }, refinementContext) => { + if ( + (kind === 'project' && (!projectId || workspaceId !== undefined)) || + (kind !== 'project' && projectId !== undefined) + ) { + refinementContext.addIssue({ + code: 'custom', + path: ['projectId'], + message: 'A Project context requires only a Project target', + }) + } + if ( + owner && + ((kind !== 'file' && kind !== 'file_selection') || + (owner.entityType !== 'workspace' && workspaceId) || + (owner.entityType === 'workspace' && workspaceId && owner.entityId !== workspaceId)) + ) + refinementContext.addIssue({ + code: 'custom', + path: ['owner'], + message: 'File owner conflicts with the context target', + }) if (kind === 'workspace' && !workspaceId) { refinementContext.addIssue({ code: 'custom', @@ -543,6 +532,7 @@ function collectChatMcpServerIds( } async function resolveAgentContexts(params: { + requestMode: UnifiedChatRequest['mode'] contexts?: UnifiedChatRequest['contexts'] resourceAttachments?: UnifiedChatRequest['resourceAttachments'] organizationId?: string @@ -554,8 +544,9 @@ async function resolveAgentContexts(params: { chatId?: string resolvedSecretTraceRegistry?: ExecutionContext['resolvedSecretTraceRegistry'] requestId: string -}): Promise> { +}): Promise { const { + requestMode, contexts, resourceAttachments, organizationId, @@ -569,7 +560,7 @@ async function resolveAgentContexts(params: { requestId, } = params - let agentContexts: Array<{ type: string; content: string; tag?: string; path?: string }> = [] + let agentContexts: ChatContextItem[] = [] if (Array.isArray(contexts) && contexts.length > 0) { try { @@ -580,109 +571,27 @@ async function resolveAgentContexts(params: { workspaceId, chatId, resolvedSecretTraceRegistry, - organizationId + organizationId, + requestMode ) } catch (error) { logger.error(`[${requestId}] Failed to process contexts`, error) } } - const authorizedResources: z.infer[] = [] - if ( - Array.isArray(resourceAttachments) && - resourceAttachments.length > 0 && - (workspaceId || organizationId) - ) { - const results = await Promise.allSettled( - resourceAttachments.map(async (resource) => { - if (resource.type === 'sources') return null - // The live browser panel resolves from the attachment itself: its - // page state is client-held (the desktop app's embedded browser), - // not a workspace entity the server could look up. - if (resource.type === 'browser') { - if (!resource.url) return null - const title = resource.title?.trim() - return { - type: 'active_resource', - tag: resource.active ? '@active_tab' : '@open_tab', - content: `The user's ${ - resource.active ? 'currently visible browser tab' : 'other open browser tab' - } is open on: ${ - title ? `"${title}" — ` : '' - }${resource.url}. ${browserAvailable ? 'Browser tools are available for inspecting and interacting with this tab.' : 'This attachment supplies only the title and URL; browser tools are unavailable.'}`, - } - } - if (resource.type === 'search') { - if ( - !resource.search || - !searchResourceMatchesOwner(resource.search, { organizationId, workspaceId }) - ) - return null - if (persistResources) - authorizedResources.push( - mothershipResourceSchema.parse({ ...resource, title: 'Search results' }) - ) - return { - type: 'active_resource', - tag: resource.active ? '@active_tab' : '@open_tab', - content: `The user's Search results tab has this retrieval address: ${JSON.stringify(resource.search)}. This is query context, not retrieved evidence; use the search tools for current authorized results.`, - } - } - const target = - organizationId || resource.workspaceId - ? await resolveInvocationWorkspace( - { userId, workspaceId, organizationId, chatId }, - resource.workspaceId - ) - : workspaceId - ? { workspaceId } - : null - if (!target) return null - const ctx = await withWorkspaceInvocationScope( - { workspaceId: target.workspaceId, organizationId }, - () => - resolveActiveResourceContext( - resource.type, - resource.id, - target.workspaceId, - userId, - chatId, - resource.viewId, - resource.currentView - ) - ) - if (!ctx) return null - if (persistResources && isPersistableAttachment(resource)) - authorizedResources.push( - mothershipResourceSchema.parse({ - ...resource, - title: resource.title ?? GENERIC_RESOURCE_TITLE[resource.type], - ...(organizationId - ? { workspaceId: target.workspaceId } - : { workspaceId: undefined, workspaceName: undefined }), - }) - ) - return { - ...ctx, - ...(organizationId - ? { content: `Workspace ${target.workspaceId}:\n${ctx.content}` } - : {}), - tag: resource.active ? '@active_tab' : '@open_tab', - } - }) - ) - - for (const result of results) { - if (result.status === 'fulfilled' && result.value) { - agentContexts.push(result.value) - } else if (result.status === 'rejected') { - logger.error(`[${requestId}] Failed to resolve resource attachment`, result.reason) - } - } - } - - if (chatId && authorizedResources.length) - await persistChatResources(chatId, sanitizeChatResources(authorizedResources)) + agentContexts.push( + ...(await resolveResourceAttachmentContexts({ + resourceAttachments, + userId, + workspaceId, + organizationId, + chatId, + requestMode, + persistResources, + browserAvailable, + requestId, + })) + ) return agentContexts } @@ -1331,6 +1240,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { }, () => resolveAgentContexts({ + requestMode: body.mode, contexts: normalizedContexts, resourceAttachments: body.resourceAttachments, organizationId: branch.kind === 'organization' ? branch.organizationId : undefined, @@ -1394,6 +1304,12 @@ export async function handleUnifiedChatPost(req: NextRequest) { executionContext.assistantSearch = body.assistantSearch executionContext.userPermission = userPermission ?? undefined + const invocation = authoringInvocation({ chatId: actualChatId, workspaceId }) + if (invocation) + executionContext.copilotResourceAdmission = createCopilotResourceAdmission({ + userId: authenticatedUserId, + invocation, + }) /** Trace catalog preparation and attachment tracking before durable admission. */ const preparedPayload = await withCopilotSpan( @@ -1490,6 +1406,9 @@ export async function handleUnifiedChatPost(req: NextRequest) { if (!lease || !sendClaim?.claimToken || sendClaim.storageMethod !== 'database') { throw new Error('Chat admission requires its controller and durable send claim') } + const resourceAuthoringVersion = await durableResourceAuthoringVersion( + executionContext.copilotResourceAdmission !== undefined + ) admittedRun = await admitChatTurn.execute({ principal: { kind: 'session', @@ -1513,6 +1432,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { }, recovery: { kind: 'interactive_stream', + resourceAuthoringVersion, request: { ...requestPayload, messageId: userMessageId, chatId: actualChatId }, goRoute: branch.goRoute, clientToolPickupExpected, diff --git a/apps/sim/lib/mothership/chat/process-contents.test.ts b/apps/sim/lib/mothership/chat/process-contents.test.ts index 249b0cf8e97..511f6be2a16 100644 --- a/apps/sim/lib/mothership/chat/process-contents.test.ts +++ b/apps/sim/lib/mothership/chat/process-contents.test.ts @@ -24,6 +24,10 @@ import { permissionGroupsResolveMock, permissionGroupsResolveMockFns, } from '@sim/testing/mocks/permission-groups-resolve.mock' +import { + readWorkspaceFileMetadataMock, + readWorkspaceFileMetadataMockFns, +} from '@sim/testing/mocks/read-workspace-file-metadata.mock' import { tableApplicationRowsMock, tableApplicationRowsMockFns, @@ -65,10 +69,10 @@ const isIntegrationDeploymentAvailable = integrationsAvailabilityMockFns.mockIsIntegrationDeploymentAvailableForVisibility const resolveInvocationWorkspace = mothershipWorkspaceTargetMockFns.mockResolveInvocationWorkspace const readWorkspaceContext = mothershipChatWorkspaceContextMockFns.mockReadWorkspaceContextExecute +const readWorkspaceFileMetadata = readWorkspaceFileMetadataMockFns.mockReadWorkspaceFileMetadata const { getSkillUseCase, - readWorkspaceFileMetadata, readTableView, readWorkflowMetadata, listWorkflowFolders, @@ -79,7 +83,6 @@ const { searchDocsExecute, } = vi.hoisted(() => ({ getSkillUseCase: vi.fn(), - readWorkspaceFileMetadata: vi.fn(), readTableView: vi.fn(), readWorkflowMetadata: vi.fn(), listWorkflowFolders: vi.fn< @@ -121,9 +124,10 @@ vi.mock('@/lib/skills/application/use-cases', () => ({ })) vi.mock('@/lib/mcp/application/use-cases', () => mcpUseCasesMock) vi.mock('@/lib/uploads/contexts/workspace/file-manager', () => workspaceFileManagerMock) -vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', () => ({ - readWorkspaceFileMetadata: { execute: readWorkspaceFileMetadata }, -})) +vi.mock( + '@/lib/workspace-files/application/read-workspace-file-metadata', + () => readWorkspaceFileMetadataMock +) vi.mock('@/lib/table/application/rows', () => tableApplicationRowsMock) vi.mock('@/lib/core/security/encryption', () => encryptionMock) vi.mock('@/lib/table/application/views', () => ({ @@ -1031,6 +1035,19 @@ describe('processContextsServer - file_selection contexts', () => { expect(ctx.path).toBeTruthy() }) + it('tags an unlabeled selection as a bare mention, never as the active resource', async () => { + getWorkspaceFile.mockResolvedValue({ name: 'notes.md', folderPath: null }) + + const [ctx] = await processContextsServer( + [{ kind: 'file_selection', fileId: 'file-1', label: '', text: 'passage' } as ChatContext], + 'user-1', + 'explain this', + 'ws-1' + ) + + expect(ctx.tag).toBe('@') + }) + it('drops the selection when the file does not resolve', async () => { getWorkspaceFile.mockResolvedValue(null) diff --git a/apps/sim/lib/mothership/chat/process-contents.ts b/apps/sim/lib/mothership/chat/process-contents.ts index 3763159b997..14ba217617d 100644 --- a/apps/sim/lib/mothership/chat/process-contents.ts +++ b/apps/sim/lib/mothership/chat/process-contents.ts @@ -23,13 +23,14 @@ import { import type { TraceSpan } from '@/lib/logs/types' import { toOverview } from '@/lib/logs/views' import { createCopilotChatKnowledgePrincipal } from '@/lib/mothership/application/execute-knowledge-use-case' +import { executeCopilotProjectDiscovery } from '@/lib/mothership/application/execute-project-use-case' import { resolveInvocationWorkspace } from '@/lib/mothership/application/workspace-target' import { COPILOT_APPLICATION_DELEGATION_TTL_MS, createCopilotChatPrincipal, + createCopilotResourceAdmission, createTrustedOrganizationCopilotPrincipal, } from '@/lib/mothership/auth/application-delegation' -import { createCopilotChatFilePrincipal } from '@/lib/mothership/auth/file-delegation' import { createCopilotChatTablePrincipal } from '@/lib/mothership/auth/table-delegation' import { getBlockVisibilityForCopilot } from '@/lib/mothership/block-visibility' import { readWorkspaceContext } from '@/lib/mothership/chat/application/workspace-context' @@ -53,14 +54,18 @@ import { } from '@/lib/mothership/chat/selection-context' import { quoteChatTranscript } from '@/lib/mothership/chat/text-transcript' import { createRegistryContentGate } from '@/lib/mothership/content-delivery' +import { getCopilotFileOwnerAdapter, isOwnerScopedFile } from '@/lib/mothership/file-owners' +import type { ChatFileIngress } from '@/lib/mothership/file-owners/types' +import { FileOperationOwner } from '@/lib/mothership/generated/file-owner' +import type { ResourceAddress } from '@/lib/mothership/generated/resources' import { QueryLogs } from '@/lib/mothership/generated/tool-catalog-v1' -import { canonicalWorkspaceFilePath } from '@/lib/mothership/vfs/path-utils' import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' import { intersectIntegrationAllowlists, resolveAccessControlBlockType, } from '@/lib/permission-groups/integration-allowlist' +import { listProjects } from '@/lib/projects/application' import { skillDelegationPolicy } from '@/lib/skills/application/authorization' import { getSkillUseCase } from '@/lib/skills/application/use-cases' import { queryTableRows } from '@/lib/table/application/rows' @@ -71,7 +76,7 @@ import type { ColumnDefinition } from '@/lib/table/types' import { workflowDelegationPolicy } from '@/lib/workflows/application/authorization' import { readWorkflowMetadata } from '@/lib/workflows/application/read-workflow' import { getBuiltinSkillById } from '@/lib/workflows/skills/builtin-skills' -import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/read-workspace-file-metadata' +import type { OwnedFileTarget } from '@/lib/workspace-files/ownership' import { getBlockRegistry } from '@/blocks/registry' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -99,6 +104,7 @@ type AgentContextType = | 'browser_tab' | 'terminal_tab' | 'workspace' + | 'project' interface AgentContext { type: AgentContextType @@ -106,6 +112,7 @@ interface AgentContext { content: string /** A CLI-readable file address; other resources carry canonical references in content. */ path?: string + resource?: ResourceAddress } const logger = createLogger('ProcessContents') @@ -149,7 +156,8 @@ export async function processContextsServer( currentWorkspaceId?: string, chatId?: string, resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry, - organizationId?: string + organizationId?: string, + requestMode?: string ): Promise { if (!Array.isArray(contexts) || contexts.length === 0) return [] if (contexts.length > MAX_CHAT_CONTEXTS) { @@ -321,6 +329,7 @@ export async function processContextsServer( tag: ctx.label ? `@${ctx.label}` : '@', content: result.content, path: result.path, + resource: result.resource, } } if (ctx.kind === 'file_selection' && ctx.fileId && workspaceId) { @@ -410,11 +419,57 @@ export async function processContextsServer( const resolveContext = async (ctx: ChatContext): Promise => { try { + if ((ctx.kind === 'file' || ctx.kind === 'file_selection') && ctx.owner !== undefined) { + const parsed = FileOperationOwner.safeParse(ctx.owner) + if (!parsed.success) return null + if (isOwnerScopedFile(parsed.data)) { + if (ctx.workspaceId) return null + return await resolveOwnedFileContext( + { owner: parsed.data, fileId: ctx.fileId }, + { userId, workspaceId: currentWorkspaceId, chatId, requestMode }, + ctx + ) + } + if ( + (ctx.workspaceId && ctx.workspaceId !== parsed.data.entityId) || + (!organizationId && currentWorkspaceId !== parsed.data.entityId) + ) + return null + ctx = { ...ctx, workspaceId: parsed.data.entityId } + } // Global code-owned templates do not require an arbitrary workspace. const builtin = ctx.kind === 'skill' && organizationId ? getBuiltinSkillById(ctx.skillId) : undefined if (builtin) return { type: 'skill', tag: ctx.label ? `@${ctx.label}` : '@', content: builtin.content } + if (ctx.kind === 'project') { + if (!organizationId || !chatId || (requestMode !== 'agent' && requestMode !== 'plan')) + return null + const { projects } = await executeCopilotProjectDiscovery( + { + userId, + chatId, + organizationId, + requestMode, + toolCallId: `context:project:${ctx.projectId}`, + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId, + invocation: { kind: 'chat', chatId }, + }), + }, + listProjects, + { organizationId, projectId: ctx.projectId, limit: 1 } + ) + const [project] = projects + return project + ? { + type: 'project', + tag: ctx.label ? `@${ctx.label}` : '@', + content: `The user tagged this Project. Shared documentation belongs to this Project. Its environments are workspaces; target the appropriate environment for workflows and execution files. A Project tag does not select an environment.\n${JSON.stringify(project)}`, + } + : null + } if (ctx.kind === 'workspace') { return organizationId && chatId ? await describeWorkspace(ctx.workspaceId, ctx.label, userId, organizationId, chatId) @@ -1034,25 +1089,10 @@ async function resolveFileResource( userId: string, chatId?: string ): Promise { - const principal = createCopilotChatFilePrincipal({ - userId, - workspaceId, - chatId, - }) - const { file: record } = await readWorkspaceFileMetadata.execute({ - principal, - input: { fileId, assertedWorkspaceId: workspaceId }, - }) - return { - type: 'active_resource', - tag: '@active_resource', - content: '', - path: canonicalWorkspaceFilePath({ - folderPath: record.folderPath, - name: record.name, - prefix: record.vfsNamespace, - }), - } + return resolveOwnedFileContext( + { owner: { entityType: 'workspace', entityId: workspaceId }, fileId }, + { userId, workspaceId, chatId } + ) } /** @@ -1087,34 +1127,39 @@ async function resolveFileSelectionResource( chatId?: string ): Promise { if (!userId) throw new Error('File selection context requires a user ID') - const principal = createCopilotChatFilePrincipal({ - userId, - workspaceId, - chatId, - }) - const { file: record } = await readWorkspaceFileMetadata.execute({ - principal, - input: { fileId, assertedWorkspaceId: workspaceId }, - }) - const path = canonicalWorkspaceFilePath({ - folderPath: record.folderPath, - name: record.name, - prefix: record.vfsNamespace, - }) - const snippet = truncateSelectionText(text) - const lineRange = - startLine && endLine && endLine !== startLine - ? ` (lines ${startLine}-${endLine})` - : startLine - ? ` (line ${startLine})` - : '' - const fence = codeFenceFor(snippet) - const content = `Selected passage from ${record.name}${lineRange}:\n\n${fence}\n${snippet}\n${fence}` + return resolveOwnedFileContext( + { owner: { entityType: 'workspace', entityId: workspaceId }, fileId }, + { userId, workspaceId, chatId }, + { kind: 'file_selection', fileId, fileName: label, text, label, startLine, endLine } + ) +} + +/** Reads canonical owner metadata while selection text remains the user's quoted snapshot. */ +export async function resolveOwnedFileContext( + target: OwnedFileTarget, + ingress: ChatFileIngress, + selected?: Extract +): Promise { + const file = await getCopilotFileOwnerAdapter(target.owner).readChatMetadata(ingress, target) + let content = '' + if (selected?.kind === 'file_selection') { + const snippet = truncateSelectionText(selected.text) + const { startLine, endLine } = selected + const range = + startLine && endLine && endLine !== startLine + ? ` (lines ${startLine}-${endLine})` + : startLine + ? ` (line ${startLine})` + : '' + const fence = codeFenceFor(snippet) + content = `Selected passage from ${file.name}${range}:\n\n${fence}\n${snippet}\n${fence}` + } return { - type: 'file_selection', - tag: label ? `@${label}` : '@', + type: selected?.kind ?? 'active_resource', + tag: selected ? (selected.label ? `@${selected.label}` : '@') : '@active_resource', content, - path, + path: file.path, + resource: { type: 'file', id: file.id, title: file.name, path: file.path, owner: file.owner }, } } diff --git a/apps/sim/lib/mothership/chat/project-file-context.integration.ts b/apps/sim/lib/mothership/chat/project-file-context.integration.ts new file mode 100644 index 00000000000..bd2b847f1d9 --- /dev/null +++ b/apps/sim/lib/mothership/chat/project-file-context.integration.ts @@ -0,0 +1,994 @@ +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname } from 'node:path' +import { db } from '@sim/db' +import { copilotChats, permissions, user, workspace, workspaceFiles } from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { Logger } from '@sim/logger' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { createRouteContext } from '@sim/testing/helpers/http' +import { authMock, authMockFns } from '@sim/testing/mocks/auth.mock' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { createMockRequest } from '@sim/testing/mocks/request.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { toRecord } from '@sim/utils/object' +import { and, eq, inArray } from 'drizzle-orm' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { createCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' +import { changeChatResources } from '@/lib/mothership/chat/application/change-resources' +import { resolveFileAttachment } from '@/lib/mothership/chat/file-attachment' +import { copyPersistedMessageContext } from '@/lib/mothership/chat/persisted-message' +import { processContextsServer } from '@/lib/mothership/chat/process-contents' +import { resolveResourceAttachmentContexts } from '@/lib/mothership/chat/resource-attachments' +import { createEvent } from '@/lib/mothership/request/session/event' +import { encodeSSEEnvelope } from '@/lib/mothership/request/session/sse' +import { persistResourceEffect } from '@/lib/mothership/resources/persist-effect' +import { persistChatResources } from '@/lib/mothership/resources/persistence' +import { createSearchResource } from '@/lib/mothership/resources/search' +import { type MothershipResource, sanitizeChatResources } from '@/lib/mothership/resources/types' +import { executeTool, registerHandler } from '@/lib/mothership/tool-executor/executor' +import { createServerToolHandler } from '@/lib/mothership/tools/registry/server-tool-adapter' +import { + POST as addCopilotResource, + DELETE as removeCopilotResource, +} from '@/app/api/copilot/chat/resources/route' +import { DELETE as removeMothershipResource } from '@/app/api/mothership/chat/resources/route' +import { GET as readMothershipChat } from '@/app/api/mothership/chats/[chatId]/route' + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) + +vi.mock('@/lib/auth', () => authMock) + +const identityChecks: { + name: string + status: 'passed' | 'failed' + durationMs: number + error?: string +}[] = [] + +function identityCheck(name: string, run: () => Promise) { + it(name, async () => { + const started = performance.now() + try { + await run() + identityChecks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + identityChecks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} + +const fixtures: { + userId: string + workspaceId: string + fileId: string + originWorkspaceId?: string +}[] = [] +beforeEach(() => { + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation(async (flag) => flag === 'projects') + vi.stubEnv('PROJECTS', 'true') +}) +async function fixture(crossProject = false) { + const userId = generateId() + const workspaceId = generateId() + const chatId = generateId() + const fileId = generateId() + await db.insert(user).values({ + id: userId, + name: 'Resource reader', + email: `${userId}@project-context.invalid`, + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + }) + await insertWorkspaceFixture(db, { + id: workspaceId, + ownerId: userId, + billedAccountUserId: userId, + name: 'Context workspace', + }) + await db.insert(permissions).values({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: 'admin', + }) + const [binding] = await db + .select({ projectId: workspace.projectId }) + .from(workspace) + .where(eq(workspace.id, workspaceId)) + if (!binding?.projectId) throw new Error('Project binding missing') + const projectId = binding.projectId + const originWorkspaceId = crossProject ? generateId() : workspaceId + if (crossProject) { + await insertWorkspaceFixture(db, { + id: originWorkspaceId, + ownerId: userId, + billedAccountUserId: userId, + name: 'Origin in another Project', + }) + await db.insert(permissions).values({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: originWorkspaceId, + permissionType: 'read', + }) + } + fixtures.push({ userId, workspaceId, fileId, originWorkspaceId }) + await db + .insert(copilotChats) + .values({ id: chatId, userId, workspaceId: originWorkspaceId, type: 'mothership' }) + await db.insert(workspaceFiles).values({ + id: fileId, + projectId: projectId, + key: `project/${projectId}/context.md`, + userId, + context: 'project', + originalName: 'Canonical.md', + contentType: 'text/markdown', + sizeBytes: 8, + }) + const owner = { entityType: 'project' as const, entityId: projectId } + const context = { + userId, + chatId, + workflowId: '', + requestMode: 'agent', + toolCallId: generateId(), + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId, + invocation: { kind: 'chat', chatId }, + }), + } + return { + userId, + workspaceId: originWorkspaceId, + targetWorkspaceId: workspaceId, + projectId, + chatId, + fileId, + owner, + context, + } +} + +describe('Project resource context and opening use canonical metadata and current authority', () => { + identityCheck( + 'opens a different Project from the chat origin and rechecks reconstructed resume admission', + async () => { + const f = await fixture(true) + registerHandler('open_resource', createServerToolHandler('open_resource')) + const result = await executeTool( + 'open_resource', + { resources: [{ type: 'file', id: f.fileId, owner: f.owner }] }, + f.context + ) + expect(result.success, result.error).toBe(true) + expect(result.resources).toEqual([ + { type: 'file', id: f.fileId, title: 'Canonical.md', owner: f.owner }, + ]) + await db + .delete(permissions) + .where(and(eq(permissions.userId, f.userId), eq(permissions.entityId, f.targetWorkspaceId))) + const resumed = { + ...f.context, + toolCallId: generateId(), + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: f.userId, + invocation: { kind: 'chat' as const, chatId: f.chatId }, + }), + } + const revoked = await executeTool( + 'open_resource', + { resources: [{ type: 'file', id: f.fileId, owner: f.owner }] }, + resumed + ) + expect(revoked.success).toBe(false) + expect(revoked.resources).toBeUndefined() + } + ) + + identityCheck( + 'cross-Project file and selection context retain their owner through authorized resolution and message persistence', + async () => { + const f = await fixture(true) + const selected = { + kind: 'file_selection' as const, + fileId: f.fileId, + owner: f.owner, + label: 'Selection', + fileName: 'Forged.md', + text: 'User-selected text', + startLine: 2, + } + for (const requestMode of [undefined, 'execute', 'assistant', 'build']) + expect( + await processContextsServer( + [selected], + f.userId, + '', + f.workspaceId, + f.chatId, + undefined, + undefined, + requestMode + ) + ).toEqual([]) + const planned = await processContextsServer( + [selected], + f.userId, + '', + f.workspaceId, + f.chatId, + undefined, + undefined, + 'plan' + ) + expect(planned).toHaveLength(1) + const contexts = await processContextsServer( + [{ kind: 'file', fileId: f.fileId, label: 'File', owner: f.owner }, selected], + f.userId, + '', + f.workspaceId, + f.chatId, + undefined, + undefined, + 'agent' + ) + expect(contexts).toHaveLength(2) + for (const context of contexts) + expect(context).toMatchObject({ + resource: { type: 'file', id: f.fileId, title: 'Canonical.md', owner: f.owner }, + }) + expect(contexts[1]?.content).toContain('Canonical.md') + expect(copyPersistedMessageContext(selected)).toMatchObject({ owner: f.owner }) + await db + .delete(permissions) + .where(and(eq(permissions.userId, f.userId), eq(permissions.entityId, f.targetWorkspaceId))) + expect( + await processContextsServer( + [selected], + f.userId, + '', + f.workspaceId, + f.chatId, + undefined, + undefined, + 'agent' + ) + ).toEqual([]) + } + ) + + it('persists Project effect identity and cannot remove it using another owner', async () => { + const f = await fixture() + const resource = { type: 'file' as const, id: f.fileId, title: 'Canonical.md', owner: f.owner } + await persistResourceEffect(f.chatId, { op: 'upsert', effectId: 'open', resource }) + for (const wrongOwner of [ + { ...f.owner, entityId: generateId() }, + { entityType: 'workspace' as const, entityId: f.owner.entityId }, + ]) + await persistResourceEffect(f.chatId, { + op: 'remove', + effectId: `wrong-owner:${wrongOwner.entityType}`, + resource: { ...resource, owner: wrongOwner }, + }) + const [stored] = await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect(stored?.resources).toEqual([resource]) + await persistResourceEffect(f.chatId, { op: 'remove', effectId: 'close', resource }) + await persistResourceEffect(f.chatId, { op: 'upsert', effectId: 'open', resource }) + const [replayed] = await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect(replayed?.resources).toEqual([]) + }) + + it('rejects contradictory owners, mixed batches and caller-supplied canonical metadata', async () => { + const f = await fixture() + registerHandler('open_resource', createServerToolHandler('open_resource')) + const resource = { type: 'file', id: f.fileId, owner: f.owner } + for (const input of [ + { workspaceId: f.workspaceId, resources: [resource] }, + { resources: [resource, { type: 'workflow', id: 'other' }] }, + { resources: [{ ...resource, title: 'forged', path: '/secret' }] }, + { resources: [{ ...resource, owner: { entityType: 'user', entityId: f.userId } }] }, + ]) { + const result = await executeTool('open_resource', input, f.context) + expect(result.success).toBe(false) + expect(result.resources).toBeUndefined() + } + const invalidSelection = { + kind: 'file_selection' as const, + fileId: f.fileId, + owner: f.owner, + workspaceId: f.workspaceId, + label: 'bad', + fileName: 'bad', + text: 'text', + } + expect( + await processContextsServer( + [invalidSelection], + f.userId, + '', + f.workspaceId, + f.chatId, + undefined, + undefined, + 'agent' + ) + ).toEqual([]) + }) +}) + +describe('canonical file panel ownership compatibility', () => { + for (const storedKind of ['legacy', 'explicit'] as const) { + identityCheck( + `removes a ${storedKind} workspace file address through its equivalent owner form`, + async () => { + const f = await fixture() + const legacy: MothershipResource = { + type: 'file', + id: f.fileId, + title: 'Workspace file', + workspaceId: f.workspaceId, + } + const explicit: MothershipResource = { + type: 'file', + id: f.fileId, + title: 'Workspace file', + owner: { entityType: 'workspace', entityId: f.workspaceId }, + } + const projectResource: MothershipResource = { + type: 'file', + id: f.fileId, + title: 'Project file', + owner: f.owner, + } + const table: MothershipResource = { + type: 'table', + id: f.fileId, + title: 'Table', + workspaceId: f.workspaceId, + } + await db + .update(copilotChats) + .set({ resources: [storedKind === 'legacy' ? legacy : explicit, projectResource, table] }) + .where(eq(copilotChats.id, f.chatId)) + const result = await changeChatResources.execute({ + principal: createSessionPrincipal({ userId: f.userId }), + input: { + chatId: f.chatId, + change: { kind: 'remove', resources: [storedKind === 'legacy' ? explicit : legacy] }, + }, + }) + expect(result.resources).toEqual([projectResource, table]) + const [persisted] = await db + .select({ resources: copilotChats.resources, workspaceId: copilotChats.workspaceId }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect(persisted).toEqual({ + resources: [projectResource, table], + workspaceId: f.workspaceId, + }) + } + ) + } + + identityCheck( + 'heals equivalent stored file aliases during upsert and reorder without inventing an owner', + async () => { + const f = await fixture() + const principal = createSessionPrincipal({ userId: f.userId }) + const legacy: MothershipResource = { + type: 'file', + id: f.fileId, + title: 'Notes', + workspaceId: f.workspaceId, + } + const explicit: MothershipResource = { + type: 'file', + id: f.fileId, + title: 'Notes', + owner: { entityType: 'workspace', entityId: f.workspaceId }, + } + const implicit: MothershipResource = { + type: 'file', + id: generateId(), + title: 'Old unresolved panel', + } + const table: MothershipResource = { + type: 'table', + id: generateId(), + title: 'Table', + workspaceId: f.workspaceId, + viewId: generateId(), + } + await db + .update(copilotChats) + .set({ resources: [legacy, implicit, table] }) + .where(eq(copilotChats.id, f.chatId)) + const added = await changeChatResources.execute({ + principal, + input: { chatId: f.chatId, change: { kind: 'upsert', resources: [explicit] } }, + }) + /** The healed tab keeps its owner and the address browsers open and close it by. */ + const healed = { ...explicit, workspaceId: f.workspaceId } + expect(added.resources).toEqual([healed, implicit, table]) + const reordered = await changeChatResources.execute({ + principal, + input: { + chatId: f.chatId, + change: { kind: 'reorder', resources: [table, legacy, implicit] }, + }, + }) + expect(reordered.resources).toEqual([table, healed, implicit]) + const [stored] = await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect(stored?.resources).toEqual([table, healed, implicit]) + } + ) + + identityCheck( + 'keeps new Project tabs behind the projects flag while stored tabs stay removable', + async () => { + const f = await fixture() + const principal = createSessionPrincipal({ userId: f.userId }) + const projectTab: MothershipResource = { + type: 'file', + id: f.fileId, + title: 'context.md', + owner: { entityType: 'project', entityId: f.projectId }, + } + const stored = async () => + ( + await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + )[0]?.resources ?? [] + const setProjects = (enabled: boolean) => { + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation( + async (flag) => enabled && flag === 'projects' + ) + vi.stubEnv('PROJECTS', enabled ? 'true' : 'false') + } + const change = (kind: 'upsert' | 'remove') => + changeChatResources.execute({ + principal, + input: { chatId: f.chatId, change: { kind, resources: [projectTab] } }, + }) + + setProjects(false) + const before = await stored() + await expect(change('upsert')).rejects.toThrow('Projects are not enabled on this deployment') + expect(await stored()).toEqual(before) + + setProjects(true) + await change('upsert') + expect(await stored()).toContainEqual( + expect.objectContaining({ id: f.fileId, owner: projectTab.owner }) + ) + + setProjects(false) + await change('remove') + expect(await stored()).not.toContainEqual(expect.objectContaining({ id: f.fileId })) + } + ) + + identityCheck( + 'rejects conflicting or unsupported file removal owners at both HTTP route boundaries', + async () => { + const f = await fixture() + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: f.userId }, + session: { id: generateId() }, + }) + const resource: MothershipResource = { + type: 'file', + id: f.fileId, + title: 'Project file', + owner: f.owner, + } + await db + .update(copilotChats) + .set({ resources: [resource] }) + .where(eq(copilotChats.id, f.chatId)) + for (const [path, handler] of [ + ['/api/copilot/chat/resources', removeCopilotResource], + ['/api/mothership/chat/resources', removeMothershipResource], + ] as const) { + for (const address of [ + { owner: f.owner, workspaceId: f.workspaceId }, + { + owner: { entityType: 'workspace', entityId: generateId() }, + workspaceId: f.workspaceId, + }, + { owner: { entityType: 'organization', entityId: generateId() } }, + { owner: f.owner, resourceType: 'table' }, + ]) { + const response = await handler( + createMockRequest({ + method: 'DELETE', + url: `http://localhost${path}`, + body: { + chatId: f.chatId, + resourceType: 'file', + resourceId: f.fileId, + ...address, + }, + }), + undefined + ) + expect( + response.status, + JSON.stringify({ path, address, body: await response.json() }) + ).toBe(400) + } + } + const [stored] = await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect(stored?.resources).toEqual([resource]) + } + ) +}) + +describe('first-message file panel persistence', () => { + for (const panelForm of ['implicit', 'legacy'] as const) + for (const storedForm of ['none', 'legacy'] as const) + identityCheck( + `saves a ${panelForm} workspace file panel once over a ${storedForm} stored tab`, + async () => { + const f = await fixture() + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: f.userId }, + session: { id: generateId() }, + }) + const fileId = generateId() + await db.insert(workspaceFiles).values({ + id: fileId, + workspaceId: f.workspaceId, + key: `workspace/${f.workspaceId}/draft.md`, + userId: f.userId, + context: 'workspace', + originalName: 'Canonical file.md', + contentType: 'text/markdown', + sizeBytes: 8, + }) + try { + const legacy = { type: 'file' as const, id: fileId, workspaceId: f.workspaceId } + if (storedForm === 'legacy') + await db + .update(copilotChats) + .set({ resources: [{ ...legacy, title: 'Old title' }] }) + .where(eq(copilotChats.id, f.chatId)) + const panel = + panelForm === 'legacy' + ? { ...legacy, title: 'Draft' } + : { type: 'file' as const, id: fileId, title: 'Draft' } + const { tab } = await resolveFileAttachment(panel, { + userId: f.userId, + workspaceId: f.workspaceId, + chatId: f.chatId, + requestMode: 'agent', + }) + if (!tab) throw new Error('Expected a persistable tab') + await persistChatResources(f.chatId, sanitizeChatResources([tab])) + const history = await readMothershipChat( + createMockRequest({ + method: 'GET', + url: `http://localhost/api/mothership/chats/${f.chatId}`, + }), + createRouteContext({ chatId: f.chatId }) + ) + expect(history.status).toBe(200) + const saved = toRecord(toRecord(await history.json()).chat) + .resources as MothershipResource[] + expect(saved).toHaveLength(1) + expect(saved[0]?.id).toBe(fileId) + /** An existing tab keeps its specific title; a new tab takes the canonical name. */ + expect(saved[0]?.title).toBe( + storedForm === 'legacy' ? 'Old title' : 'Canonical file.md' + ) + const [stored] = await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect(stored?.resources).toHaveLength(1) + } finally { + await db.delete(workspaceFiles).where(eq(workspaceFiles.id, fileId)) + } + } + ) +}) + +describe('open tabs that no longer resolve', () => { + /** The real module loggers, observed so each skipped tab's log level is the asserted output. */ + function observeLogs() { + const spies = { + info: vi.spyOn(Logger.prototype, 'info'), + warn: vi.spyOn(Logger.prototype, 'warn'), + error: vi.spyOn(Logger.prototype, 'error'), + } + return { + calls: (level: keyof typeof spies, message: string) => + spies[level].mock.calls + .filter(([text]) => text.includes(message)) + .map(([, detail]) => detail), + restore: () => { + for (const spy of Object.values(spies)) spy.mockRestore() + }, + } + } + + async function insertWorkspaceFile( + f: Awaited>, + name: string, + deletedAt: Date | null = null + ) { + const id = generateId() + await db.insert(workspaceFiles).values({ + id, + workspaceId: f.workspaceId, + key: `workspace/${f.workspaceId}/${id}.md`, + userId: f.userId, + context: 'workspace', + originalName: name, + contentType: 'text/markdown', + sizeBytes: 4, + deletedAt, + }) + return id + } + + const browserTab = { + type: 'browser' as const, + id: 'browser-tab', + title: 'Docs', + url: 'https://docs.invalid/page', + } + + identityCheck( + 'skips a deleted file tab at info and a rejected tab at error while live tabs reach the turn and persist', + async () => { + const f = await fixture() + const liveId = await insertWorkspaceFile(f, 'Live.md') + const deletedId = await insertWorkspaceFile(f, 'Deleted.md', new Date()) + const logs = observeLogs() + try { + const contexts = await resolveResourceAttachmentContexts({ + resourceAttachments: [ + { type: 'file', id: deletedId, title: 'Deleted.md' }, + { type: 'file', id: generateId(), title: 'Rejected', viewId: generateId() }, + { type: 'file', id: liveId, title: 'Live.md' }, + browserTab, + ], + userId: f.userId, + workspaceId: f.workspaceId, + chatId: f.chatId, + requestMode: 'agent', + persistResources: true, + requestId: generateId(), + }) + expect(contexts).toHaveLength(2) + expect(contexts[0]).toMatchObject({ + tag: '@open_tab', + resource: { type: 'file', id: liveId, title: 'Live.md' }, + }) + expect(contexts[1]?.content).toContain(browserTab.url) + expect(logs.calls('info', 'Skipped resource attachment that no longer exists')).toEqual([ + { resourceType: 'file' }, + ]) + expect(logs.calls('error', 'Failed to resolve resource attachment')).toHaveLength(1) + expect( + logs.calls('warn', 'Skipped resource attachment the user can no longer access') + ).toEqual([]) + } finally { + logs.restore() + await db.delete(workspaceFiles).where(inArray(workspaceFiles.id, [liveId, deletedId])) + } + const [stored] = await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect((stored?.resources as MothershipResource[]).map((resource) => resource.id)).toEqual([ + liveId, + ]) + } + ) + + identityCheck( + 'skips a file tab the user can no longer read at warning level and keeps tabs that need no access', + async () => { + const f = await fixture() + const fileId = await insertWorkspaceFile(f, 'Revoked.md') + await db + .delete(permissions) + .where(and(eq(permissions.userId, f.userId), eq(permissions.entityId, f.workspaceId))) + const logs = observeLogs() + try { + const contexts = await resolveResourceAttachmentContexts({ + resourceAttachments: [{ type: 'file', id: fileId, title: 'Revoked.md' }, browserTab], + userId: f.userId, + workspaceId: f.workspaceId, + chatId: f.chatId, + requestMode: 'agent', + persistResources: true, + requestId: generateId(), + }) + expect(contexts).toHaveLength(1) + expect(contexts[0]?.content).toContain(browserTab.url) + expect( + logs.calls('warn', 'Skipped resource attachment the user can no longer access') + ).toEqual([{ resourceType: 'file' }]) + expect(logs.calls('info', 'Skipped resource attachment that no longer exists')).toEqual([]) + expect(logs.calls('error', 'Failed to resolve resource attachment')).toEqual([]) + } finally { + logs.restore() + await db.delete(workspaceFiles).where(eq(workspaceFiles.id, fileId)) + } + const [stored] = await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect(stored?.resources).toEqual([]) + } + ) + + identityCheck( + 'persists new-chat tabs in the order the user has them, not the order they resolve', + async () => { + const f = await fixture() + const fileId = await insertWorkspaceFile(f, 'First.md') + /** The search tab resolves synchronously, so it finishes before the file tab's lookups. */ + const search = createSearchResource({ + query: 'notes', + scope: { kind: 'workspace', workspaceId: f.workspaceId }, + }) + try { + await resolveResourceAttachmentContexts({ + resourceAttachments: [{ type: 'file', id: fileId, title: 'First.md' }, search], + userId: f.userId, + workspaceId: f.workspaceId, + chatId: f.chatId, + requestMode: 'agent', + persistResources: true, + requestId: generateId(), + }) + const [stored] = await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect((stored?.resources as MothershipResource[]).map((resource) => resource.id)).toEqual([ + fileId, + search.id, + ]) + } finally { + await db.delete(workspaceFiles).where(eq(workspaceFiles.id, fileId)) + } + } + ) +}) + +describe('legacy browser file address compatibility', () => { + identityCheck( + 'persists and hydrates owned file folders without mixing their owner addresses', + async () => { + const f = await fixture() + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: f.userId }, + session: { id: generateId() }, + }) + const folderId = generateId() + const resources: MothershipResource[] = [ + { + type: 'filefolder', + id: folderId, + title: 'Workspace folder', + owner: { entityType: 'workspace', entityId: f.workspaceId }, + }, + { type: 'filefolder', id: folderId, title: 'Project folder', owner: f.owner }, + ] + for (const resource of resources) { + const response = await addCopilotResource( + createMockRequest({ + method: 'POST', + url: 'http://localhost/api/copilot/chat/resources', + body: { chatId: f.chatId, resource }, + }), + undefined + ) + expect(response.status, JSON.stringify(await response.json())).toBe(200) + } + const history = await readMothershipChat( + createMockRequest({ + method: 'GET', + url: `http://localhost/api/mothership/chats/${f.chatId}`, + }), + createRouteContext({ chatId: f.chatId }) + ) + expect(history.status).toBe(200) + expect(toRecord(toRecord(await history.json()).chat).resources).toEqual([ + { ...resources[0], workspaceId: f.workspaceId }, + resources[1], + ]) + + const removed = await removeMothershipResource( + createMockRequest({ + method: 'DELETE', + url: 'http://localhost/api/mothership/chat/resources', + body: { + chatId: f.chatId, + resourceType: 'filefolder', + resourceId: folderId, + owner: f.owner, + }, + }), + undefined + ) + expect(removed.status).toBe(200) + const [stored] = await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect(stored.resources).toEqual([resources[0]]) + + const conflicting = await addCopilotResource( + createMockRequest({ + method: 'POST', + url: 'http://localhost/api/copilot/chat/resources', + body: { chatId: f.chatId, resource: { ...resources[1], workspaceId: f.workspaceId } }, + }), + undefined + ) + expect(conflicting.status).toBe(400) + } + ) + + identityCheck( + 'projects Workspace aliases only in HTTP responses while keeping canonical storage', + async () => { + const f = await fixture() + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: f.userId }, + session: { id: generateId() }, + }) + const workspaceFile: MothershipResource = { + type: 'file', + id: generateId(), + title: 'Workspace notes', + owner: { entityType: 'workspace', entityId: f.workspaceId }, + } + const projectFile: MothershipResource = { + type: 'file', + id: f.fileId, + title: 'Project notes', + owner: f.owner, + } + const ownerlessFile: MothershipResource = { + type: 'file', + id: generateId(), + title: 'Legacy unresolved file', + } + const table: MothershipResource = { + type: 'table', + id: generateId(), + title: 'Table', + workspaceId: f.workspaceId, + } + const canonical = [workspaceFile, projectFile, ownerlessFile, table] + await db + .update(copilotChats) + .set({ resources: canonical }) + .where(eq(copilotChats.id, f.chatId)) + const expectedWire = [ + { ...workspaceFile, workspaceId: f.workspaceId }, + projectFile, + ownerlessFile, + table, + ] + const history = await readMothershipChat( + createMockRequest({ + method: 'GET', + url: `http://localhost/api/mothership/chats/${f.chatId}`, + }), + createRouteContext({ chatId: f.chatId }) + ) + expect(history.status).toBe(200) + const historyBody = toRecord(await history.json()) + expect(toRecord(historyBody.chat).resources).toEqual(expectedWire) + const mutation = await addCopilotResource( + createMockRequest({ + method: 'POST', + url: 'http://localhost/api/copilot/chat/resources', + body: { chatId: f.chatId, resource: workspaceFile }, + }), + undefined + ) + expect(mutation.status).toBe(200) + expect(toRecord(await mutation.json()).resources).toEqual(expectedWire) + const [stored] = await db + .select({ resources: copilotChats.resources }) + .from(copilotChats) + .where(eq(copilotChats.id, f.chatId)) + expect(stored?.resources).toEqual(canonical) + } + ) + + identityCheck( + 'projects Workspace aliases in the shared live and replay encoder without mutating events', + async () => { + const workspaceId = generateId() + const workspaceFile: MothershipResource = { + type: 'file', + id: generateId(), + title: 'Workspace file', + owner: { entityType: 'workspace', entityId: workspaceId }, + } + const projectFile: MothershipResource = { + type: 'file', + id: generateId(), + title: 'Project file', + owner: { entityType: 'project', entityId: generateId() }, + } + const ownerless: MothershipResource = { type: 'file', id: generateId(), title: 'Legacy file' } + const table: MothershipResource = { + type: 'table', + id: generateId(), + title: 'Table', + workspaceId, + } + for (const resource of [workspaceFile, projectFile, ownerless, table]) { + const event = createEvent({ + streamId: generateId(), + cursor: '0', + seq: 0, + requestId: generateId(), + type: 'resource', + payload: { op: 'upsert', resource }, + }) + const canonical = structuredClone(event) + const encoded = new TextDecoder().decode(encodeSSEEnvelope(event)) + expect(encoded.startsWith('data: ')).toBe(true) + expect(encoded.endsWith('\n\n')).toBe(true) + const frame = toRecord(JSON.parse(encoded.slice(6).trim())) + const expected = resource === workspaceFile ? { ...resource, workspaceId } : resource + expect(toRecord(frame.payload).resource).toEqual(expected) + expect(event).toEqual(canonical) + } + } + ) +}) + +afterAll(async () => { + const reportPath = process.env.CHAT_RESOURCE_IDENTITY_REPORT_PATH + if (reportPath) { + await mkdir(dirname(reportPath), { recursive: true }) + await writeFile(reportPath, JSON.stringify({ checks: identityChecks }, null, 2)) + } + for (const f of fixtures) { + await db.delete(workspaceFiles).where(eq(workspaceFiles.id, f.fileId)) + await deleteWorkspaceFixture(db, eq(workspace.id, f.workspaceId)) + if (f.originWorkspaceId && f.originWorkspaceId !== f.workspaceId) + await deleteWorkspaceFixture(db, eq(workspace.id, f.originWorkspaceId)) + await db.delete(user).where(eq(user.id, f.userId)) + } + await db.$client.end() +}) diff --git a/apps/sim/lib/mothership/chat/resource-attachments.ts b/apps/sim/lib/mothership/chat/resource-attachments.ts new file mode 100644 index 00000000000..fa11f12ce4d --- /dev/null +++ b/apps/sim/lib/mothership/chat/resource-attachments.ts @@ -0,0 +1,210 @@ +import { createLogger } from '@sim/logger' +import type { z } from 'zod' +import { + type mothershipResourceAttachmentSchema, + mothershipResourceSchema, +} from '@/lib/api/contracts/mothership-resources' +import { withWorkspaceInvocationScope } from '@/lib/core/application/workspace-invocation-scope' +import { asOrchestrationError } from '@/lib/core/orchestration/types' +import { resolveInvocationWorkspace } from '@/lib/mothership/application/workspace-target' +import { resolveFileAttachment } from '@/lib/mothership/chat/file-attachment' +import { resolveActiveResourceContext } from '@/lib/mothership/chat/process-contents' +import type { ChatContextItem } from '@/lib/mothership/generated/protocol' +import { persistChatResources } from '@/lib/mothership/resources/persistence' +import { searchResourceMatchesOwner } from '@/lib/mothership/resources/search' +import { isEphemeralResource, sanitizeChatResources } from '@/lib/mothership/resources/types' + +const logger = createLogger('UnifiedChatAPI') + +type ResourceAttachment = z.infer + +const GENERIC_RESOURCE_TITLE: Record = { + search: 'Search results', + sources: 'Sources', + workflow: 'Workflow', + table: 'Table', + integration: 'Integration', + file: 'File', + dashboard: 'Dashboard', + test: 'Test', + knowledgebase: 'Knowledge Base', + folder: 'Folder', + filefolder: 'File Folder', + task: 'Task', + log: 'Log', + generic: 'Resource', + browser: 'Browser', + terminal: 'Terminal', +} + +/** + * Synthetic client-side panels are context-only: never persisted to the chat. + * Browser tabs are among them — the desktop app restores its own pages — so + * their page title and URL remain request context only. + */ +function isPersistableAttachment(resource: ResourceAttachment): boolean { + return !isEphemeralResource({ + type: resource.type, + id: resource.id, + title: resource.title ?? '', + }) +} + +/** + * Resolves the user's open tabs into turn context against current authority. A tab whose resource + * is gone or no longer readable is skipped rather than failing the turn, and only tabs that + * resolved are persisted to a new chat. + */ +export async function resolveResourceAttachmentContexts(params: { + resourceAttachments?: ResourceAttachment[] + userId: string + workspaceId?: string + organizationId?: string + chatId?: string + requestMode?: string + persistResources?: boolean + browserAvailable?: boolean + requestId: string +}): Promise { + const { + resourceAttachments, + userId, + workspaceId, + organizationId, + chatId, + requestMode, + persistResources, + browserAvailable, + requestId, + } = params + const agentContexts: ChatContextItem[] = [] + /** Indexed by attachment so concurrent resolution persists tabs in the order the user has them. */ + const authorizedResources: (z.infer | undefined)[] = [] + if ( + Array.isArray(resourceAttachments) && + resourceAttachments.length > 0 && + (workspaceId || organizationId) + ) { + const results = await Promise.allSettled( + resourceAttachments.map(async (resource, index) => { + if (resource.type === 'sources') return null + // The live browser panel resolves from the attachment itself: its + // page state is client-held (the desktop app's embedded browser), + // not a workspace entity the server could look up. + if (resource.type === 'browser') { + if (!resource.url) return null + const title = resource.title?.trim() + return { + type: 'active_resource', + tag: resource.active ? '@active_tab' : '@open_tab', + content: `The user's ${ + resource.active ? 'currently visible browser tab' : 'other open browser tab' + } is open on: ${ + title ? `"${title}" — ` : '' + }${resource.url}. ${browserAvailable ? 'Browser tools are available for inspecting and interacting with this tab.' : 'This attachment supplies only the title and URL; browser tools are unavailable.'}`, + } + } + if (resource.type === 'search') { + if ( + !resource.search || + !searchResourceMatchesOwner(resource.search, { organizationId, workspaceId }) + ) + return null + if (persistResources) + authorizedResources[index] = mothershipResourceSchema.parse({ + ...resource, + title: 'Search results', + }) + return { + type: 'active_resource', + tag: resource.active ? '@active_tab' : '@open_tab', + content: `The user's Search results tab has this retrieval address: ${JSON.stringify(resource.search)}. This is query context, not retrieved evidence; use the search tools for current authorized results.`, + } + } + if (resource.type === 'file') { + const { context, tab } = await resolveFileAttachment(resource, { + userId, + workspaceId, + organizationId, + chatId, + requestMode, + }) + if (persistResources && tab) authorizedResources[index] = tab + return context + } + const target = + organizationId || resource.workspaceId + ? await resolveInvocationWorkspace( + { userId, workspaceId, organizationId, chatId }, + resource.workspaceId + ) + : workspaceId + ? { workspaceId } + : null + if (!target) return null + if ( + resource.owner && + (resource.owner.entityType !== 'workspace' || + resource.owner.entityId !== target.workspaceId) + ) + throw new Error('File owner does not match the selected workspace') + const ctx = await withWorkspaceInvocationScope( + { workspaceId: target.workspaceId, organizationId }, + () => + resolveActiveResourceContext( + resource.type, + resource.id, + target.workspaceId, + userId, + chatId, + resource.viewId, + resource.currentView + ) + ) + if (!ctx) return null + if (persistResources && isPersistableAttachment(resource)) + authorizedResources[index] = mothershipResourceSchema.parse({ + ...resource, + title: resource.title ?? GENERIC_RESOURCE_TITLE[resource.type], + ...(organizationId + ? { workspaceId: target.workspaceId } + : { workspaceId: undefined, workspaceName: undefined }), + }) + return { + ...ctx, + ...(organizationId + ? { content: `Workspace ${target.workspaceId}:\n${ctx.content}` } + : {}), + tag: resource.active ? '@active_tab' : '@open_tab', + } + }) + ) + + results.forEach((result, index) => { + if (result.status === 'fulfilled') { + if (result.value) agentContexts.push(result.value) + return + } + // A tab can outlive its resource or the viewer's access to it; the turn + // goes without that context. + const resourceType = resourceAttachments[index]?.type + const code = asOrchestrationError(result.reason)?.code + if (code === 'not_found') { + logger.info(`[${requestId}] Skipped resource attachment that no longer exists`, { + resourceType, + }) + } else if (code === 'forbidden') { + logger.warn(`[${requestId}] Skipped resource attachment the user can no longer access`, { + resourceType, + }) + } else { + logger.error(`[${requestId}] Failed to resolve resource attachment`, result.reason) + } + }) + } + + const orderedResources = authorizedResources.filter((resource) => resource !== undefined) + if (chatId && orderedResources.length) + await persistChatResources(chatId, sanitizeChatResources(orderedResources)) + return agentContexts +} diff --git a/apps/sim/lib/mothership/chat/selection-clipboard.test.ts b/apps/sim/lib/mothership/chat/selection-clipboard.test.ts index 0f4082476a5..8611247b10f 100644 --- a/apps/sim/lib/mothership/chat/selection-clipboard.test.ts +++ b/apps/sim/lib/mothership/chat/selection-clipboard.test.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from 'vitest' -import type { ChatContext } from '@/stores/panel' import { attachSelectionContextToClipboard, readSelectionContextFromClipboard, SIM_SELECTION_MIME, -} from './selection-clipboard' +} from '@/lib/mothership/chat/selection-clipboard' +import type { ChatContext } from '@/stores/panel' /** Minimal DataTransfer stand-in (jsdom-free node env). */ function fakeClipboard(initial: Record = {}) { diff --git a/apps/sim/lib/mothership/content-delivery/index.ts b/apps/sim/lib/mothership/content-delivery/index.ts index 8e61d78283f..e05d7ff90ac 100644 --- a/apps/sim/lib/mothership/content-delivery/index.ts +++ b/apps/sim/lib/mothership/content-delivery/index.ts @@ -5,4 +5,5 @@ export { markCommittedWriteResponse, } from './dispatch' export { createRegistryContentGate } from './registry-gate' -export { createSessionContentGate, WorkbenchProvenanceUnavailableError } from './session-gate' +export { deliverSandboxResponse } from './sandbox-delivery' +export { createSessionContentGate } from './session-gate' diff --git a/apps/sim/lib/mothership/content-delivery/sandbox-delivery.ts b/apps/sim/lib/mothership/content-delivery/sandbox-delivery.ts new file mode 100644 index 00000000000..b1c9d22845a --- /dev/null +++ b/apps/sim/lib/mothership/content-delivery/sandbox-delivery.ts @@ -0,0 +1,42 @@ +import { createLogger } from '@sim/logger' +import type { ContentDeliveryGate } from '@/lib/core/application' +import { ContentGateFailure } from '@/lib/core/application' +import { + dispatchUnderContentGate, + markCommittedWriteResponse, +} from '@/lib/mothership/content-delivery/dispatch' +import { WorkbenchProvenanceUnavailableError } from '@/lib/mothership/content-delivery/session-gate' + +const logger = createLogger('MothershipSandboxDelivery') + +/** + * Runs one sandbox callback under the workbench's content gate. When the machine's history cannot + * be widened for what the request read, its result is withheld: a mutation may already have + * committed, so the response says so and is never retried on the caller's behalf. + */ +export async function deliverSandboxResponse( + gate: ContentDeliveryGate, + dispatch: () => Promise, + toolCallId: string +): Promise { + try { + return (await dispatchUnderContentGate(gate, dispatch)).response + } catch (error) { + if ( + !(error instanceof ContentGateFailure) || + !(error.gateFailure instanceof WorkbenchProvenanceUnavailableError) + ) { + throw error + } + logger.warn('Sandbox response provenance could not be recorded', { toolCallId }) + const withheld = Response.json( + { + error: error.committedWrite + ? 'The change was saved, but its result could not be returned safely. Do not retry the change.' + : 'The API request ran, but its result could not be returned safely. Do not retry a mutation automatically; read the resource to check its current state.', + }, + { status: 502 } + ) + return error.committedWrite ? markCommittedWriteResponse(withheld) : withheld + } +} diff --git a/apps/sim/lib/mothership/content-delivery/session-gate.ts b/apps/sim/lib/mothership/content-delivery/session-gate.ts index 89f6b1c439e..a7adfd00f98 100644 --- a/apps/sim/lib/mothership/content-delivery/session-gate.ts +++ b/apps/sim/lib/mothership/content-delivery/session-gate.ts @@ -26,7 +26,8 @@ export class WorkbenchProvenanceUnavailableError extends Error { */ export function createSessionContentGate(scope: { sessionKey: string - workspaceId: string + /** Absent for a Project callback, which reads files no workspace owns. */ + workspaceId?: string actorUserId: string }): ContentDeliveryGate { return { diff --git a/apps/sim/lib/mothership/file-owners/index.ts b/apps/sim/lib/mothership/file-owners/index.ts new file mode 100644 index 00000000000..c9c854c0719 --- /dev/null +++ b/apps/sim/lib/mothership/file-owners/index.ts @@ -0,0 +1,4 @@ +export { + getCopilotFileOwnerAdapter, + isOwnerScopedFile, +} from '@/lib/mothership/file-owners/registry' diff --git a/apps/sim/lib/mothership/file-owners/project.ts b/apps/sim/lib/mothership/file-owners/project.ts new file mode 100644 index 00000000000..3f42dd3ef12 --- /dev/null +++ b/apps/sim/lib/mothership/file-owners/project.ts @@ -0,0 +1,77 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { executeCopilotProjectFileUseCase } from '@/lib/mothership/application/execute-project-file-use-case' +import { + authoringInvocation, + createCopilotResourceAdmission, +} from '@/lib/mothership/auth/application-delegation' +import type { CopilotFileOwnerAdapter } from '@/lib/mothership/file-owners/types' +import { canonicalWorkspaceFilePath } from '@/lib/mothership/vfs/path-utils' +import { + getProjectFileCapabilities, + getProjectFileMetadata, +} from '@/lib/projects/files/application' +import { fileOwnerVfsPath } from '@/lib/workspace-files/owner-paths' + +export const projectFileOwnerAdapter: CopilotFileOwnerAdapter = { + invocationScope: 'owner', + async readChatMetadata(ingress, target) { + if (ingress.requestMode !== 'agent' && ingress.requestMode !== 'plan') { + throw new OrchestrationError('forbidden', 'Project file context requires agent or plan mode') + } + const invocation = authoringInvocation(ingress) + if (!invocation) { + throw new OrchestrationError('forbidden', 'File context requires an authoring invocation') + } + const context = { + ...ingress, + toolCallId: `context:${target.fileId}`, + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: ingress.userId, + invocation, + }), + } + const input = { projectId: target.owner.entityId, fileId: target.fileId } + const { file } = await executeCopilotProjectFileUseCase( + context, + getProjectFileMetadata, + input, + input + ) + return { + id: file.id, + name: file.name, + owner: file.owner, + path: fileOwnerVfsPath( + file.owner, + canonicalWorkspaceFilePath({ folderPath: file.folderPath, name: file.name }) + ), + } + }, + async executeCli(request, context, owner) { + if (request.workspaceId !== undefined) { + throw new OrchestrationError( + 'validation', + 'A Project file target cannot also select a workspace' + ) + } + const { executeProjectFileCliRequest } = await import( + '@/lib/mothership/agent-cli/project-files' + ) + return executeProjectFileCliRequest(request, context, owner.entityId) + }, + async readMetadata(context, { owner, fileId }) { + const target = { projectId: owner.entityId, fileId } + const { file } = await executeCopilotProjectFileUseCase( + context, + getProjectFileMetadata, + target, + target + ) + return { id: file.id, name: file.name, owner: file.owner } + }, + capabilities(context, owner) { + const target = { projectId: owner.entityId } + return executeCopilotProjectFileUseCase(context, getProjectFileCapabilities, target, target) + }, +} diff --git a/apps/sim/lib/mothership/file-owners/registry.ts b/apps/sim/lib/mothership/file-owners/registry.ts new file mode 100644 index 00000000000..77504913216 --- /dev/null +++ b/apps/sim/lib/mothership/file-owners/registry.ts @@ -0,0 +1,23 @@ +import { projectFileOwnerAdapter } from '@/lib/mothership/file-owners/project' +import type { CopilotFileOwnerAdapter } from '@/lib/mothership/file-owners/types' +import { workspaceFileOwnerAdapter } from '@/lib/mothership/file-owners/workspace' +import { + type FileOwnerAdapters, + requireFileOwnerAdapter, +} from '@/lib/workspace-files/owner-adapters' +import type { FileOwner } from '@/lib/workspace-files/ownership' + +const adapters: FileOwnerAdapters = Object.freeze({ + workspace: workspaceFileOwnerAdapter, + project: projectFileOwnerAdapter, +}) + +/** Selects owner policy without deriving file ownership from conversation context. */ +export function getCopilotFileOwnerAdapter(owner: FileOwner): CopilotFileOwnerAdapter { + return requireFileOwnerAdapter(adapters, owner) +} + +/** Whether a file owner binds its own invocation rather than the invocation's workspace. */ +export function isOwnerScopedFile(owner: FileOwner): boolean { + return getCopilotFileOwnerAdapter(owner).invocationScope === 'owner' +} diff --git a/apps/sim/lib/mothership/file-owners/types.ts b/apps/sim/lib/mothership/file-owners/types.ts new file mode 100644 index 00000000000..1a6901d758e --- /dev/null +++ b/apps/sim/lib/mothership/file-owners/types.ts @@ -0,0 +1,45 @@ +import type { AgentCliExecutionContext } from '@/lib/mothership/agent-cli' +import type { CopilotExecutionContext } from '@/lib/mothership/auth/application-delegation' +import type { AgentCliRawResult, AgentCliRequest } from '@/lib/mothership/generated/agent-cli' +import type { FileOwnerContext } from '@/lib/mothership/generated/file-owner' +import type { EditableFileOwner, FileOwner, OwnedFileTarget } from '@/lib/workspace-files/ownership' + +export interface ChatFileIngress { + userId: string + workspaceId?: string + chatId?: string + requestMode?: string +} + +interface ChatFileMetadata { + id: string + name: string + owner: EditableFileOwner + path: string +} + +/** + * Per-owner Copilot file policy: CLI execution, metadata reads, and capability hints, each + * reauthorized by the owner's use cases. + */ +export interface CopilotFileOwnerAdapter { + /** + * `workspace`: legacy workspace files share the invocation's workspace with other resource + * kinds. `owner`: the owner binds its own invocation and never takes a workspace target. + */ + invocationScope: 'workspace' | 'owner' + executeCli( + request: AgentCliRequest, + context: AgentCliExecutionContext, + owner: FileOwner + ): Promise + readMetadata( + context: CopilotExecutionContext | undefined, + target: OwnedFileTarget + ): Promise<{ id: string; name: string; owner: EditableFileOwner }> + readChatMetadata(ingress: ChatFileIngress, target: OwnedFileTarget): Promise + capabilities?( + context: CopilotExecutionContext, + owner: FileOwner + ): Promise +} diff --git a/apps/sim/lib/mothership/file-owners/workspace.test.ts b/apps/sim/lib/mothership/file-owners/workspace.test.ts new file mode 100644 index 00000000000..5888773d115 --- /dev/null +++ b/apps/sim/lib/mothership/file-owners/workspace.test.ts @@ -0,0 +1,37 @@ +/** @vitest-environment node */ +import { + readWorkspaceFileMetadataMock, + readWorkspaceFileMetadataMockFns, +} from '@sim/testing/mocks/read-workspace-file-metadata.mock' +import { beforeEach, expect, it, vi } from 'vitest' + +vi.mock( + '@/lib/workspace-files/application/read-workspace-file-metadata', + () => readWorkspaceFileMetadataMock +) + +import { workspaceFileOwnerAdapter } from '@/lib/mothership/file-owners/workspace' + +const readMetadata = readWorkspaceFileMetadataMockFns.mockReadWorkspaceFileMetadata +const owner = { entityType: 'workspace' as const, entityId: 'workspace' } +const ingress = { userId: 'user', workspaceId: 'workspace', chatId: 'chat' } + +beforeEach(() => { + readMetadata.mockReset() +}) + +it('places a workspace file under its folder path', async () => { + readMetadata.mockResolvedValue({ file: { id: 'file', name: 'notes.md', folderPath: 'Reports' } }) + const file = await workspaceFileOwnerAdapter.readChatMetadata(ingress, { owner, fileId: 'file' }) + expect(file.path).toBe('files/Reports/notes.md') +}) + +it('refuses a file owned by another workspace before reading its metadata', async () => { + await expect( + workspaceFileOwnerAdapter.readChatMetadata(ingress, { + owner: { entityType: 'workspace', entityId: 'other-workspace' }, + fileId: 'file', + }) + ).rejects.toMatchObject({ code: 'not_found' }) + expect(readMetadata).not.toHaveBeenCalled() +}) diff --git a/apps/sim/lib/mothership/file-owners/workspace.ts b/apps/sim/lib/mothership/file-owners/workspace.ts new file mode 100644 index 00000000000..11a788bf38d --- /dev/null +++ b/apps/sim/lib/mothership/file-owners/workspace.ts @@ -0,0 +1,59 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { executeCopilotFileUseCase } from '@/lib/mothership/application/execute-file-use-case' +import { requireTrustedCopilotExecutionContext } from '@/lib/mothership/auth/application-delegation' +import { createCopilotChatFilePrincipal } from '@/lib/mothership/auth/file-delegation' +import type { CopilotFileOwnerAdapter } from '@/lib/mothership/file-owners/types' +import { workspaceFileVfsPath } from '@/lib/uploads/contexts/workspace' +import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/read-workspace-file-metadata' +import { fileOwnerVfsPath } from '@/lib/workspace-files/owner-paths' + +export const workspaceFileOwnerAdapter: CopilotFileOwnerAdapter = { + invocationScope: 'workspace', + async readChatMetadata(ingress, target) { + if (ingress.workspaceId !== target.owner.entityId) { + throw new OrchestrationError('not_found', 'File owner not found in this invocation') + } + const principal = createCopilotChatFilePrincipal({ + userId: ingress.userId, + workspaceId: target.owner.entityId, + chatId: ingress.chatId, + }) + const { file } = await readWorkspaceFileMetadata.execute({ + principal, + input: { fileId: target.fileId, assertedWorkspaceId: target.owner.entityId }, + }) + return { + id: file.id, + name: file.name, + owner: { entityType: 'workspace', entityId: target.owner.entityId }, + path: fileOwnerVfsPath(target.owner, workspaceFileVfsPath(file)), + } + }, + async executeCli(request, context, owner) { + if (request.workspaceId !== undefined && request.workspaceId !== owner.entityId) { + throw new OrchestrationError( + 'validation', + 'Workspace file owner does not match the invocation target' + ) + } + const { executeWorkspaceCliRequest } = await import('@/lib/mothership/agent-cli/workspace') + return executeWorkspaceCliRequest(request, context, owner.entityId) + }, + async readMetadata(context, target) { + const trusted = requireTrustedCopilotExecutionContext(context) + if (trusted.workspaceId !== target.owner.entityId) { + throw new OrchestrationError('not_found', 'File owner not found in this invocation') + } + const { file } = await executeCopilotFileUseCase( + context, + readWorkspaceFileMetadata, + { fileId: target.fileId, assertedWorkspaceId: target.owner.entityId }, + { fileId: target.fileId } + ) + return { + id: target.fileId, + name: file.name, + owner: { entityType: 'workspace', entityId: target.owner.entityId }, + } + }, +} diff --git a/apps/sim/lib/mothership/generated/agent-cli.ts b/apps/sim/lib/mothership/generated/agent-cli.ts index 1377e0687fe..cfca0e3f2c5 100644 --- a/apps/sim/lib/mothership/generated/agent-cli.ts +++ b/apps/sim/lib/mothership/generated/agent-cli.ts @@ -11,8 +11,10 @@ */ import { z } from "zod"; +import { FileOperationOwner } from "./file-owner"; import { ArtifactObservations } from "./observations"; import { ResourceChanges } from "./resources"; +import { fileCopyDestinationSchema, fileCopySourceSchema } from "./sim-file-copy.generated"; /** Relative sink paths resolve under the chat workbench home. */ export const AgentCliSandboxFileSink = z.object({ @@ -48,6 +50,7 @@ export const AgentCliServiceInvocation = z.object({ kind: z.literal("service"), name: z.enum([ "list_workspaces", + "list_user_projects", "search_workspace", "read_document", "settings", @@ -62,20 +65,38 @@ export const AgentCliServiceInvocation = z.object({ }); export type AgentCliServiceInvocation = z.infer; +export const AgentCliFileCopyInvocation = z.strictObject({ + kind: z.literal("file-copy"), + argv: AgentCliCliInvocation.shape.argv, + source: fileCopySourceSchema, + destination: fileCopyDestinationSchema, +}); +export type AgentCliFileCopyInvocation = z.infer; + export const AgentCliInvocation = z.discriminatedUnion("kind", [ AgentCliCliInvocation, + AgentCliFileCopyInvocation, AgentCliAugmentationInvocation, AgentCliStdoutInvocation, AgentCliServiceInvocation, ]); export type AgentCliInvocation = z.infer; -export const AgentCliRequest = z.object({ - workspaceId: z.uuid().optional(), - invocation: AgentCliInvocation, - sink: AgentCliSandboxFileSink.optional(), - curate: z.enum(["block", "knowledge-documents"]).optional(), -}); +export const AgentCliRequest = z + .object({ + fileOwner: FileOperationOwner.optional(), + workspaceId: z.uuid().optional(), + invocation: AgentCliInvocation, + sink: AgentCliSandboxFileSink.optional(), + curate: z.enum(["block", "knowledge-documents"]).optional(), + }) + .superRefine((request, context) => { + if (request.invocation.kind === "file-copy" && (request.fileOwner || request.workspaceId)) + context.addIssue({ + code: "custom", + message: "File copy carries both owners in its invocation; omit envelope fileOwner and workspaceId.", + }); + }); export type AgentCliRequest = z.infer; export const AgentCliRawResult = z.object({ diff --git a/apps/sim/lib/mothership/generated/file-owner.ts b/apps/sim/lib/mothership/generated/file-owner.ts new file mode 100644 index 00000000000..1624ceb2754 --- /dev/null +++ b/apps/sim/lib/mothership/generated/file-owner.ts @@ -0,0 +1,32 @@ +// GENERATED — do not edit. Source of truth: mothership worker packages/contracts/src/file-owner.ts +// Regenerate with `bun run contracts:sync` in the worker. + +import { z } from "zod"; + +export const FileOwner = z.strictObject({ + entityType: z.enum(["workspace", "project", "organization", "user"]), + entityId: z.string().min(1).max(200), +}); +export type FileOwner = z.infer; + +export const FileOperationOwner = FileOwner.extend({ entityType: z.enum(["workspace", "project"]) }); +export type FileOperationOwner = z.infer; + +export const FileProjectContext = z.strictObject({ + id: z.string().min(1).max(200), + name: z.string().min(1).max(500), +}); + +export const FileOwnerCapability = z.strictObject({ + owner: FileOwner, + canRead: z.boolean(), + canWrite: z.boolean(), +}); + +/** Sim-computed discovery hints; each operation still requires current Sim authorization. */ +export const FileOwnerContext = z.strictObject({ + fileOwnerProtocolVersion: z.literal(1), + project: FileProjectContext.nullable().optional(), + fileOwnerCapabilities: z.array(FileOwnerCapability).max(1000).default([]), +}); +export type FileOwnerContext = z.infer; diff --git a/apps/sim/lib/mothership/generated/observations.ts b/apps/sim/lib/mothership/generated/observations.ts index 85967464e66..63839b27da6 100644 --- a/apps/sim/lib/mothership/generated/observations.ts +++ b/apps/sim/lib/mothership/generated/observations.ts @@ -2,6 +2,7 @@ // Regenerate with `bun run contracts:sync` in the worker. import { z } from "zod"; +import { ResourceAddress } from "./resources"; export const ObservationMediaType = z.enum([ "image/png", @@ -15,6 +16,7 @@ export const ArtifactObservation = z.object({ mediaType: ObservationMediaType, data: z.string().min(1).max(11_200_000), resourceId: z.string().optional(), + resource: ResourceAddress.optional(), pageCount: z.number().int().min(1).max(20).optional(), }); export type ArtifactObservation = z.infer; diff --git a/apps/sim/lib/mothership/generated/protocol.ts b/apps/sim/lib/mothership/generated/protocol.ts index 8705d4f1c96..fe803f93934 100644 --- a/apps/sim/lib/mothership/generated/protocol.ts +++ b/apps/sim/lib/mothership/generated/protocol.ts @@ -1,6 +1,7 @@ // GENERATED — do not edit. Source of truth: mothership worker packages/contracts/src/protocol.ts // Regenerate with `bun run contracts:sync` in the worker. +import { FileOwnerContext } from "./file-owner"; /** * The sim⇄worker wire protocol surface — THE shared source of truth (P3, S31). * @@ -18,6 +19,7 @@ import { z } from "zod"; import { AssistantImage, AssistantSearch, AssistantSearchLevel } from "./assistant"; import { IntegrationCatalogContext } from "./integration-catalog"; +import { ResourceAddress } from "./resources"; import { SimConnection } from "./sim-transport"; export const PROTOCOL_VERSION = 2; @@ -180,6 +182,7 @@ export const ChatPayloadSchema = z content: z.string(), tag: z.string().optional(), path: z.string().optional(), + resource: ResourceAddress.optional(), }), ) .default([]), @@ -195,6 +198,9 @@ export const ChatPayloadSchema = z modelSelection: ModelSelectionSchema.optional(), /** Workspace orientation (contracts ChatRequest.inventory): names and ids per world. */ inventory: WorkspaceInventorySchema.optional(), + fileOwnerProtocolVersion: FileOwnerContext.shape.fileOwnerProtocolVersion.optional(), + project: FileOwnerContext.shape.project, + fileOwnerCapabilities: FileOwnerContext.shape.fileOwnerCapabilities.optional(), }) .superRefine((value, ctx) => { if ( @@ -297,6 +303,9 @@ export const TOOL_DEADLINE_CAPABILITY = "tool-deadline"; /** POST /api/mothership — the chat request sim sends. */ export interface ChatRequest extends StreamResponseReceipt { + fileOwnerProtocolVersion?: 1 | undefined; + project?: FileOwnerContext["project"] | undefined; + fileOwnerCapabilities?: FileOwnerContext["fileOwnerCapabilities"] | undefined; desktop?: DesktopContext | undefined; effort?: "none" | "low" | "medium" | "high" | "xhigh" | "max" | undefined; modelSelection?: ModelSelection | undefined; @@ -372,6 +381,8 @@ export interface ChatContextItem { content: string; tag?: string | undefined; path?: string | undefined; + /** Requested resource identity; Sim authorizes access independently on every read. */ + resource?: ResourceAddress | undefined; } /** POST /api/tools/resume — deferred tool results. */ diff --git a/apps/sim/lib/mothership/generated/resources.ts b/apps/sim/lib/mothership/generated/resources.ts index 06090025bee..db50ef5b859 100644 --- a/apps/sim/lib/mothership/generated/resources.ts +++ b/apps/sim/lib/mothership/generated/resources.ts @@ -2,6 +2,7 @@ // Regenerate with `bun run contracts:sync` in the worker. import { z } from "zod"; +import { FileOwner } from "./file-owner"; import { nativeSearchQueriesSchema, workspaceKnowledgeSearchDataSchema, @@ -41,6 +42,7 @@ export const SearchResource = z.object({ export type SearchResource = z.infer; export const ResourceAddress = z.object({ + owner: FileOwner.optional(), workspaceId: z.uuid().optional(), workspaceName: z.string().optional(), type: ResourceType, @@ -91,6 +93,7 @@ export const ResourceChange = z.discriminatedUnion("op", [ resource: z.union([ z.object({ type: ResourceType, + owner: FileOwner.optional(), workspaceId: z.uuid().optional(), id: z.string().regex(/\S/).optional(), }), diff --git a/apps/sim/lib/mothership/generated/sim-assistant-tools.generated.ts b/apps/sim/lib/mothership/generated/sim-assistant-tools.generated.ts index 2fe72fdb685..742a510aea5 100644 --- a/apps/sim/lib/mothership/generated/sim-assistant-tools.generated.ts +++ b/apps/sim/lib/mothership/generated/sim-assistant-tools.generated.ts @@ -367,8 +367,22 @@ export const listWorkspacesInputSchema = z.object({ cursor: z.string().uuid().optional().describe('Copy nextCursor from the preceding page.'), }) +/** Project discovery accepts pagination only; the conversation determines the accessible scope. */ +export const listUserProjectsInputSchema = z.object({ + cursor: z.string().min(1).optional(), + limit: z.number().int().min(1).max(100).default(50), +}) +export type ListUserProjectsInput = z.output + /** Executor identity belongs to Sim; model-facing profile guidance belongs to its caller. */ export const assistantToolContracts = [ + { + id: 'list_user_projects', + route: 'sim', + description: + 'List accessible Projects within this conversation, with visible environments and current capabilities. Follow nextCursor for more results; discovery does not grant file write access.', + inputSchema: listUserProjectsInputSchema, + }, { id: 'list_workspaces', route: 'sim', diff --git a/apps/sim/lib/mothership/generated/sim-file-copy.generated.ts b/apps/sim/lib/mothership/generated/sim-file-copy.generated.ts new file mode 100644 index 00000000000..119488ea939 --- /dev/null +++ b/apps/sim/lib/mothership/generated/sim-file-copy.generated.ts @@ -0,0 +1,44 @@ +// GENERATED — do not edit. Source: Sim apps/sim/lib/api/contracts/file-copy-input.ts +// Regenerate with `bun run generate:cli-inventory` in the worker. + +import { z } from 'zod' +const MAX_WORKSPACE_FILE_BULK_REQUEST_IDS = 1000 + +const copyIdSchema = z.string().trim().min(1).max(200) +const copyOwnerSchema = z.strictObject({ + entityType: z.enum(['workspace', 'project']).describe('Resource scope that owns these files.'), + entityId: z.string().min(1).max(200).describe('Identifier of the owning workspace or Project.'), +}) +const copyIdsSchema = z + .array(copyIdSchema) + .max(MAX_WORKSPACE_FILE_BULK_REQUEST_IDS, 'Too many selected file items') + .refine((ids) => new Set(ids).size === ids.length, 'Selected identifiers must be unique') + .default([]) + +export const fileCopySourceSchema = z + .strictObject({ + owner: copyOwnerSchema.describe('Owner from which to read the selected files and folders.'), + fileIds: copyIdsSchema.describe('Identifiers of individual files to copy.'), + folderIds: copyIdsSchema.describe( + 'Identifiers of folders to copy with their active descendants and files.' + ), + }) + .refine((source) => source.fileIds.length + source.folderIds.length > 0, { + message: 'Select at least one file or folder to copy', + }) +export type FileCopySource = z.output + +export const fileCopyDestinationSchema = z.strictObject({ + owner: copyOwnerSchema.describe('Owner in which to create the copied files and folders.'), + folderId: copyIdSchema + .nullable() + .default(null) + .describe('Existing destination folder identifier. Omit or use null for the owner root.'), +}) +export type FileCopyDestination = z.output + +export const fileCopyInputSchema = z.strictObject({ + source: fileCopySourceSchema.describe('Selection to read under the source owner.'), + destination: fileCopyDestinationSchema.describe('Destination requiring file write access.'), +}) +export type FileCopyInput = z.output diff --git a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts index 6a0cc58e12e..2ece49b3f3c 100644 --- a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts @@ -8,6 +8,7 @@ export interface ToolCatalogEntry { description?: | 'Create a workspace in the conversation’s organization under the current user’s workspace-creation policy. Returns its ID for subsequent explicitly workspace-scoped commands. Includes a starter workflow unless skipDefaultWorkflow is true.' | 'Discover and configure organization Search sources. list/get return accessible sources and indexing status; providers returns available integration approvals; approve changes a provider approval when authorized; setup returns the existing connection UI for the user to complete. Put the returned setupUrl in a clickable Markdown link at the end of the reply. Setup does not mean connected or indexed. Use search_workspace and read_document to retrieve source content.' + | 'List accessible Projects within this conversation, with visible environments and current capabilities. Follow nextCursor for more results; discovery does not grant file write access.' | 'List currently accessible workspaces with roles and explicit capability restrictions. Bulk results report copilotAllowed and deniedCapabilities; exact workspaceId returns the full capability map. Omitted restrictions never authorize an operation.' | 'Read and manage account, organization, and workspace settings. list finds sections; get returns current values, updateSchema and operation names; describe returns one operation’s exact input schema; update changes narrow preferences; execute performs a listed operation; open returns the existing user setup flow. When user setup is needed, put the returned setupUrl in a clickable Markdown link at the end of the reply. Workspace resources retain their CLI commands. Account is the acting user; organization is the conversation’s organization. Every operation checks current permissions and entitlements.' | 'Read and save the selected workspace’s single dashboard, validated YAML over live tables. Load the create-dashboard skill for the schema. get returns content and revision, or nulls when the workspace has no dashboard yet; set with no revision creates it. Replacing an existing dashboard requires expectedRevision from get, so a concurrent edit is never overwritten. Use open_resource with type dashboard to show the result.' @@ -84,6 +85,7 @@ export interface ToolCatalogEntry { | 'knowledge' | 'list_deployment_versions' | 'list_integration_tools' + | 'list_user_projects' | 'list_workspace_mcp_servers' | 'list_workspaces' | 'load_deployment' @@ -7922,6 +7924,21 @@ export const SearchSources: ToolCatalogEntry = { }, } +export const ListUserProjects: ToolCatalogEntry = { + id: 'list_user_projects', + description: + 'List accessible Projects within this conversation, with visible environments and current capabilities. Follow nextCursor for more results; discovery does not grant file write access.', + route: 'sim', + parameters: { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { + cursor: { type: 'string', minLength: 1 }, + limit: { default: 50, type: 'integer', minimum: 1, maximum: 100 }, + }, + }, +} + export const FfmpegOperation = { overlayAudio: 'overlay_audio', mixAudio: 'mix_audio', @@ -8495,4 +8512,5 @@ export const TOOL_CATALOG: Record = { [Workspaces.id]: Workspaces, [Settings.id]: Settings, [SearchSources.id]: SearchSources, + [ListUserProjects.id]: ListUserProjects, } diff --git a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts index 369aee5b4cf..7b184bd3fd3 100644 --- a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts @@ -8094,4 +8094,23 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, + list_user_projects: { + parameters: { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { + cursor: { + type: 'string', + minLength: 1, + }, + limit: { + default: 50, + type: 'integer', + minimum: 1, + maximum: 100, + }, + }, + }, + resultSchema: undefined, + }, } diff --git a/apps/sim/lib/mothership/request/application/recover-stream.ts b/apps/sim/lib/mothership/request/application/recover-stream.ts index c5605556a3e..62125711caf 100644 --- a/apps/sim/lib/mothership/request/application/recover-stream.ts +++ b/apps/sim/lib/mothership/request/application/recover-stream.ts @@ -21,6 +21,7 @@ import { hasLiveSimToolExecution, revokeExhaustedSimToolExecutions, } from '@/lib/mothership/async-runs/repository' +import { createCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' import { defineAuthorizedChatUseCase } from '@/lib/mothership/chat/application/authorized-chat-use-case' import { resolveOwnedChatContext } from '@/lib/mothership/chat/application/context' import { buildOnComplete, buildOnError } from '@/lib/mothership/chat/completion' @@ -29,7 +30,10 @@ import { claimRunController, planRecovery, } from '@/lib/mothership/request/lifecycle/controller-ownership' -import { StreamRecoveryConfigSchema } from '@/lib/mothership/request/lifecycle/recovery-config' +import { + RESOURCE_AUTHORING_VERSION, + StreamRecoveryConfigSchema, +} from '@/lib/mothership/request/lifecycle/recovery-config' import { createSSEStream } from '@/lib/mothership/request/lifecycle/start' import { isTerminalStreamStatus } from '@/lib/mothership/request/session' import { @@ -272,6 +276,10 @@ async function recoverStreamRun(context: StreamRunContext, admission?: AbortSign executionId: run.executionId, workflowId: run.workflowId ?? undefined, goRoute: config.data.goRoute, + copilotResourceAdmission: + config.data.resourceAuthoringVersion === RESOURCE_AUTHORING_VERSION + ? createCopilotResourceAdmission({ userId, invocation: { kind: 'chat', chatId } }) + : undefined, billingAttribution, userPermission: userPermission ?? undefined, interactive: true, diff --git a/apps/sim/lib/mothership/request/lifecycle/file-owner-protocol.integration.ts b/apps/sim/lib/mothership/request/lifecycle/file-owner-protocol.integration.ts new file mode 100644 index 00000000000..3dfc4a911bc --- /dev/null +++ b/apps/sim/lib/mothership/request/lifecycle/file-owner-protocol.integration.ts @@ -0,0 +1,78 @@ +import { createServer } from 'node:http' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { supportsFileOwnerProtocol } from '@/lib/mothership/request/lifecycle/file-owner-protocol' + +let flakyHits = 0 + +const worker = createServer((request, response) => { + switch (request.url) { + case '/flaky/healthz': + flakyHits++ + if (flakyHits === 1) response.writeHead(503).end() + else response.writeHead(200, { 'X-Mothership-File-Owner-Protocol': '1' }).end() + return + case '/supported/healthz': + case '/nested/worker/healthz': + response.writeHead(200, { 'X-Mothership-File-Owner-Protocol': '1' }).end() + return + case '/future/healthz': + response.writeHead(200, { 'X-Mothership-File-Owner-Protocol': '2' }).end() + return + case '/malformed/healthz': + response.writeHead(200, { 'X-Mothership-File-Owner-Protocol': '01' }).end() + return + case '/unhealthy/healthz': + response.writeHead(503, { 'X-Mothership-File-Owner-Protocol': '1' }).end() + return + case '/redirect/healthz': + response.writeHead(302, { location: '/supported/healthz' }).end() + return + case '/broken/healthz': + request.socket.destroy() + return + case '/unresponsive/healthz': + return + default: + response.writeHead(200).end() + } +}) + +let endpoint: string + +beforeAll(async () => { + await new Promise((resolve) => worker.listen(0, '127.0.0.1', resolve)) + const address = worker.address() + if (!address || typeof address === 'string') throw new Error('Worker fixture failed to bind') + endpoint = `http://127.0.0.1:${address.port}` +}) + +afterAll( + () => + new Promise((resolve, reject) => { + worker.close((error) => (error ? reject(error) : resolve())) + worker.closeAllConnections() + }) +) + +describe('file-owner worker negotiation over HTTP', () => { + it.each([ + ['supported', true], + ['nested/worker/', true], + ['old', false], + ['future', false], + ['malformed', false], + ['unhealthy', false], + ['redirect', false], + ['broken', false], + ['unresponsive', false], + ] as const)('accepts only the healthy, exact protocol at %s', async (path, supported) => { + expect(await supportsFileOwnerProtocol(`${endpoint}/${path}`)).toBe(supported) + }) + + it('retries a failed probe on the next call and remembers only the healthy answer', async () => { + expect(await supportsFileOwnerProtocol(`${endpoint}/flaky`)).toBe(false) + expect(await supportsFileOwnerProtocol(`${endpoint}/flaky`)).toBe(true) + expect(await supportsFileOwnerProtocol(`${endpoint}/flaky/`)).toBe(true) + expect(flakyHits).toBe(2) + }) +}) diff --git a/apps/sim/lib/mothership/request/lifecycle/file-owner-protocol.ts b/apps/sim/lib/mothership/request/lifecycle/file-owner-protocol.ts new file mode 100644 index 00000000000..451627422dc --- /dev/null +++ b/apps/sim/lib/mothership/request/lifecycle/file-owner-protocol.ts @@ -0,0 +1,32 @@ +import { LRUCache } from 'lru-cache' + +const PROTOCOL_PROBE_TTL_MS = 30_000 + +/** + * Worker protocol support keyed by base URL, shared by turn hints and sandbox leases. Only an OK + * health response is cached; a failed probe rejects, so an outage reads as unsupported for that + * call without being remembered. + */ +const protocolSupport = new LRUCache({ + max: 64, + ttl: PROTOCOL_PROBE_TTL_MS, + fetchMethod: async (baseURL) => { + // boundary-raw-fetch: the selected worker advertises its live file-owner protocol in health headers. + const response = await fetch(`${baseURL}/healthz`, { + method: 'GET', + redirect: 'error', + signal: AbortSignal.timeout(2_000), + }) + if (!response.ok) throw new Error(`Worker health probe returned ${response.status}`) + return response.headers.get('X-Mothership-File-Owner-Protocol') === '1' + }, +}) + +/** Negotiates against the selected worker; redirects cannot establish another peer's support. */ +export async function supportsFileOwnerProtocol(baseURL: string): Promise { + try { + return (await protocolSupport.fetch(baseURL.replace(/\/$/, ''))) === true + } catch { + return false + } +} diff --git a/apps/sim/lib/mothership/request/lifecycle/recovery-config.test.ts b/apps/sim/lib/mothership/request/lifecycle/recovery-config.test.ts index cf4ea5ad5f3..a00a2fba458 100644 --- a/apps/sim/lib/mothership/request/lifecycle/recovery-config.test.ts +++ b/apps/sim/lib/mothership/request/lifecycle/recovery-config.test.ts @@ -1,5 +1,12 @@ -import { describe, expect, it } from 'vitest' -import { StreamRecoveryConfigSchema } from '@/lib/mothership/request/lifecycle/recovery-config' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { describe, expect, it, vi } from 'vitest' +import { createCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' +import { + StreamRecoveryConfigSchema, + streamRecoveryConfig, +} from '@/lib/mothership/request/lifecycle/recovery-config' + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) const identity = { userId: 'actor', @@ -84,3 +91,28 @@ it('preserves the explicit Plan wire mode on recovery', () => { .mode ).toBe('plan') }) + +describe('recovery marker written by an admitted turn', () => { + const options = { + interactive: true, + userId: identity.userId, + chatId: identity.chatId, + goRoute: '/api/mothership', + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: identity.userId, + invocation: { kind: 'chat', chatId: identity.chatId }, + }), + } + + it('stays off a recovery config older builds must still parse while Projects are off', async () => { + featureFlagsMockFns.mockIsFeatureEnabled.mockResolvedValue(false) + const recovery = await streamRecoveryConfig(options, workspaceRequest) + expect(recovery).not.toHaveProperty('resourceAuthoringVersion') + }) + + it('restores authoring admission on recovery while Projects are on', async () => { + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation(async (flag) => flag === 'projects') + const recovery = await streamRecoveryConfig(options, workspaceRequest) + expect(recovery?.resourceAuthoringVersion).toBe(1) + }) +}) diff --git a/apps/sim/lib/mothership/request/lifecycle/recovery-config.ts b/apps/sim/lib/mothership/request/lifecycle/recovery-config.ts index 00b3f20ff43..e6a4334178f 100644 --- a/apps/sim/lib/mothership/request/lifecycle/recovery-config.ts +++ b/apps/sim/lib/mothership/request/lifecycle/recovery-config.ts @@ -1,7 +1,9 @@ import { z } from 'zod' +import { hasCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' import { COPILOT_REQUEST_MODES } from '@/lib/mothership/constants' import { ChatPayloadSchema } from '@/lib/mothership/generated/protocol' import type { CopilotLifecycleOptions } from '@/lib/mothership/request/lifecycle/run' +import { isProjectsEnabled } from '@/lib/projects/rollout.server' export const BillingAdmissionSchema = z .object({ @@ -19,9 +21,27 @@ export const DurableChatRequestSchema = ChatPayloadSchema.safeExtend({ }).transform( ({ byokApiKey, delegationToken, receivedTextChars, receivedActivity, ...request }) => request ) +/** + * Marks a durable turn admitted to author resources, so recovery restores the admission. Every + * writer and the recovery reader share this one value. + */ +export const RESOURCE_AUTHORING_VERSION = 1 + +/** + * The marker an admitted turn stores for recovery. Builds before it parse recovery strictly and + * would refuse to recover the run, and only Project authoring reads it, so it is written only + * while Projects are on. + */ +export async function durableResourceAuthoringVersion( + admitted: boolean +): Promise { + return admitted && (await isProjectsEnabled()) ? RESOURCE_AUTHORING_VERSION : undefined +} + export const StreamRecoveryConfigSchema = z .object({ kind: z.literal('interactive_stream'), + resourceAuthoringVersion: z.literal(RESOURCE_AUTHORING_VERSION).optional(), billingAdmission: BillingAdmissionSchema.optional(), request: DurableChatRequestSchema, goRoute: z.enum(['/api/mothership', '/api/copilot']), @@ -41,11 +61,20 @@ export const StreamRecoveryConfigSchema = z } ) -export function streamRecoveryConfig( +export async function streamRecoveryConfig( options: CopilotLifecycleOptions, request: Record ) { if (options.interactive !== true) return undefined + const resourceAuthoringVersion = await durableResourceAuthoringVersion( + hasCopilotResourceAdmission({ + ...options, + ...options.executionContext, + copilotToolExecution: true, + copilotResourceAdmission: + options.executionContext?.copilotResourceAdmission ?? options.copilotResourceAdmission, + }) + ) return StreamRecoveryConfigSchema.parse({ kind: 'interactive_stream', request, @@ -53,5 +82,6 @@ export function streamRecoveryConfig( clientToolPickupExpected: options.clientToolPickupExpected ?? true, userTimezone: options.executionContext?.userTimezone, requestMode: options.executionContext?.requestMode, + ...(resourceAuthoringVersion ? { resourceAuthoringVersion } : {}), }) } diff --git a/apps/sim/lib/mothership/request/lifecycle/run.ts b/apps/sim/lib/mothership/request/lifecycle/run.ts index df28b660535..6b2dc60a264 100644 --- a/apps/sim/lib/mothership/request/lifecycle/run.ts +++ b/apps/sim/lib/mothership/request/lifecycle/run.ts @@ -16,6 +16,7 @@ import { } from '@/lib/billing/core/attribution' import { env } from '@/lib/core/config/env' import { isCopilotToolPermissionsEnabled, isHosted } from '@/lib/core/config/env-flags' +import { withFileOwnerContext } from '@/lib/mothership/application/file-owner-context' import { SIM_TOOL_EXECUTION_LEASE_SECONDS } from '@/lib/mothership/async-runs/execution-lease' import type { AsyncCompletionSignal } from '@/lib/mothership/async-runs/lifecycle' import { @@ -23,6 +24,7 @@ import { getChatViewDesktopLeaseRemainingMs, updateRunStatus, } from '@/lib/mothership/async-runs/repository' +import type { CopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' import { CLIENT_TOOL_RESULT_TIMEOUT_MS, TOOL_WATCHDOG_RESUME_GRACE_MS, @@ -278,6 +280,7 @@ function resultContent(context: StreamingContext, options: CopilotLifecycleOptio export interface CopilotLifecycleOptions extends OrchestratorOptions { /** Internal benchmark runner policy, independent of browser and model payloads. */ benchmark?: 'plan' | 'tool-free' | 'distill' | 'reconstruct' | 'resolve' + copilotResourceAdmission?: CopilotResourceAdmission /** Trusted entry point for Search metering; never read from model arguments. */ searchSurface?: 'copilot' | 'slack' mcpBlockId?: string @@ -494,6 +497,8 @@ export async function runCopilotLifecycle( } execContext.messageId = payloadMsgId execContext.benchmark = options.benchmark + if (lifecycleOptions.copilotResourceAdmission) + execContext.copilotResourceAdmission = lifecycleOptions.copilotResourceAdmission if (options.recovery?.userTimezone) execContext.userTimezone = options.recovery.userTimezone execContext.requestMode = requestMode execContext.searchSurface = lifecycleOptions.searchSurface ?? 'copilot' @@ -1231,6 +1236,8 @@ async function runCheckpointLoop( payload = { ...payload, organizationId: lifecycleOrganizationId, chatId: execContext.chatId } } + payload = await withFileOwnerContext(payload, execContext, mothershipBaseURL, initialRoute) + for (;;) { await options.assertControllerOwnership?.() context.streamComplete = false diff --git a/apps/sim/lib/mothership/request/lifecycle/start.ts b/apps/sim/lib/mothership/request/lifecycle/start.ts index d8aa60e17f8..189a252c2af 100644 --- a/apps/sim/lib/mothership/request/lifecycle/start.ts +++ b/apps/sim/lib/mothership/request/lifecycle/start.ts @@ -339,7 +339,7 @@ export function createSSEStream(params: StreamingOrchestrationParams): ReadableS ...(params.admittedRun?.requestContext ?? {}), requestId, controllerToken: lease?.value, - recovery: streamRecoveryConfig(orchestrateOptions, requestPayload), + recovery: await streamRecoveryConfig(orchestrateOptions, requestPayload), } try { if (chatId && !orchestrateOptions.recovery) { diff --git a/apps/sim/lib/mothership/request/session/sse.ts b/apps/sim/lib/mothership/request/session/sse.ts index 99b8b58fe12..a1a92dfcf0e 100644 --- a/apps/sim/lib/mothership/request/session/sse.ts +++ b/apps/sim/lib/mothership/request/session/sse.ts @@ -1,9 +1,17 @@ +import { isRecordLike } from '@sim/utils/object' import { SSE_HEADERS } from '@/lib/core/utils/sse' +import { presentChatResourceForBrowser } from '@/lib/mothership/resources/presentation' const encoder = new TextEncoder() export function encodeSSEEnvelope(envelope: unknown): Uint8Array { - return encoder.encode(`data: ${JSON.stringify(envelope)}\n\n`) + let presented = envelope + if (isRecordLike(envelope) && envelope.type === 'resource' && isRecordLike(envelope.payload)) { + const resource = presentChatResourceForBrowser(envelope.payload.resource) + if (resource !== envelope.payload.resource) + presented = { ...envelope, payload: { ...envelope.payload, resource } } + } + return encoder.encode(`data: ${JSON.stringify(presented)}\n\n`) } export const SSE_RESPONSE_HEADERS = { diff --git a/apps/sim/lib/mothership/request/tools/executor.ts b/apps/sim/lib/mothership/request/tools/executor.ts index 361d143963e..21d0d593cb8 100644 --- a/apps/sim/lib/mothership/request/tools/executor.ts +++ b/apps/sim/lib/mothership/request/tools/executor.ts @@ -16,6 +16,7 @@ import { markAsyncToolRunning, upsertAsyncToolCall, } from '@/lib/mothership/async-runs/repository' +import { hasCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' import { withToolServiceMeter } from '@/lib/mothership/billing/service-meter' import { CLIENT_TOOL_RESULT_TIMEOUT_MS, @@ -412,6 +413,9 @@ async function executeToolWithWatchdog( chatId: toolContext.chatId, workspaceId: toolContext.workspaceId, organizationId: toolContext.organizationId, + resourceAdmitted: + toolContext.requestMode !== 'assistant' && hasCopilotResourceAdmission(toolContext), + mothershipBaseURL: toolContext.mothershipBaseURL, }, signal, onEvent, diff --git a/apps/sim/lib/mothership/request/tools/resolved-secret-result.ts b/apps/sim/lib/mothership/request/tools/resolved-secret-result.ts index 22ebd083a8b..4a40c1f7c84 100644 --- a/apps/sim/lib/mothership/request/tools/resolved-secret-result.ts +++ b/apps/sim/lib/mothership/request/tools/resolved-secret-result.ts @@ -61,6 +61,7 @@ const RESERVED_DISCLOSURE_KEYS = new Set(['resultWithheld', 'withheldReason', 'e const UNKNOWN_INPUT_PROVENANCE_REASONS = new Set([ 'mounted-file-provenance-unavailable', 'workspace-file-provenance-unknown', + 'project-file-provenance-unavailable', 'file-source-unidentified', 'table-snapshot-unsafe-for-mount', 'table-result-provenance-unavailable', diff --git a/apps/sim/lib/mothership/request/tools/resources.test.ts b/apps/sim/lib/mothership/request/tools/resources.test.ts index 659f9572895..e02f57bf93e 100644 --- a/apps/sim/lib/mothership/request/tools/resources.test.ts +++ b/apps/sim/lib/mothership/request/tools/resources.test.ts @@ -99,7 +99,15 @@ describe('handleResourceSideEffects', () => { } ) it.each([ - { chat: 'organization', organizationId: 'org', expected: { workspaceId: 'workspace-a' } }, + { + chat: 'organization', + organizationId: 'org', + /** The address stays beside the owner: browsers open and close the tab by `workspaceId`. */ + expected: { + workspaceId: 'workspace-a', + owner: { entityType: 'workspace', entityId: 'workspace-a' }, + }, + }, { chat: 'workspace', organizationId: undefined, expected: {} }, ])( 'addresses extracted exports in a $chat chat like its open tabs', diff --git a/apps/sim/lib/mothership/request/tools/resources.ts b/apps/sim/lib/mothership/request/tools/resources.ts index 2a3ded3cf7b..63ffa35f094 100644 --- a/apps/sim/lib/mothership/request/tools/resources.ts +++ b/apps/sim/lib/mothership/request/tools/resources.ts @@ -1,6 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { toRecord } from '@sim/utils/object' +import { omit, toRecord } from '@sim/utils/object' import { MothershipStreamV1EventType, MothershipStreamV1ResourceOp, @@ -20,6 +20,11 @@ import { } from '@/lib/mothership/resources/persistence' import { searchResultFromToolResult } from '@/lib/mothership/resources/search-tool-result' import { changeStoredChatResources } from '@/lib/mothership/resources/store' +import { + getChatResourceWorkspaceId, + hasValidChatResourceOwner, + normalizeChatResource, +} from '@/lib/mothership/resources/types' const logger = createLogger('CopilotResourceEffects') @@ -66,7 +71,13 @@ export async function handleResourceSideEffects( if (hasDeleteCapability(toolName)) { const deleted = extractDeletedResourcesFromToolResult(toolName, params, result.output).map( - (resource) => ({ ...resource, ...(workspaceId ? { workspaceId } : {}) }) + (resource) => { + const resourceWorkspaceId = getChatResourceWorkspaceId(resource, workspaceId) + return normalizeChatResource({ + ...omit(resource, ['workspaceId']), + ...(resourceWorkspaceId ? { workspaceId: resourceWorkspaceId } : {}), + }) + } ) const projectedDeleted = extractDeletedResourcesFromToolResult( toolName, @@ -99,6 +110,7 @@ export async function handleResourceSideEffects( type: resource.type, id: resource.id, ...(resource.workspaceId ? { workspaceId: resource.workspaceId } : {}), + ...(resource.owner ? { owner: resource.owner } : {}), title: projected?.title ?? '', }, }, @@ -122,14 +134,18 @@ export async function handleResourceSideEffects( : [] const paired = projectedResources.length === rawResources.length - ? rawResources.map((resource, index) => ({ - ...projectedResources[index], - type: resource.type, - id: resource.id, - ...((resource.workspaceId ?? workspaceId) - ? { workspaceId: resource.workspaceId ?? workspaceId } - : {}), - })) + ? rawResources + .map((resource, index) => { + const resourceWorkspaceId = getChatResourceWorkspaceId(resource, workspaceId) + return normalizeChatResource({ + ...omit(projectedResources[index], ['owner', 'workspaceId']), + type: resource.type, + id: resource.id, + ...(resource.owner ? { owner: resource.owner } : {}), + ...(resourceWorkspaceId ? { workspaceId: resourceWorkspaceId } : {}), + }) + }) + .filter(hasValidChatResourceOwner) : [] const nonTabFileIds = await findNonTabFileIds( paired.flatMap((resource) => (resource.type === 'file' ? [resource.id] : [])) diff --git a/apps/sim/lib/mothership/resources/file-tabs.ts b/apps/sim/lib/mothership/resources/file-tabs.ts index 73cbff282c0..d50b2b7ceb9 100644 --- a/apps/sim/lib/mothership/resources/file-tabs.ts +++ b/apps/sim/lib/mothership/resources/file-tabs.ts @@ -7,13 +7,13 @@ import type { BaseServerTool } from '@/lib/mothership/tools/server/base-tool' const logger = createLogger('FileTabs') -/** Row contexts that open as a file tab: workspace files and chat uploads. */ -const FILE_TAB_CONTEXTS = ['workspace', 'mothership'] +/** Row contexts that open as a file tab: workspace/Project files and chat uploads. */ +const FILE_TAB_CONTEXTS = ['workspace', 'project', 'mothership'] /** * The ids among `fileIds` whose rows must not open as a file tab: any context other than a - * workspace file or chat upload belongs to another resource (a test's source, for one), which - * opens as that resource instead. + * workspace/Project file or chat upload belongs to another resource (a test's source, for + * one), which opens as that resource instead. */ export async function findNonTabFileIds(fileIds: string[]): Promise> { if (fileIds.length === 0) return new Set() diff --git a/apps/sim/lib/mothership/resources/presentation.ts b/apps/sim/lib/mothership/resources/presentation.ts new file mode 100644 index 00000000000..47debfa6e4b --- /dev/null +++ b/apps/sim/lib/mothership/resources/presentation.ts @@ -0,0 +1,16 @@ +import { isRecordLike } from '@sim/utils/object' + +/** Derives the file address understood by cached browsers without changing canonical storage. */ +export function presentChatResourceForBrowser(resource: T): T { + if ( + !isRecordLike(resource) || + (resource.type !== 'file' && resource.type !== 'filefolder') || + !isRecordLike(resource.owner) || + resource.owner.entityType !== 'workspace' || + typeof resource.owner.entityId !== 'string' || + !resource.owner.entityId.trim() || + resource.workspaceId === resource.owner.entityId + ) + return resource + return { ...resource, workspaceId: resource.owner.entityId } +} diff --git a/apps/sim/lib/mothership/resources/store.ts b/apps/sim/lib/mothership/resources/store.ts index d04427d5486..3a858b2c338 100644 --- a/apps/sim/lib/mothership/resources/store.ts +++ b/apps/sim/lib/mothership/resources/store.ts @@ -3,7 +3,8 @@ import { copilotChats, mothershipResourceEffects } from '@sim/db/schema' import { and, eq, isNull, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import { - getChatResourceKey, + chatResourceKeyFor, + hasValidChatResourceOwner, type MothershipResource, type MothershipResourceUpdate, mergeChatResource, @@ -54,7 +55,10 @@ export async function serializeChatResourceWrite( export type ChatResourceChange = | { kind: 'upsert'; resources: MothershipResourceUpdate[] } - | { kind: 'remove'; resources: Pick[] } + | { + kind: 'remove' + resources: Pick[] + } | { kind: 'reorder'; resources: MothershipResource[] } | { kind: 'clear-view'; tableId: string; viewId: string; workspaceId?: string } @@ -64,15 +68,21 @@ export async function changeStoredChatResources( change: ChatResourceChange, effectId?: string ): Promise { + if ( + change.kind !== 'clear-view' && + change.resources.some((resource) => !hasValidChatResourceOwner(resource)) + ) + throw new OrchestrationError('validation', 'Resource owner conflicts with its address') return serializeChatResourceWrite(chatId, () => db.transaction(async (tx) => { await setChatResourceTxTimeouts(tx) const [chat] = await tx - .select({ resources: copilotChats.resources }) + .select({ resources: copilotChats.resources, workspaceId: copilotChats.workspaceId }) .from(copilotChats) .where(and(eq(copilotChats.id, chatId), isNull(copilotChats.deletedAt))) .for('update') if (!chat) throw new OrchestrationError('not_found', 'Chat not found') + const keyOf = chatResourceKeyFor(chat.workspaceId ?? undefined) const existing = sanitizeChatResources(Array.isArray(chat.resources) ? chat.resources : []) if (effectId) { const [applied] = await tx @@ -96,24 +106,15 @@ export async function changeStoredChatResources( return unPinned }) } else if (change.kind === 'remove') { - resources = existing.filter( - (resource) => - !change.resources.some( - (removed) => - removed.type === resource.type && - removed.workspaceId === resource.workspaceId && - (removed.id === resource.id || - removed.type === 'browser' || - removed.type === 'terminal') - ) - ) + const removedKeys = new Set(change.resources.map(keyOf)) + resources = existing.filter((resource) => !removedKeys.has(keyOf(resource))) } else { const incoming = sanitizeChatResources( change.resources.filter((resource) => resource.id !== 'streaming-file') ) - const byKey = new Map(existing.map((resource) => [getChatResourceKey(resource), resource])) + const byKey = new Map(existing.map((resource) => [keyOf(resource), resource])) if (change.kind === 'reorder') { - const keys = incoming.map(getChatResourceKey) + const keys = incoming.map(keyOf) if ( keys.length !== byKey.size || new Set(keys).size !== keys.length || @@ -131,7 +132,7 @@ export async function changeStoredChatResources( }) } else { for (const resource of incoming) { - const key = getChatResourceKey(resource) + const key = keyOf(resource) const previous = byKey.get(key) const merged = mergeChatResource(previous, resource) byKey.set(key, effectId ? { ...merged, title: resource.title || merged.title } : merged) diff --git a/apps/sim/lib/mothership/resources/types.test.ts b/apps/sim/lib/mothership/resources/types.test.ts index 17547c19ba8..445398a0a49 100644 --- a/apps/sim/lib/mothership/resources/types.test.ts +++ b/apps/sim/lib/mothership/resources/types.test.ts @@ -9,6 +9,7 @@ import { MothershipResourceType, mergeChatResource, mergePendingChatResourceUpdate, + normalizeChatResource, PERSISTED_RESOURCE_TYPES, reorderStoredChatResources, sanitizeChatResources, @@ -165,6 +166,12 @@ describe('mergeChatResource', () => { expect(mergeChatResource(stored, { ...stored })).toBe(stored) }) + it('keeps the stored entry when the newcomer repeats an equal owner', () => { + const owner = { entityType: 'project' as const, entityId: 'project-1' } + const file = resource({ type: 'file', id: 'file-1', title: 'Notes', owner }) + expect(mergeChatResource(file, { ...file, owner: { ...owner } })).toBe(file) + }) + it('replaces a placeholder title but never a specific one', () => { const placeholder = resource({ type: 'table', id: 'tbl-1', title: 'Table' }) expect(mergeChatResource(placeholder, stored).title).toBe('Invoices') @@ -267,6 +274,11 @@ describe('reorderStoredChatResources', () => { // length comparison would reject every reorder for that chat forever. expect(reorderStoredChatResources([table, table, file], [file, table])).toEqual([file, table]) }) + + it('matches a stored unscoped tab to the client tab its chat workspace owns', () => { + const owned = { ...file, owner: { entityType: 'workspace' as const, entityId: 'ws' } } + expect(reorderStoredChatResources([file, table], [table, owned], 'ws')).toEqual([table, file]) + }) }) describe('organization resource identity', () => { @@ -286,7 +298,7 @@ describe('organization resource identity', () => { }) expect(getChatResourceSelectionId(a)).toBe(a.id) expect(mergeChatResource(a, { ...a, path: 'files/new.png' })).toMatchObject({ - workspaceId: 'a', + owner: { entityType: 'workspace', entityId: 'a' }, workspaceName: 'Design', }) }) @@ -302,3 +314,15 @@ it.each(['', ' ', 'r1'])( expect(sanitizeChatResources([named])[0]?.title).toBe('Canonical workflow') } ) + +describe('normalizeChatResource', () => { + it('adds the workspace owner while keeping the address current browsers read', () => { + const resource = { type: 'file' as const, id: 'file', title: 'notes.md', workspaceId: 'ws' } + const normalized = normalizeChatResource(resource) + expect(normalized).toMatchObject({ + workspaceId: 'ws', + owner: { entityType: 'workspace', entityId: 'ws' }, + }) + expect(getChatResourceKey(normalized)).toBe(getChatResourceKey(resource)) + }) +}) diff --git a/apps/sim/lib/mothership/resources/types.ts b/apps/sim/lib/mothership/resources/types.ts index 4cc79d7d243..05f5efb5ae4 100644 --- a/apps/sim/lib/mothership/resources/types.ts +++ b/apps/sim/lib/mothership/resources/types.ts @@ -1,3 +1,4 @@ +import type { FileOwner } from '@/lib/mothership/generated/file-owner' import type { SearchResource } from '@/lib/mothership/generated/resources' export const MothershipResourceType = { @@ -22,10 +23,11 @@ export type MothershipResourceType = (typeof MothershipResourceType)[keyof typeof MothershipResourceType] export interface MothershipResource { + owner?: FileOwner type: MothershipResourceType id: string title: string - /** Canonical resource owner; independent of the conversation's owner. */ + /** Non-file workspace scope, or the legacy workspace file address. */ workspaceId?: string workspaceName?: string path?: string @@ -58,6 +60,7 @@ export interface MothershipResourceUpdate extends MothershipResource { * than filled in. */ export interface WorkspaceResourceRef { + owner?: FileOwner type: MothershipResourceType id?: string path?: string @@ -65,21 +68,90 @@ export interface WorkspaceResourceRef { workspaceId?: string } -/** Scope is part of identity for aliases such as integration names and folder paths. */ +/** Explicit file ownership never inherits the conversation's workspace. */ +export function getChatResourceWorkspaceId( + resource: Pick, + fallbackWorkspaceId?: string +): string | undefined { + if (!resource.owner) return resource.workspaceId ?? fallbackWorkspaceId + return resource.owner.entityType === 'workspace' ? resource.owner.entityId : undefined +} + +/** File and file-folder tabs are the resources that carry an explicit owner. */ +export function isFileResourceType(type: string): boolean { + return type === 'file' || type === 'filefolder' +} + +/** File owner pairs must agree with any legacy address supplied alongside them. */ +export function hasValidChatResourceOwner( + resource: Pick +): boolean { + if (!isFileResourceType(resource.type)) return !resource.owner + if (!resource.owner) + return resource.workspaceId === undefined || hasAddressableId(resource.workspaceId) + if (!hasAddressableId(resource.owner.entityId)) return false + if (resource.owner.entityType === 'workspace') + return resource.workspaceId === undefined || resource.workspaceId === resource.owner.entityId + return resource.owner.entityType === 'project' && resource.workspaceId === undefined +} + +/** + * Adds the explicit owner to a known workspace file address without inferring ownership for older + * unscoped panels. The address stays alongside it: browsers read `workspaceId` to open and close a + * workspace file tab, and the resource key is the same either way. + */ +export function normalizeChatResource( + resource: MothershipResourceUpdate +): MothershipResourceUpdate { + if ( + !isFileResourceType(resource.type) || + resource.workspaceId === undefined || + !hasValidChatResourceOwner(resource) + ) + return resource + return { + ...resource, + owner: resource.owner ?? { entityType: 'workspace', entityId: resource.workspaceId }, + } +} + +/** + * Tab identity, shared by the server store and the browser. Scope is part of identity for aliases + * such as integration names and folder paths. The chat's workspace is the fallback scope, so an + * unscoped tab and the same tab addressed by that workspace are one tab on both sides. + */ export function getChatResourceKey( - resource: Pick + resource: Pick, + chatWorkspaceId?: string ): string { - return resource.workspaceId - ? JSON.stringify([resource.workspaceId, resource.type, resource.id]) + if (resource.owner && resource.owner.entityType !== 'workspace') + return JSON.stringify([ + resource.owner.entityType, + resource.owner.entityId, + resource.type, + resource.id, + ]) + const workspaceId = getChatResourceWorkspaceId(resource, chatWorkspaceId) + return workspaceId + ? JSON.stringify([workspaceId, resource.type, resource.id]) : `${resource.type}:${resource.id}` } +/** {@link getChatResourceKey} bound to one chat's workspace, for comparing that chat's tabs. */ +export function chatResourceKeyFor( + chatWorkspaceId?: string +): (resource: Pick) => string { + return (resource) => getChatResourceKey(resource, chatWorkspaceId) +} + /** UUID resources keep their existing deep links; workspace aliases need their owner in the URL. */ export function getChatResourceSelectionId(resource: MothershipResource): string { + if (resource.owner && resource.owner.entityType !== 'workspace') + return getChatResourceKey(resource) const canonicalId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( resource.id ) - return resource.workspaceId && !canonicalId && !isEphemeralResource(resource) + return getChatResourceWorkspaceId(resource) && !canonicalId && !isEphemeralResource(resource) ? getChatResourceKey(resource) : resource.id } @@ -194,14 +266,16 @@ function withoutDesktopSessionResources( /** * The canonical form of a chat's resource list: legacy desktop panel rows and - * unaddressable resources dropped. Every path that reads or writes stored - * resources goes through this, which is what heals chats that already hold - * one. + * unaddressable resources dropped, known file addresses normalized. Every path + * that reads or writes stored resources goes through this, which is what heals + * chats that already hold one. */ export function sanitizeChatResources( resources: readonly MothershipResource[] ): MothershipResource[] { - return withoutDesktopSessionResources(resources).filter(isAddressableResource) + return withoutDesktopSessionResources(resources) + .filter((resource) => isAddressableResource(resource) && hasValidChatResourceOwner(resource)) + .map(normalizeChatResource) } /** @@ -211,7 +285,8 @@ export function sanitizeChatResources( */ export function reorderStoredChatResources( storedResources: readonly MothershipResource[], - requestedOrder: readonly MothershipResource[] + requestedOrder: readonly MothershipResource[], + chatWorkspaceId?: string ): MothershipResource[] | null { const stored = sanitizeChatResources(storedResources) const requested = sanitizeChatResources(requestedOrder) @@ -221,8 +296,9 @@ export function reorderStoredChatResources( // writers) sends one fewer entry from the deduplicated client. Matching on // sets keeps that reorder valid and collapses the duplicate on write, where // a length check would reject every reorder for that chat forever. - const storedByKey = new Map(stored.map((resource) => [getChatResourceKey(resource), resource])) - const requestedKeys = Array.from(new Set(requested.map(getChatResourceKey))) + const keyOf = chatResourceKeyFor(chatWorkspaceId) + const storedByKey = new Map(stored.map((resource) => [keyOf(resource), resource])) + const requestedKeys = Array.from(new Set(requested.map(keyOf))) if (requestedKeys.length !== storedByKey.size) return null const reordered: MothershipResource[] = [] @@ -252,6 +328,7 @@ export const GENERIC_RESOURCE_TITLES = new Set([ * and its no-op check. */ const MERGED_FIELDS = { + owner: true, title: true, workspaceId: true, workspaceName: true, @@ -277,6 +354,8 @@ export function mergeChatResource( prev: MothershipResource | undefined, next: MothershipResourceUpdate ): MothershipResource { + prev = prev ? normalizeChatResource(prev) : undefined + next = normalizeChatResource(next) if (!prev) { // Copied, never aliased: the result lands in React state, the query cache // and the pending-write queue at once, and `next` is the caller's object. @@ -286,6 +365,11 @@ export function mergeChatResource( const { viewId: _previousViewId, ...prevWithoutViewId } = prev const merged: MothershipResource = { ...(next.clearViewId === true ? prevWithoutViewId : prev), + ...(next.owner !== undefined && + (next.owner.entityType !== prev.owner?.entityType || + next.owner.entityId !== prev.owner.entityId) + ? { owner: next.owner } + : {}), ...(next.workspaceId !== undefined ? { workspaceId: next.workspaceId } : {}), ...(next.workspaceName !== undefined ? { workspaceName: next.workspaceName } : {}), ...(next.path !== undefined ? { path: next.path } : {}), diff --git a/apps/sim/lib/mothership/tool-executor/executor.ts b/apps/sim/lib/mothership/tool-executor/executor.ts index de809a77612..a92b9000ef3 100644 --- a/apps/sim/lib/mothership/tool-executor/executor.ts +++ b/apps/sim/lib/mothership/tool-executor/executor.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { type PermissionType, permissionSatisfies } from '@sim/platform-authz/workspace' import { toError } from '@sim/utils/errors' +import { openResourceInputSchema } from '@/lib/api/contracts/mothership-resource-tools' import { observeContentDelivery } from '@/lib/core/application' import { withWorkspaceInvocationScope } from '@/lib/core/application/workspace-invocation-scope' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' @@ -9,6 +10,7 @@ import { ASSISTANT_TOOLS, isAssistantIntegrationTool } from '@/lib/mothership/as import { createRegistryContentGate } from '@/lib/mothership/content-delivery' import { prepareCopilotEnvironmentContext } from '@/lib/mothership/environment-context' import { isSearchIntegrationToolsEnabled } from '@/lib/mothership/feature-flags' +import { isOwnerScopedFile } from '@/lib/mothership/file-owners' import { projectToolErrorMessageForCopilot } from '@/lib/mothership/request/tools/resolved-secret-result' import { recordSecretUsage } from '@/lib/secrets/usage/record' import { executeTool as executeAppTool } from '@/tools' @@ -60,6 +62,20 @@ export async function executeTool( ) { return { success: false, error: 'This tool is unavailable in this benchmark stage.' } } + if (toolId === 'open_resource') { + const parsed = openResourceInputSchema.safeParse(params) + if ( + parsed.success && + parsed.data.resources.every( + (resource) => + resource.type === 'file' && resource.owner && isOwnerScopedFile(resource.owner) + ) + ) { + if (context.targetWorkspaceId) + return { success: false, error: 'These file owners do not take a workspace target' } + return executeBoundTool(toolId, params, context) + } + } if (context.organizationId) { if ( context.workspaceId || diff --git a/apps/sim/lib/mothership/tool-executor/types.ts b/apps/sim/lib/mothership/tool-executor/types.ts index e3270a53134..d0f5c40d4a1 100644 --- a/apps/sim/lib/mothership/tool-executor/types.ts +++ b/apps/sim/lib/mothership/tool-executor/types.ts @@ -1,5 +1,6 @@ import type { BillingAttributionSnapshot } from '@/lib/billing/core/attribution' import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters' +import type { CopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' import type { MothershipResourceUpdate } from '@/lib/mothership/resources/types' import type { SecretMountPolicy } from '@/lib/mothership/secret-mount-policy' import type { ExecutorDelegationOrigin } from '@/executor/types' @@ -36,6 +37,8 @@ export interface ToolExecutionContext { boundWorkflowExecutionId?: string billingAttribution?: BillingAttributionSnapshot copilotToolExecution?: boolean + /** In-process authoring admission; never supplied by model arguments or checkpoint JSON. */ + copilotResourceAdmission?: CopilotResourceAdmission /** Trusted lifecycle classification stamped by the server, never from model parameters. */ copilotInteractionMode?: 'interactive' | 'headless' requestMode?: string diff --git a/apps/sim/lib/mothership/tools/activity.ts b/apps/sim/lib/mothership/tools/activity.ts index f2c5ba7fa30..add0260af05 100644 --- a/apps/sim/lib/mothership/tools/activity.ts +++ b/apps/sim/lib/mothership/tools/activity.ts @@ -239,6 +239,7 @@ export const TOOL_ACTIVITIES: Readonly = { list_file_folders: 'Listing folders', list_integration_tools: 'Listing integration tools', list_user_workspaces: 'Listing workspaces', + list_user_projects: 'Listing projects', list_workspace_mcp_servers: 'Listing MCP servers', load_deployment: 'Loading deployment', save_upload: 'Saving upload', diff --git a/apps/sim/lib/mothership/tools/handlers/sim-cli.ts b/apps/sim/lib/mothership/tools/handlers/sim-cli.ts index 0faa098d88a..f188468b19f 100644 --- a/apps/sim/lib/mothership/tools/handlers/sim-cli.ts +++ b/apps/sim/lib/mothership/tools/handlers/sim-cli.ts @@ -40,6 +40,11 @@ export async function executeSimCli( signal: context.abortSignal, toolCallId: context.toolCallId, copilotToolExecution: context.copilotToolExecution, + copilotResourceAdmission: context.copilotResourceAdmission, + copilotInteractionMode: context.copilotInteractionMode, + executorDelegationOrigin: context.executorDelegationOrigin, + mcpBlockId: context.mcpBlockId, + boundWorkflowExecutionId: context.boundWorkflowExecutionId, requestMode: context.requestMode, searchSurface: context.searchSurface, assistantSearch: context.assistantSearch, diff --git a/apps/sim/lib/mothership/tools/registry/server-tool-adapter.ts b/apps/sim/lib/mothership/tools/registry/server-tool-adapter.ts index 35c5034b581..c4f022f0739 100644 --- a/apps/sim/lib/mothership/tools/registry/server-tool-adapter.ts +++ b/apps/sim/lib/mothership/tools/registry/server-tool-adapter.ts @@ -29,6 +29,10 @@ export function createServerToolHandler(toolId: string): ToolHandler { executionId: context.executionId, toolCallId: context.toolCallId, copilotToolExecution: context.copilotToolExecution, + copilotResourceAdmission: context.copilotResourceAdmission, + mcpBlockId: context.mcpBlockId, + executorDelegationOrigin: context.executorDelegationOrigin, + boundWorkflowExecutionId: context.boundWorkflowExecutionId, requestMode: context.requestMode, searchSurface: context.searchSurface, assistantSearch: context.assistantSearch, diff --git a/apps/sim/lib/mothership/tools/sandbox-callback.ts b/apps/sim/lib/mothership/tools/sandbox-callback.ts new file mode 100644 index 00000000000..577821e2db7 --- /dev/null +++ b/apps/sim/lib/mothership/tools/sandbox-callback.ts @@ -0,0 +1,40 @@ +import type { AgentCliExecutionContext } from '@/lib/mothership/agent-cli' +import { createCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' +import type { SandboxResourceScope } from '@/lib/mothership/tools/sandbox-resources' + +/** Lease credentials and owner hints the callback consumed; the in-process request never carries them on. */ +const CALLBACK_ONLY_HEADERS = [ + 'host', + 'cookie', + 'authorization', + 'connection', + 'transfer-encoding', + 'x-api-key', + 'x-mothership-workspace-id', + 'x-mothership-file-owner', +] as const + +/** The callback request's headers with its credentials and routing hints removed. */ +export function forwardableSandboxHeaders(request: Request): Headers { + const headers = new Headers(request.headers) + for (const name of CALLBACK_ONLY_HEADERS) headers.delete(name) + return headers +} + +/** The admitted chat invocation a live callback lease acts for; each operation reauthorizes it. */ +export function sandboxCallbackContext( + scope: SandboxResourceScope, + signal?: AbortSignal +): AgentCliExecutionContext { + return { + userId: scope.userId, + chatId: scope.chatId, + toolCallId: scope.toolCallId, + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: scope.userId, + invocation: { kind: 'chat', chatId: scope.chatId }, + }), + ...(signal ? { signal } : {}), + } +} diff --git a/apps/sim/lib/mothership/tools/sandbox-file-copy.ts b/apps/sim/lib/mothership/tools/sandbox-file-copy.ts new file mode 100644 index 00000000000..1cf7e30d7f1 --- /dev/null +++ b/apps/sim/lib/mothership/tools/sandbox-file-copy.ts @@ -0,0 +1,102 @@ +import { createLogger } from '@sim/logger' +import { generateId } from '@sim/utils/id' +import { NextRequest } from 'next/server' +import { v2CopyFileItemsContract } from '@/lib/api/contracts/v2/file-copy' +import { V2_PARSE_DEFAULTS, v2InvalidBodyResponse } from '@/lib/api/server/routes/v2-json-route' +import { parseRequest } from '@/lib/api/server/validation' +import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' +import { createFileCopyCliTransport } from '@/lib/mothership/agent-cli/file-copy-transport' +import { createResourceEffectTransport } from '@/lib/mothership/agent-cli/resource-effects' +import type { ResourceChange } from '@/lib/mothership/generated/resources' +import { sandboxCallbackContext } from '@/lib/mothership/tools/sandbox-callback' +import { + recordSandboxResourceEffects, + type SandboxResourceScope, +} from '@/lib/mothership/tools/sandbox-resources' + +const logger = createLogger('SandboxFileCopy') + +/** A live callback delegates the exact pair; the shared operation owns both authorization and durable provenance. */ +export async function proxySandboxFileCopyRequest( + request: Request, + token: string, + scope: SandboxResourceScope +): Promise { + if ( + scope.fileOwnerProtocolVersion !== 1 || + request.headers.has('x-mothership-file-owner') || + request.headers.has('x-mothership-workspace-id') + ) + return Response.json( + { error: 'Paired file owner routing is unavailable or contradictory' }, + { status: 403 } + ) + if (request.method !== 'POST') + return Response.json({ error: 'Method not allowed' }, { status: 405 }) + const parsed = await parseRequest( + v2CopyFileItemsContract, + new NextRequest(request.clone()), + {}, + { + ...V2_PARSE_DEFAULTS, + invalidJsonResponse: () => v2InvalidBodyResponse(request), + } + ) + if (!parsed.success) return parsed.response + const endpoint = getInternalApiBaseUrl() + const transport = createFileCopyCliTransport( + endpoint, + sandboxCallbackContext(scope), + parsed.data.body + ) + const forwarded = new Request(`${endpoint.replace(/\/$/, '')}${v2CopyFileItemsContract.path}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(parsed.data.body), + signal: request.signal, + }) + const effects: ResourceChange[] = [] + let completed: Response | undefined + const dispatch: typeof fetch = async (input, init) => { + completed = await transport(input, init) + return completed + } + let response: Response + try { + response = await createResourceEffectTransport(endpoint, dispatch, effects)(forwarded) + } catch (error) { + if (!completed) throw error + response = completed + logger.warn('Copy callback effect projection failed after API completion', { + toolCallId: scope.toolCallId, + }) + } + const requestId = generateId() + await recordSandboxResourceEffects( + token, + scope, + effects.map((effect, index): ResourceChange & { effectId: string } => { + const effectId = `${scope.runId}:${scope.toolCallId}:${requestId}:${index}` + if (!scope.organizationId) return { ...effect, effectId } + /** Organization chats address workspace resources explicitly, as the workspace route does. */ + if (effect.op === 'upsert' && effect.resource.owner?.entityType === 'workspace') + return { + ...effect, + effectId, + resource: { ...effect.resource, workspaceId: effect.resource.owner.entityId }, + } + if ( + effect.op === 'refresh' && + 'owner' in effect.resource && + effect.resource.owner?.entityType === 'workspace' + ) + return { + ...effect, + effectId, + resource: { ...effect.resource, workspaceId: effect.resource.owner.entityId }, + } + return { ...effect, effectId } + }) + ) + return response +} diff --git a/apps/sim/lib/mothership/tools/sandbox-project-files.ts b/apps/sim/lib/mothership/tools/sandbox-project-files.ts new file mode 100644 index 00000000000..61241f4c1bc --- /dev/null +++ b/apps/sim/lib/mothership/tools/sandbox-project-files.ts @@ -0,0 +1,140 @@ +import { createLogger } from '@sim/logger' +import { generateId } from '@sim/utils/id' +import { getRedisClient } from '@/lib/core/config/redis' +import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' +import { createDurableSecretProvenanceRegistry } from '@/lib/execution/durable-secret-provenance' +import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' +import { readSessionSecretProvenance } from '@/lib/execution/remote-sandbox/session-file-provenance' +import { createProjectFileCliTransport } from '@/lib/mothership/agent-cli/project-file-transport' +import { createProjectFileUploadTransport } from '@/lib/mothership/agent-cli/project-file-upload-transport' +import { createProjectFileWriteTransport } from '@/lib/mothership/agent-cli/project-file-write-transport' +import { createResourceEffectTransport } from '@/lib/mothership/agent-cli/resource-effects' +import { createSessionContentGate, deliverSandboxResponse } from '@/lib/mothership/content-delivery' +import type { ResourceChange } from '@/lib/mothership/generated/resources' +import { + forwardableSandboxHeaders, + sandboxCallbackContext, +} from '@/lib/mothership/tools/sandbox-callback' +import { + recordSandboxResourceEffects, + type SandboxResourceScope, +} from '@/lib/mothership/tools/sandbox-resources' +import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +const logger = createLogger('SandboxProjectFiles') + +/** Authenticated callback lease supplies identity; every selected Project operation reauthorizes it. */ +export async function proxySandboxProjectFileRequest( + request: Request, + token: string, + path: string, + search: string, + scope: SandboxResourceScope, + projectId: string +): Promise { + const endpoint = getInternalApiBaseUrl() + const sessionKey = chatSandboxSessionKey(scope.chatId) + const context = sandboxCallbackContext(scope, request.signal) + const history = async () => { + request.signal.throwIfAborted() + const provider = resolveProvider() + const machine = await provider.findSessionSandbox?.(sessionKey, {}) + if (!machine) throw new Error('The active workbench is unavailable') + const provenance = await readSessionSecretProvenance(sessionKey, { + providerId: provider.id, + sandboxId: machine.sandboxId, + }) + if (provenance.status !== 'exact') throw new Error('Workbench source provenance is unavailable') + return provenance + } + const resolveSecretTraceRegistry = async () => + (await createDurableSecretProvenanceRegistry(await history(), { + userId: scope.userId, + })) ?? new ResolvedSecretTraceRegistry([], { userId: scope.userId }) + const bindingKey = `mothership:sandbox-resources:${token}:project-uploads:${encodeURIComponent(projectId)}` + const redis = () => { + const client = getRedisClient() + if (!client) throw new Error('Workbench upload binding is unavailable') + return client + } + const transport = createProjectFileUploadTransport({ + endpoint, + projectId, + context, + resolveSecretTraceRegistry, + uploadBinding: { + async record(uploadId) { + await redis().eval( + "redis.call('SADD', KEYS[1], ARGV[1]); redis.call('EXPIRE', KEYS[1], 600); return 1", + 1, + bindingKey, + uploadId + ) + }, + async contains(uploadId) { + return (await redis().sismember(bindingKey, uploadId)) === 1 + }, + }, + /** Direct uploads conservatively retain the physical machine's complete source history. */ + async uploadProvenance() { + const provenance = await history() + request.signal.throwIfAborted() + return { + status: 'exact', + entries: provenance.entries.map((entry) => { + if (!entry.sourceUserId) throw new Error('Workbench source identity is unavailable') + return { ...entry, sourceUserId: entry.sourceUserId } + }), + } + }, + fallback: createProjectFileWriteTransport({ + endpoint, + projectId, + context, + resolveSecretTraceRegistry, + fallback: createProjectFileCliTransport(endpoint, context, { projectId }), + }), + }) + const forwarded = new Request(`${endpoint.replace(/\/$/, '')}${path}${search}`, { + method: request.method, + headers: forwardableSandboxHeaders(request), + signal: request.signal, + redirect: 'manual', + ...(request.method === 'GET' || request.method === 'HEAD' + ? {} + : { body: request.body, duplex: 'half' }), + }) + /** The same workbench gate as workspace callbacks: whatever the request reads widens the machine's history first. */ + const gate = createSessionContentGate({ sessionKey, actorUserId: scope.userId }) + const effects: ResourceChange[] = [] + let completed: Response | undefined + const dispatch: typeof fetch = async (input, init) => { + completed = await deliverSandboxResponse(gate, () => transport(input, init), scope.toolCallId) + return completed + } + let response: Response + try { + response = await createResourceEffectTransport(endpoint, dispatch, effects)(forwarded) + } catch (error) { + if (!completed) throw error + response = completed + logger.warn('Project callback effect projection failed after API completion', { + toolCallId: scope.toolCallId, + }) + } + const requestId = generateId() + await recordSandboxResourceEffects( + token, + scope, + effects.map((effect, index) => ({ + ...effect, + effectId: `${scope.runId}:${scope.toolCallId}:${requestId}:${index}`, + })) + ) + return new Response(request.method === 'HEAD' ? null : response.body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }) +} diff --git a/apps/sim/lib/mothership/tools/sandbox-resource-transport.test.ts b/apps/sim/lib/mothership/tools/sandbox-resource-transport.test.ts index 86d5dd13e03..67b8f649dc0 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resource-transport.test.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resource-transport.test.ts @@ -5,7 +5,7 @@ import { } from '@sim/testing/mocks/mothership-workspace-target.mock' import { urlsMockFns } from '@sim/testing/mocks/urls.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' -import { isCopilotRequest } from '@/lib/api/server/routes/copilot-request' +import { copilotRequestPrincipal, isCopilotRequest } from '@/lib/api/server/routes/copilot-request' import { type ContentSource, noteContentDeliveryOperation, @@ -14,18 +14,21 @@ import { import { assertWorkspaceInvocationScope } from '@/lib/core/application/workspace-invocation-scope' import { OrchestrationError } from '@/lib/core/orchestration/types' import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key' +import { projectFileOperations } from '@/lib/projects/files/application/operations' import { fileOperations } from '@/lib/workspace-files/application/operations' -const { readScope, recordEffects, fetcher, routeMatcher, recordInput, mint } = vi.hoisted(() => ({ +const { readScope, recordEffects, fetcher, routeMatcher, recordInput } = vi.hoisted(() => ({ readScope: vi.fn(), recordEffects: vi.fn(async () => {}), fetcher: vi.fn(), routeMatcher: vi.fn(), recordInput: vi.fn(), - mint: vi.fn(), })) vi.mock('@/lib/mothership/application/workspace-target', () => mothershipWorkspaceTargetMock) -vi.mock('@/lib/api/server/routes/in-process-transport', () => ({ matchV2Route: routeMatcher })) +vi.mock('@/lib/api/server/routes/in-process-transport', () => ({ + matchV2Route: routeMatcher, + dispatchInProcessV2Request: (request: Request) => fetcher(request), +})) vi.mock('@/lib/mothership/tools/sandbox-resources', () => ({ readSandboxResourceScope: readScope, recordSandboxResourceEffects: recordEffects, @@ -58,7 +61,6 @@ function request(path: string, init?: RequestInit) { beforeEach(() => { readScope.mockResolvedValue(scope) - mint.mockResolvedValue('server-only-identity') target.mockResolvedValue({ workspaceId: scope.workspaceId }) routeMatcher.mockReturnValue({ params: { tableId: 'table' }, @@ -146,7 +148,6 @@ describe('private sandbox v2 resource transport', () => { ) ).status ).toBe(403) - expect(mint).not.toHaveBeenCalled() expect(fetcher).not.toHaveBeenCalled() target.mockResolvedValueOnce({ workspaceId: 'target' }) fetcher.mockImplementation(async (req: Request) => { @@ -164,7 +165,6 @@ describe('private sandbox v2 resource transport', () => { ).status ).toBe(200) expect(target).toHaveBeenLastCalledWith(orgScope, 'target') - expect(mint).not.toHaveBeenCalled() }) it('rejects stale scopes and non-v2 destinations before dispatch', async () => { @@ -268,7 +268,11 @@ it('records nothing into the workbench for a route that returns no stored conten load: async () => ({ GET: fetcher }), }) fetcher.mockResolvedValue(Response.json({ data: { id: 'agent' } })) - await proxySandboxResourceRequest(request('/api/v2/tools/function_execute'), token) + const response = await proxySandboxResourceRequest( + request('/api/v2/tools/function_execute'), + token + ) + expect(await response.json()).toEqual({ data: { id: 'agent' } }) expect(fetcher).toHaveBeenCalledOnce() expect(recordInput).not.toHaveBeenCalled() }) @@ -293,6 +297,42 @@ it('replaces a successful response its sourced operation never classified with a expect(recordInput).not.toHaveBeenCalled() }) +const OWNER_ROUTING_REFUSED = { + status: 403, + error: 'File owner routing is unavailable or contradictory', +} +const OWNER_PATH_MISMATCH = { status: 400, error: 'File owner does not match the API path' } + +it.each([ + [ + 'contradictory workspace', + '/api/v2/projects/project/files', + 1, + { 'x-mothership-workspace-id': 'workspace' }, + OWNER_ROUTING_REFUSED, + ], + ['another Project path', '/api/v2/projects/other/files', 1, {}, OWNER_PATH_MISMATCH], + ['non-file path', '/api/v2/tables', 1, {}, OWNER_PATH_MISMATCH], + ['unnegotiated lease', '/api/v2/projects/project/files', undefined, {}, OWNER_ROUTING_REFUSED], +] as const)( + 'refuses %s before a Project callback can dispatch', + async (_name, path, protocol, headers, refusal) => { + readScope.mockResolvedValue({ ...scope, fileOwnerProtocolVersion: protocol }) + fetcher.mockResolvedValue(Response.json({ data: [] })) + const response = await proxySandboxResourceRequest( + request(path, { + headers: { + ...headers, + 'x-mothership-file-owner': JSON.stringify({ entityType: 'project', entityId: 'project' }), + }, + }), + token + ) + expect(response.status).toBe(refusal.status) + expect(await response.json()).toEqual({ error: refusal.error }) + expect(fetcher).not.toHaveBeenCalled() + } +) /** * The file read and download use cases refuse a Chat principal no gate observes, so the * workbench's reads depend on this proxy installing its session gate around the route handler. @@ -331,18 +371,97 @@ it('withholds a result whose provenance the workbench history could not record', expect(fetcher).toHaveBeenCalledOnce() }) -vi.mock('@/lib/mothership/chat/delegation', () => ({ mintDelegationToken: mint })) +it('reconstructs Project authority from the admitted lease without borrowing its workspace', async () => { + readScope.mockResolvedValue({ ...scope, fileOwnerProtocolVersion: 1 }) + routeMatcher.mockReturnValue({ + pattern: '/api/v2/projects/{projectId}/files', + params: { projectId: 'project' }, + load: async () => ({ GET: fetcher }), + }) + fetcher.mockImplementation(async (request: Request) => { + const principal = copilotRequestPrincipal(request, projectFileOperations.list, { + operation: projectFileOperations.list, + delegationAudience: 'sim:project-files', + }) + expect(principal).toMatchObject({ + kind: 'resource_delegated', + subjectUserId: scope.userId, + invocation: { kind: 'chat', chatId: scope.chatId }, + scope: { kind: 'entity', entityType: 'project', entityId: 'project' }, + }) + expect(principal).not.toHaveProperty('workspaceId') + return Response.json({ data: [] }) + }) + const response = await proxySandboxResourceRequest( + request('/api/v2/projects/project/files', { + headers: { + 'x-mothership-file-owner': JSON.stringify({ entityType: 'project', entityId: 'project' }), + }, + }), + token + ) + expect(response.status).toBe(200) + expect(target).not.toHaveBeenCalled() +}) + +const workspaceOwner = { entityType: 'workspace', entityId: 'workspace' } + +it.each([ + [ + 'a workspace pair', + { + source: { owner: workspaceOwner, fileIds: ['file'] }, + destination: { owner: workspaceOwner }, + }, + ], + ['an invalid body', { source: {} }], +])( + 'keeps %s copy on the workspace route when the lease has no file-owner protocol', + async (_name, copy) => { + readScope.mockResolvedValue({ ...scope, fileOwnerProtocolVersion: undefined }) + routeMatcher.mockReturnValue({ params: {}, load: async () => ({ POST: fetcher }) }) + const dispatched: string[] = [] + fetcher.mockImplementation(async (forwarded: Request) => { + dispatched.push(`${new URL(forwarded.url).pathname} ${await forwarded.text()}`) + return Response.json({ data: { copied: [] } }) + }) + const body = JSON.stringify(copy) + const response = await proxySandboxResourceRequest( + request('/api/v2/files/copy', { method: 'POST', body }), + token + ) + expect(response.status).toBe(200) + expect(dispatched).toEqual([`/api/v2/files/copy ${body}`]) + } +) -it('never resolves a real credential for an invalid callback scope', async () => { - readScope.mockResolvedValue(null) - expect((await proxySandboxResourceRequest(request('/api/v2/tools'), token)).status).toBe(403) - expect(mint).not.toHaveBeenCalled() +it('refuses a Project copy destination on a lease without the file-owner protocol', async () => { + readScope.mockResolvedValue({ ...scope, fileOwnerProtocolVersion: undefined }) + routeMatcher.mockReturnValue({ params: {}, load: async () => ({ POST: fetcher }) }) + const response = await proxySandboxResourceRequest( + request('/api/v2/files/copy', { + method: 'POST', + body: JSON.stringify({ + source: { owner: workspaceOwner, fileIds: ['file'] }, + destination: { owner: { entityType: 'project', entityId: 'project' } }, + }), + }), + token + ) + expect(response.status).toBe(403) + expect(await response.json()).toEqual({ + error: 'Paired file owner routing is unavailable or contradictory', + }) expect(fetcher).not.toHaveBeenCalled() }) -it('keeps the server identity out of callback response headers and body', async () => { - fetcher.mockResolvedValue(Response.json({ data: [] })) - const response = await proxySandboxResourceRequest(request('/api/v2/tools'), token) - expect(mint).not.toHaveBeenCalled() - expect(JSON.stringify([...response.headers])).not.toContain('server-only-identity') - expect(await response.text()).not.toContain('server-only-identity') + +it('validates a copy body itself once the lease negotiated the file-owner protocol', async () => { + readScope.mockResolvedValue({ ...scope, fileOwnerProtocolVersion: 1 }) + routeMatcher.mockReturnValue({ params: {}, load: async () => ({ POST: fetcher }) }) + const response = await proxySandboxResourceRequest( + request('/api/v2/files/copy', { method: 'POST', body: JSON.stringify({ source: {} }) }), + token + ) + expect(response.status).toBe(400) + expect(fetcher).not.toHaveBeenCalled() }) diff --git a/apps/sim/lib/mothership/tools/sandbox-resource-transport.ts b/apps/sim/lib/mothership/tools/sandbox-resource-transport.ts index b6361660268..ea22ac4e8f7 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resource-transport.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resource-transport.ts @@ -1,24 +1,25 @@ import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { NextRequest } from 'next/server' +import { fileCopyInputSchema } from '@/lib/api/contracts/file-copy-input' +import { v2CopyFileItemsContract } from '@/lib/api/contracts/v2/file-copy' import { markCopilotRequest } from '@/lib/api/server/routes/copilot-request' import { matchV2Route } from '@/lib/api/server/routes/in-process-transport' -import { ContentGateFailure } from '@/lib/core/application' import { withWorkspaceInvocationScope } from '@/lib/core/application/workspace-invocation-scope' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import { createResourceEffectTransport } from '@/lib/mothership/agent-cli/resource-effects' import { resolveInvocationWorkspace } from '@/lib/mothership/application/workspace-target' -import { - createSessionContentGate, - dispatchUnderContentGate, - markCommittedWriteResponse, - WorkbenchProvenanceUnavailableError, -} from '@/lib/mothership/content-delivery' +import { createSessionContentGate, deliverSandboxResponse } from '@/lib/mothership/content-delivery' +import { FileOperationOwner } from '@/lib/mothership/generated/file-owner' import type { ResourceChange } from '@/lib/mothership/generated/resources' +import { forwardableSandboxHeaders } from '@/lib/mothership/tools/sandbox-callback' +import { proxySandboxFileCopyRequest } from '@/lib/mothership/tools/sandbox-file-copy' +import { proxySandboxProjectFileRequest } from '@/lib/mothership/tools/sandbox-project-files' import { readSandboxResourceScope, recordSandboxResourceEffects, + type SandboxResourceScope, } from '@/lib/mothership/tools/sandbox-resources' import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key' @@ -45,11 +46,44 @@ export async function proxySandboxResourceRequest( const scope = await readSandboxResourceScope(token, request.headers.get('x-api-key')) if (!scope) return Response.json({ error: 'Sandbox tool execution is no longer active' }, { status: 403 }) + /** + * Paired owners need the negotiated protocol. Without it, only a copy that names a non-workspace + * owner is refused there; workspace copies and invalid bodies keep the workspace route and its + * v2 validation response. + */ + if ( + path === v2CopyFileItemsContract.path && + (scope.fileOwnerProtocolVersion === 1 || (await copiesBeyondWorkspaces(request))) + ) + return proxySandboxFileCopyRequest(request, token, scope) + const ownerHeader = request.headers.get('x-mothership-file-owner') + let owner: FileOperationOwner | undefined + if (ownerHeader !== null) { + if (scope.fileOwnerProtocolVersion !== 1 || request.headers.has('x-mothership-workspace-id')) + return Response.json( + { error: 'File owner routing is unavailable or contradictory' }, + { status: 403 } + ) + try { + owner = FileOperationOwner.parse(JSON.parse(ownerHeader)) + } catch { + return Response.json({ error: 'Invalid file owner' }, { status: 400 }) + } + const ownerPath = + owner.entityType === 'project' + ? `/api/v2/projects/${encodeURIComponent(owner.entityId)}/files` + : '/api/v2/files' + if (path !== ownerPath && !path.startsWith(`${ownerPath}/`)) + return Response.json({ error: 'File owner does not match the API path' }, { status: 400 }) + if (owner.entityType === 'project') { + return proxySandboxProjectFileRequest(request, token, path, url.search, scope, owner.entityId) + } + } let targetWorkspaceId: string try { const target = await resolveInvocationWorkspace( scope, - request.headers.get('x-mothership-workspace-id') ?? undefined + owner?.entityId ?? request.headers.get('x-mothership-workspace-id') ?? undefined ) targetWorkspaceId = target.workspaceId } catch (error) { @@ -70,25 +104,14 @@ async function proxyAuthorizedSandboxRequest( token: string, path: string, url: URL, - scope: NonNullable>>, + scope: SandboxResourceScope, targetWorkspaceId: string ): Promise { const endpoint = getInternalApiBaseUrl() const target = `${endpoint.replace(/\/$/, '')}${path}${url.search}` - const headers = new Headers(request.headers) - for (const header of [ - 'host', - 'cookie', - 'authorization', - 'connection', - 'transfer-encoding', - 'x-mothership-workspace-id', - ]) - headers.delete(header) - headers.delete('x-api-key') const init: RequestInit & { duplex: 'half' } = { method: request.method, - headers, + headers: forwardableSandboxHeaders(request), signal: request.signal, redirect: 'manual', duplex: 'half', @@ -125,29 +148,7 @@ async function proxyAuthorizedSandboxRequest( }) const deliver = async () => { dispatched = true - try { - response = (await dispatchUnderContentGate(gate, dispatch)).response - } catch (error) { - if ( - !(error instanceof ContentGateFailure) || - !(error.gateFailure instanceof WorkbenchProvenanceUnavailableError) - ) { - throw error - } - /** A mutation may already have committed; the result is withheld, never retried for it. */ - logger.warn('Sandbox response provenance could not be recorded', { - toolCallId: scope.toolCallId, - }) - const withheld = Response.json( - { - error: error.committedWrite - ? 'The change was saved, but its result could not be returned safely. Do not retry the change.' - : 'The API request ran, but its result could not be returned safely. Do not retry a mutation automatically; read the resource to check its current state.', - }, - { status: 502 } - ) - response = error.committedWrite ? markCommittedWriteResponse(withheld) : withheld - } + response = await deliverSandboxResponse(gate, dispatch, scope.toolCallId) return response } try { @@ -201,3 +202,19 @@ async function proxyAuthorizedSandboxRequest( headers: response.headers, }) } + +/** A valid copy naming a non-workspace source or destination needs the file-owner protocol. */ +async function copiesBeyondWorkspaces(request: Request): Promise { + let body: unknown + try { + body = await request.clone().json() + } catch { + return false + } + const parsed = fileCopyInputSchema.safeParse(body) + return ( + parsed.success && + (parsed.data.source.owner.entityType !== 'workspace' || + parsed.data.destination.owner.entityType !== 'workspace') + ) +} diff --git a/apps/sim/lib/mothership/tools/sandbox-resources.test.ts b/apps/sim/lib/mothership/tools/sandbox-resources.test.ts index 4186d665480..29e1aa6e9fe 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resources.test.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resources.test.ts @@ -1,3 +1,4 @@ +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' import { mothershipAsyncRunsMock, mothershipAsyncRunsMockFns, @@ -6,6 +7,8 @@ import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { StreamEvent } from '@/lib/mothership/request/types' +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) + const { values, lists, seen, persist, redis } = vi.hoisted(() => { const values = new Map() const lists = new Map() @@ -298,3 +301,40 @@ describe('sandbox resource ownership and publication', () => { expect(persist).toHaveBeenCalledTimes(1) }) }) + +it.each([ + [true, true, true, 1], + [false, true, true, undefined], + [true, false, true, undefined], + [true, true, false, undefined], +] as const)( + 'negotiates the callback protocol only with admission=%s projects=%s peer=%s', + async (admitted, enabled, supported, expected) => { + featureFlagsMockFns.mockIsFeatureEnabled.mockResolvedValue(enabled) + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response(null, { + headers: supported ? { 'X-Mothership-File-Owner-Protocol': '1' } : {}, + }) + ) + ) + await withSandboxResourceScope( + { + ...identity, + resourceAdmitted: admitted, + mothershipBaseURL: `https://worker-${supported ? 'current' : 'previous'}.invalid`, + }, + new AbortController().signal, + undefined, + async () => { + const endpoint = await sandboxResourceEndpoint('https://sim.test', args, 'secret') + const token = endpoint.split('/').at(-1) + if (!token) throw new Error('Callback token missing') + const scope = await readSandboxResourceScope(token, 'secret') + expect(scope?.fileOwnerProtocolVersion).toBe(expected) + } + ) + } +) diff --git a/apps/sim/lib/mothership/tools/sandbox-resources.ts b/apps/sim/lib/mothership/tools/sandbox-resources.ts index b34107d594a..6f0245558cd 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resources.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resources.ts @@ -8,9 +8,11 @@ import { getRedisClient } from '@/lib/core/config/redis' import type { SimToolExecutionOwner } from '@/lib/mothership/async-runs/execution-lease' import { isActiveSandboxResourceOwner } from '@/lib/mothership/async-runs/repository' import { ResourcePayload } from '@/lib/mothership/generated/resources' +import { supportsFileOwnerProtocol } from '@/lib/mothership/request/lifecycle/file-owner-protocol' import type { StreamEvent } from '@/lib/mothership/request/types' import { persistResourceEffect } from '@/lib/mothership/resources/persist-effect' import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key' +import { isProjectsEnabled } from '@/lib/projects/rollout.server' const logger = createLogger('MothershipSandboxResources') const CONTEXT_TTL_SECONDS = 600 @@ -25,21 +27,25 @@ const scopeSchema = z workspaceId: z.string().optional(), organizationId: z.string().optional(), apiKeyHash: z.string(), + fileOwnerProtocolVersion: z.literal(1).optional(), }) .refine( (scope) => Boolean(scope.workspaceId) !== Boolean(scope.organizationId), 'Exactly one chat owner is required' ) -type SandboxResourceScope = z.infer +export type SandboxResourceScope = z.infer interface ActiveSandboxResourceScope { identity: SimToolExecutionOwner & { chatId: string workspaceId?: string organizationId?: string + resourceAdmitted?: boolean + mothershipBaseURL?: string } signal: AbortSignal token?: string + fileOwnerProtocolVersion?: 1 } const resourceScope = new AsyncLocalStorage() @@ -49,6 +55,11 @@ export function activeSandboxChatOwner() { return resourceScope.getStore()?.identity } +/** Negotiated routing hint; callback authorization remains bound to the active lease. */ +export function activeSandboxFileOwnerProtocol() { + return resourceScope.getStore()?.fileOwnerProtocolVersion +} + const contextKey = (token: string) => `mothership:sandbox-resources:${token}:context` const inboxKey = (token: string) => `mothership:sandbox-resources:${token}:inbox` const seenKey = (token: string) => `mothership:sandbox-resources:${token}:seen` @@ -78,7 +89,25 @@ export async function sandboxResourceEndpoint( ) throw new Error('Sandbox resource scope does not match the active tool') const token = active.token ?? generateId() - const scope: SandboxResourceScope = { ...identity, apiKeyHash: sha256Hex(apiKey) } + const fileOwnerProtocolVersion = + identity.resourceAdmitted && + identity.mothershipBaseURL && + (await isProjectsEnabled()) && + (await supportsFileOwnerProtocol(identity.mothershipBaseURL)) + ? 1 + : undefined + const scope: SandboxResourceScope = { + toolCallId: identity.toolCallId, + runId: identity.runId, + userId: identity.userId, + ownerToken: identity.ownerToken, + chatId: identity.chatId, + workspaceId: identity.workspaceId, + organizationId: identity.organizationId, + apiKeyHash: sha256Hex(apiKey), + fileOwnerProtocolVersion, + } + active.fileOwnerProtocolVersion = fileOwnerProtocolVersion await redisClient().set(contextKey(token), JSON.stringify(scope), 'EX', CONTEXT_TTL_SECONDS) active.token = token if (active.signal.aborted) { diff --git a/apps/sim/lib/mothership/tools/sandbox-session.test.ts b/apps/sim/lib/mothership/tools/sandbox-session.test.ts index d85b08cf212..75f6d6eeff7 100644 --- a/apps/sim/lib/mothership/tools/sandbox-session.test.ts +++ b/apps/sim/lib/mothership/tools/sandbox-session.test.ts @@ -126,7 +126,11 @@ describe('deployment-owned workbench tooling', () => { ) }) -vi.mock('@/lib/mothership/tools/sandbox-resources', () => ({ sandboxResourceEndpoint: endpoint })) +vi.mock('@/lib/mothership/tools/sandbox-resources', () => ({ + sandboxResourceEndpoint: endpoint, + activeSandboxChatOwner: () => undefined, + activeSandboxFileOwnerProtocol: () => undefined, +})) it('does not inject authentication when no active scoped callback can be established', async () => { connection.mockReturnValue({ mode: 'direct' }) diff --git a/apps/sim/lib/mothership/tools/sandbox-session.ts b/apps/sim/lib/mothership/tools/sandbox-session.ts index c262958226f..1cbd26b006c 100644 --- a/apps/sim/lib/mothership/tools/sandbox-session.ts +++ b/apps/sim/lib/mothership/tools/sandbox-session.ts @@ -17,7 +17,11 @@ import { WorkbenchBootstrap } from '@/lib/mothership/generated/workbench' import { fetchGo } from '@/lib/mothership/request/go/fetch' import { mothershipRequestHeaders } from '@/lib/mothership/request/headers' import { getMothershipBaseURL } from '@/lib/mothership/server/agent-url' -import { sandboxResourceEndpoint } from '@/lib/mothership/tools/sandbox-resources' +import { + activeSandboxChatOwner, + activeSandboxFileOwnerProtocol, + sandboxResourceEndpoint, +} from '@/lib/mothership/tools/sandbox-resources' import { getSimConnection } from '@/lib/mothership/transport/connection' import { containsResolvedSecret, @@ -70,7 +74,8 @@ async function workbenchCli( ) signal?.throwIfAborted() const runtime = await readFile(path, 'utf8') - const baseURL = await getMothershipBaseURL({ userId }) + const baseURL = + activeSandboxChatOwner()?.mothershipBaseURL ?? (await getMothershipBaseURL({ userId })) const deadline = AbortSignal.timeout(15_000) const response = await fetchGo(`${baseURL}/api/workbench/bootstrap`, { headers: mothershipRequestHeaders(), @@ -147,6 +152,7 @@ export async function buildMothershipSandboxSession(args: { ? { SIM_ORGANIZATION_ID: args.organizationId } : { SIM_WORKSPACE: args.workspaceId! }), SIM_ENDPOINT: scopedEndpoint, + ...(activeSandboxFileOwnerProtocol() === 1 ? { SIM_FILE_OWNER_PROTOCOL: '1' } : {}), } } } catch (error) { diff --git a/apps/sim/lib/mothership/tools/server/base-tool.ts b/apps/sim/lib/mothership/tools/server/base-tool.ts index e018a525c48..146fd76afbe 100644 --- a/apps/sim/lib/mothership/tools/server/base-tool.ts +++ b/apps/sim/lib/mothership/tools/server/base-tool.ts @@ -1,6 +1,8 @@ import type { z } from 'zod' import type { BillingAttributionSnapshot } from '@/lib/billing/core/attribution' import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters' +import type { CopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' +import type { ExecutorDelegationOrigin } from '@/executor/types' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' export interface ServerToolContext { @@ -18,6 +20,11 @@ export interface ServerToolContext { toolCallId?: string /** True only for contexts built by the authenticated Copilot execution pipeline. */ copilotToolExecution?: boolean + copilotInteractionMode?: 'interactive' | 'headless' + copilotResourceAdmission?: CopilotResourceAdmission + mcpBlockId?: string + executorDelegationOrigin?: ExecutorDelegationOrigin + boundWorkflowExecutionId?: string billingAttribution?: BillingAttributionSnapshot userPermission?: string chatId?: string diff --git a/apps/sim/lib/mothership/tools/server/files/edit-roundtrip.test.ts b/apps/sim/lib/mothership/tools/server/files/edit-roundtrip.test.ts index 224672334b9..13ffcf24854 100644 --- a/apps/sim/lib/mothership/tools/server/files/edit-roundtrip.test.ts +++ b/apps/sim/lib/mothership/tools/server/files/edit-roundtrip.test.ts @@ -1,3 +1,4 @@ +import { readWorkspaceFileMetadataMock } from '@sim/testing/mocks/read-workspace-file-metadata.mock' import { generateId } from '@sim/utils/id' import { describe, expect, it, vi } from 'vitest' import type { ServerToolContext } from '@/lib/mothership/tools/server/base-tool' @@ -40,9 +41,10 @@ vi.mock('@/lib/workspace-files/application/delete-workspace-file', () => ({ vi.mock('@/lib/workspace-files/application/read-workspace-file-content', () => ({ readWorkspaceFileContent: { execute: vi.fn() }, })) -vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', () => ({ - readWorkspaceFileMetadata: { execute: vi.fn() }, -})) +vi.mock( + '@/lib/workspace-files/application/read-workspace-file-metadata', + () => readWorkspaceFileMetadataMock +) vi.mock('@/lib/workspace-files/application/rename-workspace-file', () => ({ renameWorkspaceFile: { execute: vi.fn() }, })) diff --git a/apps/sim/lib/mothership/tools/server/open-resource.test.ts b/apps/sim/lib/mothership/tools/server/open-resource.test.ts index 67610aab989..e09e77f6aa6 100644 --- a/apps/sim/lib/mothership/tools/server/open-resource.test.ts +++ b/apps/sim/lib/mothership/tools/server/open-resource.test.ts @@ -1,10 +1,13 @@ +import { + readWorkspaceFileMetadataMock, + readWorkspaceFileMetadataMockFns, +} from '@sim/testing/mocks/read-workspace-file-metadata.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' import { openResourceInputSchema } from '@/lib/api/contracts/mothership-resource-tools' import { OrchestrationError } from '@/lib/core/orchestration/types' const mocks = vi.hoisted(() => ({ workflow: vi.fn(), - file: vi.fn(), table: vi.fn(), view: vi.fn(), knowledge: vi.fn(), @@ -17,16 +20,14 @@ vi.mock('@/lib/workflows/application/read-workflow', async (original) => { readWorkflowMetadata: { ...actual.readWorkflowMetadata, execute: mocks.workflow }, } }) -vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', async (original) => { - const actual = - await original< - typeof import('@/lib/workspace-files/application/read-workspace-file-metadata') - >() - return { - ...actual, - readWorkspaceFileMetadata: { ...actual.readWorkspaceFileMetadata, execute: mocks.file }, - } -}) +vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', async () => ({ + ...readWorkspaceFileMetadataMock, + readWorkspaceFileMetadata: { + ...readWorkspaceFileMetadataMock.readWorkspaceFileMetadata, + operation: (await import('@/lib/workspace-files/application/operations')).fileOperations + .readMetadata, + }, +})) vi.mock('@/lib/table/application/tables', async (original) => { const actual = await original() return { @@ -52,6 +53,8 @@ vi.mock('@/lib/logs/application/read-log-detail', async (original) => { import { openResourceServerTool } from '@/lib/mothership/tools/server/open-resource' +const readFileMetadata = readWorkspaceFileMetadataMockFns.mockReadWorkspaceFileMetadata + const context = { userId: 'actor', workspaceId: 'ws-a', @@ -63,7 +66,7 @@ const invoke = (raw: unknown, ctx = context) => openResourceServerTool.execute(openResourceInputSchema.parse(raw), ctx) beforeEach(() => { mocks.workflow.mockResolvedValue({ workflow: { name: 'Canonical workflow' } }) - mocks.file.mockResolvedValue({ file: { name: 'Canonical file' } }) + readFileMetadata.mockResolvedValue({ file: { name: 'Canonical file' } }) mocks.table.mockResolvedValue({ table: { name: 'Canonical table' } }) mocks.view.mockResolvedValue({ view: { id: 'view' } }) mocks.knowledge.mockResolvedValue({ knowledgeBase: { name: 'Canonical knowledge' } }) @@ -117,7 +120,7 @@ describe('resource opening authorization boundary', () => { expect(mocks.workflow).not.toHaveBeenCalled() }) it('publishes no batch result after an authorization failure', async () => { - mocks.file.mockRejectedValue(new OrchestrationError('not_found', 'File not found')) + readFileMetadata.mockRejectedValue(new OrchestrationError('not_found', 'File not found')) await expect( invoke({ resources: [ diff --git a/apps/sim/lib/mothership/tools/server/open-resource.ts b/apps/sim/lib/mothership/tools/server/open-resource.ts index e98302b4708..863beec5925 100644 --- a/apps/sim/lib/mothership/tools/server/open-resource.ts +++ b/apps/sim/lib/mothership/tools/server/open-resource.ts @@ -12,7 +12,6 @@ import { logOperations } from '@/lib/logs/application/operations' import { readLogDetailUseCase } from '@/lib/logs/application/read-log-detail' import { createCopilotApplicationAdapter } from '@/lib/mothership/application/adapter' import { executeDashboardUseCase } from '@/lib/mothership/application/execute-dashboard-use-case' -import { executeCopilotFileUseCase } from '@/lib/mothership/application/execute-file-use-case' import { executeCopilotKnowledgeUseCase } from '@/lib/mothership/application/execute-knowledge-use-case' import { executeCopilotTableUseCase } from '@/lib/mothership/application/execute-table-use-case' import { executeWorkflowTestUseCase } from '@/lib/mothership/application/execute-workflow-test-use-case' @@ -21,6 +20,7 @@ import { COPILOT_APPLICATION_DELEGATION_TTL_MS, requireTrustedCopilotExecutionContext, } from '@/lib/mothership/auth/application-delegation' +import { getCopilotFileOwnerAdapter, isOwnerScopedFile } from '@/lib/mothership/file-owners' import { assertServerToolNotAborted, type BaseServerTool, @@ -29,7 +29,6 @@ import { readTableDefinitionUseCase } from '@/lib/table/application/tables' import { readTableViewUseCase } from '@/lib/table/application/views' import { getWorkflowTestDetail } from '@/lib/workflow-tests/application/tests' import { readWorkflowMetadata } from '@/lib/workflows/application/read-workflow' -import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/read-workspace-file-metadata' const executeLogUseCase = createCopilotApplicationAdapter({ domain: 'logs', @@ -47,15 +46,45 @@ export const openResourceServerTool: BaseServerTool resource.owner && isOwnerScopedFile(resource.owner))) { + const owned = input.resources.flatMap((resource) => + resource.type === 'file' && + resource.owner && + isOwnerScopedFile(resource.owner) && + !resource.viewId + ? [{ owner: resource.owner, fileId: resource.id }] + : [] + ) + if (input.workspaceId || owned.length !== input.resources.length) + throw new OrchestrationError( + 'validation', + 'Owner-scoped files cannot be mixed with workspace targets' + ) + const resources: OpenResourceOutput['resources'] = [] + for (const target of owned) { + assertServerToolNotAborted(context) + const file = await getCopilotFileOwnerAdapter(target.owner).readMetadata(context, target) + resources.push({ type: 'file', id: file.id, title: file.name, owner: file.owner }) + } + assertServerToolNotAborted(context) + return { resources } + } + const trusted = requireTrustedCopilotExecutionContext(context) if (input.workspaceId && input.workspaceId !== trusted.workspaceId) throw new OrchestrationError('not_found', 'Workspace not found in this invocation') const workspaceId = trusted.workspaceId const resources: OpenResourceOutput['resources'] = [] for (const resource of input.resources) { assertServerToolNotAborted(context) + if ( + resource.owner && + (resource.type !== 'file' || + isOwnerScopedFile(resource.owner) || + resource.owner.entityId !== workspaceId) + ) + throw new OrchestrationError('not_found', 'File owner not found in this invocation') if (resource.viewId && resource.type !== 'table') throw new OrchestrationError('validation', 'Saved views apply only to tables') const base = { type: resource.type, id: resource.id } @@ -107,13 +136,19 @@ export const openResourceServerTool: BaseServerTool = { + name: 'list_user_projects', + inputSchema: listUserProjectsInputSchema, + async execute(raw, context) { + try { + const result = await executeCopilotProjectDiscovery( + context, + listProjects, + listUserProjectsInputSchema.parse(raw) + ) + const capabilities = await mapWithConcurrency( + result.projects, + PROJECT_CAPABILITY_CONCURRENCY, + async (project) => { + const target = { projectId: project.id } + try { + return await executeCopilotProjectFileUseCase( + context, + getProjectFileCapabilities, + target, + target + ) + } catch (error) { + const code = asOrchestrationError(error)?.code + if (code !== 'forbidden' && code !== 'not_found' && code !== 'conflict') throw error + return null + } + } + ) + const fileOwnerCapabilities = capabilities.filter((capability) => capability !== null) + return { success: true, ...result, fileOwnerCapabilities } + } catch (error) { + return { + success: false, + message: messageForCopilotApplicationError(error, 'Project discovery failed'), + } + } + }, +} diff --git a/apps/sim/lib/mothership/tools/server/router.ts b/apps/sim/lib/mothership/tools/server/router.ts index 2e5711c2ae2..eafccac8e7c 100644 --- a/apps/sim/lib/mothership/tools/server/router.ts +++ b/apps/sim/lib/mothership/tools/server/router.ts @@ -31,6 +31,7 @@ import { ffmpegServerTool } from '@/lib/mothership/tools/server/media/ffmpeg' import { generateAudioServerTool } from '@/lib/mothership/tools/server/media/generate-audio' import { generateVideoServerTool } from '@/lib/mothership/tools/server/media/generate-video' import { openResourceServerTool } from '@/lib/mothership/tools/server/open-resource' +import { listProjectsServerTool } from '@/lib/mothership/tools/server/project-list' import { organizationSearchSourcesServerTool } from '@/lib/mothership/tools/server/search-sources' import { settingsServerTool } from '@/lib/mothership/tools/server/settings' import { testsServerTool } from '@/lib/mothership/tools/server/tests' @@ -70,6 +71,7 @@ const baseServerToolRegistry: Record = { [searchDocsServerTool.name]: searchDocsServerTool, [searchWorkspaceServerTool.name]: searchWorkspaceServerTool, [listWorkspacesServerTool.name]: listWorkspacesServerTool, + [listProjectsServerTool.name]: listProjectsServerTool, [workspacesServerTool.name]: workspacesServerTool, [organizationSearchSourcesServerTool.name]: organizationSearchSourcesServerTool, [settingsServerTool.name]: settingsServerTool, diff --git a/apps/sim/lib/projects/__integration__/foundation.integration.ts b/apps/sim/lib/projects/__integration__/foundation.integration.ts index d94ad56c2c6..5887c6dd848 100644 --- a/apps/sim/lib/projects/__integration__/foundation.integration.ts +++ b/apps/sim/lib/projects/__integration__/foundation.integration.ts @@ -34,6 +34,10 @@ import { removeUserFromOrganization, transferOrganizationOwnership, } from '@/lib/billing/organizations/membership' +import { createCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' +import { AgentCliRawResult } from '@/lib/mothership/generated/agent-cli' +import type { ToolExecutionContext } from '@/lib/mothership/tool-executor/types' +import { executeSimCli } from '@/lib/mothership/tools/handlers/sim-cli' import { prepareProjectsForAccountDeletion } from '@/lib/projects/account-deletion' import { archiveProject, @@ -1396,6 +1400,168 @@ describe('Project foundation at the database and application boundary', () => { }) describe('Project discovery delegation', () => { + beforeEach(() => { + vi.stubEnv('PROJECTS', 'true') + }) + const cliRequest = { + invocation: { kind: 'service', name: 'list_user_projects', input: { limit: 100 } }, + } + + check( + 'explicit Project CLI ownership reaches the authorized file collection without workspace fallback', + async () => { + const state = await fixture(false) + const context: ToolExecutionContext = { + userId: state.ownerId, + workflowId: '', + workspaceId: state.ids[0], + toolCallId: generateId(), + copilotToolExecution: true, + requestMode: 'agent', + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: state.ownerId, + invocation: { kind: 'workspace', workspaceId: state.ids[0] }, + }), + } + const request = { + fileOwner: { entityType: 'project', entityId: state.projectId }, + invocation: { + kind: 'cli', + argv: ['projects', 'files', 'list', state.projectId, '--output', 'json'], + }, + } + const result = await executeSimCli({ request }, context) + expect(result.success, result.error).toBe(true) + const output = AgentCliRawResult.parse(result.output) + expect(output.exitCode, output.stderr).toBe(0) + expect(JSON.parse(output.stdout)).toMatchObject({ data: [], nextCursor: null }) + const workspaceAttempt = await executeSimCli( + { + request: { + ...request, + invocation: { kind: 'cli', argv: ['files', 'list', '--output', 'json'] }, + }, + }, + context + ) + expect(workspaceAttempt.success).toBe(false) + expect(AgentCliRawResult.parse(workspaceAttempt.output).exitCode).not.toBe(0) + const foreign = await fixture(false) + const mismatch = await executeSimCli( + { + request: { + ...request, + invocation: { + kind: 'cli', + argv: ['projects', 'files', 'list', foreign.projectId, '--output', 'json'], + }, + }, + }, + context + ) + expect(mismatch.success).toBe(false) + expect(AgentCliRawResult.parse(mismatch.output).exitCode).not.toBe(0) + } + ) + + check( + 'CLI Project discovery carries admitted workspace identity to the real application boundary', + async () => { + const state = await fixture(false) + const context: ToolExecutionContext = { + userId: state.ownerId, + workflowId: '', + workspaceId: state.ids[0], + toolCallId: generateId(), + copilotToolExecution: true, + requestMode: 'agent', + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: state.ownerId, + invocation: { kind: 'workspace', workspaceId: state.ids[0] }, + }), + } + const result = await executeSimCli({ request: cliRequest }, context) + expect(result.success, result.error).toBe(true) + const output = AgentCliRawResult.parse(result.output) + expect(JSON.parse(output.stdout)).toMatchObject({ + success: true, + projects: [{ id: state.projectId }], + }) + for (const patch of [ + { copilotResourceAdmission: undefined }, + { copilotResourceAdmission: structuredClone(context.copilotResourceAdmission) }, + { userId: state.outsiderId }, + { workspaceId: state.ids[1] }, + { mcpBlockId: generateId() }, + { boundWorkflowExecutionId: generateId() }, + { executorDelegationOrigin: { workflowId: generateId(), subjectUserId: state.ownerId } }, + ]) { + const rejected = await executeSimCli( + { request: cliRequest, copilotResourceAdmission: context.copilotResourceAdmission }, + { ...context, ...patch } + ) + expect(rejected.success).toBe(false) + } + expect( + (await executeSimCli({ request: { ...cliRequest, workspaceId: state.ids[1] } }, context)) + .success + ).toBe(false) + } + ) + + check( + 'CLI Project discovery without an environment binds the owned organization chat and rechecks revocation', + async () => { + const state = await fixture() + const other = await fixture() + const chatId = generateId() + await db.insert(copilotChats).values({ + id: chatId, + userId: state.ownerId, + organizationId: state.organizationId, + type: 'mothership', + config: { conversationMode: 'agent' }, + }) + const context: ToolExecutionContext = { + userId: state.ownerId, + workflowId: '', + organizationId: state.organizationId ?? undefined, + chatOrganizationId: state.organizationId ?? undefined, + chatId, + toolCallId: generateId(), + copilotToolExecution: true, + requestMode: 'agent', + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: state.ownerId, + invocation: { kind: 'chat', chatId }, + }), + } + const result = await executeSimCli({ request: cliRequest }, context) + expect(result.success, result.error).toBe(true) + expect(JSON.parse(AgentCliRawResult.parse(result.output).stdout)).toMatchObject({ + success: true, + projects: [{ id: state.projectId }], + }) + expect( + ( + await executeSimCli( + { request: cliRequest }, + { + ...context, + organizationId: other.organizationId ?? undefined, + chatOrganizationId: other.organizationId ?? undefined, + } + ) + ).success + ).toBe(false) + await db + .update(copilotChats) + .set({ userId: state.outsiderId }) + .where(eq(copilotChats.id, chatId)) + expect((await executeSimCli({ request: cliRequest }, context)).success).toBe(false) + } + ) + check( 'binds organization discovery to the current owned chat and rechecks membership', async () => { diff --git a/apps/sim/lib/projects/application/index.ts b/apps/sim/lib/projects/application/index.ts index 2f3d714ebde..b3c9cc90775 100644 --- a/apps/sim/lib/projects/application/index.ts +++ b/apps/sim/lib/projects/application/index.ts @@ -2,6 +2,7 @@ export { createProject } from '@/lib/projects/application/create-project' export { projectOperations } from '@/lib/projects/application/operations' export { archiveProject, + getCurrentWorkspaceProject, getProject, getProjectIssueAccess, getWorkspaceProject, diff --git a/apps/sim/lib/projects/application/use-cases.ts b/apps/sim/lib/projects/application/use-cases.ts index cafe0777f37..a073ac43ae2 100644 --- a/apps/sim/lib/projects/application/use-cases.ts +++ b/apps/sim/lib/projects/application/use-cases.ts @@ -86,7 +86,7 @@ export const getProjectIssueAccess: OperationUseCase< export const listProjects: OperationUseCase< typeof projectOperations.list, - { organizationId?: string; cursor?: string; limit: number }, + { organizationId?: string; projectId?: string; cursor?: string; limit: number }, { projects: ReturnType[]; nextCursor: string | null } > = { operation: projectOperations.list, @@ -120,6 +120,7 @@ export const listProjects: OperationUseCase< ON ${project.id} = ${workspace.projectId} AND ${project.archivedAt} IS NULL WHERE TRUE ${scope.organizationId !== undefined ? sql`AND ${project.organizationId} IS NOT DISTINCT FROM ${scope.organizationId}` : sql``} + ${input.projectId !== undefined ? sql`AND ${project.id} = ${input.projectId}` : sql``} ${input.cursor ? sql`AND ${workspace.projectId} > ${input.cursor}` : sql``} ORDER BY 1 LIMIT ${input.limit + 1} @@ -250,3 +251,38 @@ export const getWorkspaceProject: OperationUseCase< }, READ_SNAPSHOT) }, } + +/** Discovers the canonical invocation workspace's parent without depending on discovery ordering. */ +export const getCurrentWorkspaceProject: OperationUseCase< + typeof projectOperations.list, + Record, + { project: ReturnType | null } +> = { + operation: projectOperations.list, + async execute({ principal }) { + requireProjectPrincipal(principal, projectOperations.list) + await requireProjectApiEnabled() + if (principal.kind !== 'resource_delegated') + throw new OrchestrationError('forbidden', 'Current Project discovery requires an invocation') + const scope = await resolveCopilotProjectScope(principal) + const workspaceId = scope.workspaceId + if (!workspaceId) return { project: null } + return db.transaction(async (tx) => { + const [membership] = await tx + .select({ projectId: workspace.projectId }) + .from(workspace) + .where(eq(workspace.id, workspaceId)) + .limit(1) + if (!membership?.projectId) return { project: null } + const [context] = await authorizeProjectsForRead(tx, principal, [membership.projectId]) + if ( + !context || + context.record.archivedAt || + context.record.organizationId !== scope.organizationId || + !context.environments.some((environment) => environment.id === workspaceId) + ) + throw new OrchestrationError('not_found', 'Project not found') + return { project: presentProject(context) } + }, READ_SNAPSHOT) + }, +} diff --git a/apps/sim/lib/projects/files/__integration__/authorization-product.integration.ts b/apps/sim/lib/projects/files/__integration__/authorization-product.integration.ts new file mode 100644 index 00000000000..a206ddd632c --- /dev/null +++ b/apps/sim/lib/projects/files/__integration__/authorization-product.integration.ts @@ -0,0 +1,440 @@ +import { mkdir, writeFile } from 'node:fs/promises' +import { createServer } from 'node:http' +import { dirname, resolve } from 'node:path' +import { db } from '@sim/db' +import { + copilotChats, + folder, + member, + organization, + permissions, + project, + user, + workspace, + workspaceFiles, +} from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { Logger } from '@sim/logger' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray } from 'drizzle-orm' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { copilotRequestPrincipal } from '@/lib/api/server/routes/copilot-request' +import { createProjectFileCliTransport } from '@/lib/mothership/agent-cli/project-file-transport' +import { markCopilotProjectFileRequest } from '@/lib/mothership/application/execute-project-file-use-case' +import { executeCopilotProjectDiscovery } from '@/lib/mothership/application/execute-project-use-case' +import { withFileOwnerContext } from '@/lib/mothership/application/file-owner-context' +import { + type CopilotExecutionContext, + createCopilotResourceAdmission, +} from '@/lib/mothership/auth/application-delegation' +import { listProjects } from '@/lib/projects/application' +import { getProjectFileMetadata } from '@/lib/projects/files/application' +import { projectFileOperations } from '@/lib/projects/files/application/operations' + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) + +const fixtures: { + users: string[] + organizationId: string + projectId: string + workspaces: string[] +}[] = [] +const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = + [] + +beforeEach(() => { + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation(async (flag) => flag === 'projects') +}) + +function check(name: string, run: () => Promise) { + it(name, async () => { + const started = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} + +async function fixture() { + const ownerId = generateId() + const readerId = generateId() + const organizationId = generateId() + const projectId = generateId() + const workspaces = [generateId(), generateId()] + fixtures.push({ users: [ownerId, readerId], organizationId, projectId, workspaces }) + await db.insert(user).values( + [ownerId, readerId].map((id) => ({ + id, + name: 'Project file fixture', + email: `${id}@files.invalid`, + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + })) + ) + await db.insert(organization).values({ + id: organizationId, + name: 'Project files', + slug: organizationId, + createdAt: new Date(), + }) + await db.insert(member).values([ + { id: generateId(), organizationId, userId: ownerId, role: 'owner', createdAt: new Date() }, + { id: generateId(), organizationId, userId: readerId, role: 'member', createdAt: new Date() }, + ]) + await db.transaction(async (tx) => { + await tx + .insert(project) + .values({ id: projectId, organizationId, ownerId, name: 'Project files' }) + await tx.insert(workspace).values( + workspaces.map((id, index) => ({ + id, + projectId, + organizationId, + ownerId, + billedAccountUserId: ownerId, + workspaceMode: 'organization' as const, + name: 'Environment', + forkedFromWorkspaceId: index ? workspaces[0] : null, + })) + ) + }) + await db.insert(permissions).values({ + id: generateId(), + entityType: 'workspace', + entityId: workspaces[0], + userId: readerId, + permissionType: 'read', + }) + return { + ownerId, + readerId, + organizationId, + projectId, + workspaces, + reader: createSessionPrincipal({ userId: readerId }), + } +} + +async function grant( + userId: string, + workspaceId: string, + permissionType: 'read' | 'write' | 'admin' +) { + await db + .delete(permissions) + .where( + and( + eq(permissions.userId, userId), + eq(permissions.entityType, 'workspace'), + eq(permissions.entityId, workspaceId) + ) + ) + await db.insert(permissions).values({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: workspaceId, + permissionType, + }) +} + +afterAll(async () => { + const reportPath = + process.env.PROJECT_FILE_AUTHORIZATION_PRODUCT_REPORT_PATH ?? + resolve('test-results/project-file-authorization-product.json') + await mkdir(dirname(reportPath), { recursive: true }) + await writeFile(reportPath, JSON.stringify({ checks }, null, 2)) + for (const f of fixtures) { + const memberships = await db + .select({ projectId: workspace.projectId }) + .from(workspace) + .where(inArray(workspace.id, f.workspaces)) + const projectIds = [f.projectId, ...memberships.map((row) => row.projectId)] + await db.delete(workspaceFiles).where(inArray(workspaceFiles.projectId, projectIds)) + await db.delete(folder).where(inArray(folder.projectId, projectIds)) + await deleteWorkspaceFixture(db, inArray(workspace.id, f.workspaces)) + await db.delete(project).where(inArray(project.id, projectIds)) + await db.delete(organization).where(eq(organization.id, f.organizationId)) + await db.delete(user).where(inArray(user.id, f.users)) + } +}) + +describe('Project file authority at the database boundary', () => { + check( + 'private v2 admission preserves the actor and rechecks live Project membership', + async () => { + const f = await fixture() + const chatId = generateId() + const fileId = generateId() + await db.insert(copilotChats).values({ + id: chatId, + userId: f.readerId, + organizationId: f.organizationId, + workspaceId: null, + type: 'mothership', + config: { conversationMode: 'agent' }, + title: 'Private Project files', + }) + await db.insert(workspaceFiles).values({ + id: fileId, + userId: f.ownerId, + projectId: f.projectId, + context: 'project', + key: `project/${f.projectId}/${fileId}`, + originalName: 'architecture.md', + contentType: 'text/markdown', + sizeBytes: 0, + }) + const request = new Request( + `http://localhost/api/v2/projects/${f.projectId}/files/${fileId}/metadata` + ) + markCopilotProjectFileRequest( + request, + { + userId: f.readerId, + organizationId: f.organizationId, + chatId, + toolCallId: generateId(), + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: f.readerId, + invocation: { kind: 'chat', chatId }, + }), + }, + { projectId: f.projectId, fileId } + ) + const principal = copilotRequestPrincipal( + request, + projectFileOperations.readMetadata, + getProjectFileMetadata + ) + if (!principal) throw new Error('Private Project admission did not produce a principal') + const input = { projectId: f.projectId, fileId } + expect(await getProjectFileMetadata.execute({ principal, input })).toMatchObject({ + file: { id: fileId, owner: { entityType: 'project', entityId: f.projectId } }, + capabilities: { canRead: true, canWrite: false }, + }) + await expect( + getProjectFileMetadata.execute({ principal, input: { ...input, fileId: generateId() } }) + ).rejects.toMatchObject({ code: 'forbidden' }) + await db.delete(permissions).where(eq(permissions.userId, f.readerId)) + await expect(getProjectFileMetadata.execute({ principal, input })).rejects.toMatchObject({ + code: 'not_found', + }) + } + ) + + check( + 'Project-only v2 transport reaches the canonical route and cannot change owners', + async () => { + const f = await fixture() + const fileId = generateId() + await db.insert(workspaceFiles).values({ + id: fileId, + userId: f.ownerId, + projectId: f.projectId, + context: 'project', + key: `project/${f.projectId}/${fileId}`, + originalName: 'transport.md', + contentType: 'text/markdown', + sizeBytes: 0, + }) + const context: CopilotExecutionContext = { + userId: f.readerId, + workspaceId: f.workspaces[0], + toolCallId: generateId(), + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: f.readerId, + invocation: { kind: 'workspace', workspaceId: f.workspaces[0] }, + }), + } + const endpoint = 'http://localhost:3000' + const filePath = `/api/v2/projects/${f.projectId}/files/${fileId}/metadata` + const transport = createProjectFileCliTransport(endpoint, context, { + projectId: f.projectId, + fileId, + }) + const response = await transport(`${endpoint}${filePath}`) + expect(response.status).toBe(200) + expect(await response.json()).toMatchObject({ + data: { id: fileId, owner: { entityType: 'project', entityId: f.projectId } }, + }) + for (const url of [ + `http://other.invalid${filePath}`, + `${endpoint}/api/v2/projects/${generateId()}/files/${fileId}/metadata`, + `${endpoint}/api/v2/projects/${f.projectId}/files/${generateId()}/metadata`, + `${endpoint}/api/v2/projects/${f.projectId}/files/folders`, + `${endpoint}/api/v2/files/${fileId}/metadata?workspaceId=${f.workspaces[0]}`, + ]) { + expect((await transport(url)).status).toBe(400) + } + const projectTransport = createProjectFileCliTransport(endpoint, context, { + projectId: f.projectId, + }) + expect( + (await projectTransport(`${endpoint}/api/v2/projects/${f.projectId}/files`)).status + ).toBe(200) + expect((await projectTransport(`${endpoint}${filePath}`)).status).toBe(200) + context.boundWorkflowExecutionId = 'workflow-run' + expect((await transport(`${endpoint}${filePath}`)).status).toBe(403) + } + ) +}) + +describe('Mothership origin navigation and independent Project authority', () => { + async function target(f: Awaited>) { + const workspaceId = generateId() + f.workspaces.push(workspaceId) + await insertWorkspaceFixture(db, { + id: workspaceId, + name: 'Independent target', + organizationId: f.organizationId, + ownerId: f.ownerId, + billedAccountUserId: f.ownerId, + workspaceMode: 'organization', + }) + const [binding] = await db.select().from(workspace).where(eq(workspace.id, workspaceId)) + if (!binding) throw new Error('Target Project missing') + await grant(f.readerId, workspaceId, 'admin') + return { workspaceId, projectId: binding.projectId } + } + + function context(f: Awaited>): CopilotExecutionContext { + return { + userId: f.readerId, + workspaceId: f.workspaces[0], + requestMode: 'agent', + toolCallId: generateId(), + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: f.readerId, + invocation: { kind: 'workspace', workspaceId: f.workspaces[0] }, + }), + } + } + + /** A worker that advertises the file-owner protocol, so turn hints are actually negotiated. */ + async function withProtocolPeer(run: (workerURL: string) => Promise) { + const peer = createServer((_request, response) => { + response.setHeader('X-Mothership-File-Owner-Protocol', '1') + response.end() + }) + await new Promise((resolve) => peer.listen(0, '127.0.0.1', resolve)) + try { + const address = peer.address() + if (!address || typeof address === 'string') throw new Error('Protocol peer unavailable') + await run(`http://127.0.0.1:${address.port}`) + } finally { + await new Promise((resolve, reject) => + peer.close((error) => (error ? reject(error) : resolve())) + ) + } + } + + check('origin A discovers accessible B while current-parent hints remain A', async () => { + const f = await fixture() + const b = await target(f) + const execution = context(f) + const listed = await executeCopilotProjectDiscovery(execution, listProjects, { limit: 100 }) + expect(listed.projects.map((row) => row.id).sort()).toEqual([f.projectId, b.projectId].sort()) + await withProtocolPeer(async (workerURL) => { + for (const [workspaceId, projectId] of [ + [f.workspaces[0], f.projectId], + [b.workspaceId, b.projectId], + ]) { + const current = { + ...execution, + workspaceId, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: f.readerId, + invocation: { kind: 'workspace', workspaceId }, + }), + } + const hints = await withFileOwnerContext({}, current, workerURL, '/api/mothership') + expect(hints.project).toMatchObject({ id: projectId }) + } + }) + await db + .delete(permissions) + .where(and(eq(permissions.userId, f.readerId), eq(permissions.entityId, b.workspaceId))) + const revoked = await executeCopilotProjectDiscovery(execution, listProjects, { limit: 100 }) + expect(revoked.projects.map((row) => row.id)).toEqual([f.projectId]) + }) + + check( + 'sends the turn without Project hints, warning only when the lookup fails unexpectedly', + async () => { + const f = await fixture() + const forged = { + message: 'hello', + fileOwnerProtocolVersion: 1, + project: { id: generateId(), name: 'Forged' }, + fileOwnerCapabilities: [ + { + owner: { entityType: 'project', entityId: generateId() }, + canRead: true, + canWrite: true, + }, + ], + } + const noProject = { + message: 'hello', + fileOwnerProtocolVersion: 1, + project: null, + fileOwnerCapabilities: [], + } + const warn = vi.spyOn(Logger.prototype, 'warn') + const hintWarnings = () => + warn.mock.calls.filter(([message]) => + message.includes('Sending turn without Project file owner hints') + ) + try { + await withProtocolPeer(async (workerURL) => { + const send = () => withFileOwnerContext(forged, context(f), workerURL, '/api/mothership') + + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation(async () => false) + expect(await send()).toEqual({ message: 'hello' }) + + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation( + async (flag) => flag === 'projects' + ) + expect((await send()).project).toEqual({ id: f.projectId, name: 'Project files' }) + + await db + .update(project) + .set({ archivedAt: new Date() }) + .where(eq(project.id, f.projectId)) + expect(await send()).toEqual(noProject) + await db.update(project).set({ archivedAt: null }).where(eq(project.id, f.projectId)) + + await db.delete(permissions).where(eq(permissions.userId, f.readerId)) + expect(await send()).toEqual(noProject) + expect(hintWarnings()).toEqual([]) + + /** The rollout switches off between admitting the turn and reading its Project. */ + featureFlagsMockFns.mockIsFeatureEnabled + .mockImplementation(async () => false) + .mockResolvedValueOnce(true) + expect(await send()).toEqual(noProject) + expect(hintWarnings()).toHaveLength(1) + }) + } finally { + warn.mockRestore() + } + } + ) +}) diff --git a/apps/sim/lib/projects/files/__integration__/content-product.integration.ts b/apps/sim/lib/projects/files/__integration__/content-product.integration.ts new file mode 100644 index 00000000000..5d1cdfbd3f5 --- /dev/null +++ b/apps/sim/lib/projects/files/__integration__/content-product.integration.ts @@ -0,0 +1,784 @@ +import { mkdtempSync } from 'node:fs' +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { db } from '@sim/db' +import { + copilotAsyncToolCalls, + copilotChats, + copilotRuns, + folder, + idempotencyKey, + member, + organization, + outboxEvent, + permissions, + subscription, + user, + workspace, + workspaceFileSecretProvenance, + workspaceFiles, +} from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { sha256Hex } from '@sim/security/hash' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { emailMailerMock } from '@sim/testing/mocks/email-mailer.mock' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { setUploadDirServer, uploadsSetupMock } from '@sim/testing/mocks/uploads-setup.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, sql } from 'drizzle-orm' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { redisUrl } = await vi.hoisted(async () => { + const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') + const redisUrl = readTestRedisUrl() + if (redisUrl) process.env.REDIS_URL = redisUrl + return { redisUrl } +}) + +vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock) +vi.mock('@/lib/messaging/email/mailer', () => emailMailerMock) + +import { v2CopyFileItemsContract } from '@/lib/api/contracts/v2/file-copy' +import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' +import { executeAgentCliRequest } from '@/lib/mothership/agent-cli' +import { createFileCopyCliTransport } from '@/lib/mothership/agent-cli/file-copy-transport' +import { createCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' +import { proxySandboxResourceRequest } from '@/lib/mothership/tools/sandbox-resource-transport' +import { listProjectsServerTool } from '@/lib/mothership/tools/server/project-list' +import { createProjectFile } from '@/lib/projects/files/application/content' +import { exportProjectFileSnapshot } from '@/lib/projects/files/application/downloads' +import { createProjectFileFolder } from '@/lib/projects/files/application/folders' +import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace/file-manager' +import { replaceWorkspaceFileSecretProvenanceInTx } from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import * as storage from '@/lib/uploads/core/storage-service' + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) + +const localStorageRoot = mkdtempSync(join(tmpdir(), 'sim-project-content-')) +setUploadDirServer(localStorageRoot) +const fixtures: { + ownerId: string + editorId: string + organizationId: string + workspaceId: string + projectId: string +}[] = [] +const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = + [] + +beforeEach(() => { + vi.restoreAllMocks() + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation(async (flag) => flag === 'projects') + vi.stubEnv('FREE_STORAGE_LIMIT_GB', '') +}) + +function check(name: string, run: () => Promise) { + it(name, async () => { + const started = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} + +async function fixture() { + const ownerId = generateId() + const editorId = generateId() + const organizationId = generateId() + const workspaceId = generateId() + await db.insert(user).values( + [ownerId, editorId].map((id) => ({ + id, + email: `${id}@content.invalid`, + name: 'File content fixture', + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + })) + ) + await db + .insert(organization) + .values({ id: organizationId, name: 'Content', slug: organizationId, createdAt: new Date() }) + await db.insert(member).values( + [ownerId, editorId].map((userId) => ({ + id: generateId(), + organizationId, + userId, + role: userId === ownerId ? 'owner' : 'member', + createdAt: new Date(), + })) + ) + await insertWorkspaceFixture(db, { + id: workspaceId, + ownerId, + billedAccountUserId: ownerId, + organizationId, + workspaceMode: 'organization', + name: 'Content environment', + }) + const [binding] = await db + .select({ projectId: workspace.projectId }) + .from(workspace) + .where(eq(workspace.id, workspaceId)) + if (!binding?.projectId) throw new Error('Project fixture missing') + await db.insert(permissions).values({ + id: generateId(), + userId: editorId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: 'admin', + }) + const ids = { ownerId, editorId, organizationId, workspaceId, projectId: binding.projectId } + fixtures.push(ids) + return { ...ids, principal: createSessionPrincipal({ userId: editorId }) } +} + +function createInput(projectId: string, content = 'Project architecture') { + return { + projectId, + name: 'architecture.md', + contentType: 'text/markdown', + content, + encoding: 'utf-8' as const, + exactName: true, + } +} + +async function ledger(organizationId: string) { + const [row] = await db + .select({ bytes: organization.storageUsedBytes }) + .from(organization) + .where(eq(organization.id, organizationId)) + return row?.bytes +} + +check( + 'Project discovery accepts its opaque pagination cursor through the tool boundary', + async () => { + const f = await fixture() + const result = await listProjectsServerTool.execute( + { cursor: 'zz-legacy-project', limit: 1 }, + { + userId: f.editorId, + workspaceId: f.workspaceId, + toolCallId: generateId(), + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: f.editorId, + invocation: { kind: 'workspace', workspaceId: f.workspaceId }, + }), + } + ) + expect(result).toMatchObject({ success: true, projects: [], nextCursor: null }) + } +) + +describe.skipIf(!redisUrl)('authenticated workbench copy callback', () => { + check( + 'organization callback copies both owners with current access, provenance and durable effects', + async () => { + const f = await fixture() + const foreign = await fixture() + const destinationWorkspaceId = generateId() + await insertWorkspaceFixture(db, { + id: destinationWorkspaceId, + ownerId: f.ownerId, + billedAccountUserId: f.ownerId, + organizationId: f.organizationId, + workspaceMode: 'organization', + name: 'Copy destination', + }) + const [destinationBinding] = await db + .select({ projectId: workspace.projectId }) + .from(workspace) + .where(eq(workspace.id, destinationWorkspaceId)) + if (!destinationBinding?.projectId) throw new Error('Destination Project missing') + const destinationPermissionId = generateId() + await db.insert(permissions).values({ + id: destinationPermissionId, + userId: f.editorId, + entityType: 'workspace', + entityId: destinationWorkspaceId, + permissionType: 'write', + }) + const source = await createProjectFile.execute({ + principal: f.principal, + input: createInput(f.projectId, 'callback copy bytes'), + }) + const evidence = { + status: 'exact' as const, + entries: [ + { + encryptedValue: 'synthetic-encrypted-evidence', + sourceUserId: f.editorId, + sourceWorkspaceId: f.workspaceId, + }, + ], + } + const [sourceRow] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.id, source.file.id)) + await db.transaction((tx) => + replaceWorkspaceFileSecretProvenanceInTx( + tx, + source.file.id, + sourceRow.contentUpdatedAt, + evidence + ) + ) + const [sourceProvenance] = await db + .select() + .from(workspaceFileSecretProvenance) + .where(eq(workspaceFileSecretProvenance.fileId, source.file.id)) + const chatId = generateId() + await db.insert(copilotChats).values({ + id: chatId, + userId: f.editorId, + organizationId: f.organizationId, + type: 'mothership', + config: { conversationMode: 'agent' }, + }) + const token = generateId() + const apiKey = generateId() + const scope = { + toolCallId: generateId(), + runId: generateId(), + userId: f.editorId, + ownerToken: generateId(), + chatId, + organizationId: f.organizationId, + apiKeyHash: sha256Hex(apiKey), + fileOwnerProtocolVersion: 1 as const, + } + await db.insert(copilotRuns).values({ + id: scope.runId, + executionId: generateId(), + streamId: generateId(), + chatId, + userId: f.editorId, + organizationId: f.organizationId, + }) + await db.insert(copilotAsyncToolCalls).values({ + runId: scope.runId, + toolCallId: scope.toolCallId, + toolName: 'run_code', + executionOwnerToken: scope.ownerToken, + executionStartedAt: new Date(), + executionLeaseExpiresAt: new Date(Date.now() + 120_000), + }) + const redis = getRedisClient() + if (!redis) throw new Error('Expected callback Redis') + const prefix = `mothership:sandbox-resources:${token}` + await redis.set(`${prefix}:context`, JSON.stringify(scope), 'EX', 120) + const sourceOwner = { entityType: 'project' as const, entityId: f.projectId } + const destinationOwner = { + entityType: 'workspace' as const, + entityId: destinationWorkspaceId, + } + const body = { + source: { owner: sourceOwner, fileIds: [source.file.id], folderIds: [] }, + destination: { owner: destinationOwner, folderId: null }, + } + const callback = ( + value: unknown = body, + headers: Record = {}, + method = 'POST' + ) => + proxySandboxResourceRequest( + new Request(`http://localhost:3000/api/mothership/sandbox/${token}/api/v2/files/copy`, { + method, + headers: { 'content-type': 'application/json', 'x-api-key': apiKey, ...headers }, + ...(method === 'POST' ? { body: JSON.stringify(value) } : {}), + }), + token + ) + const destinationRows = () => + db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.workspaceId, destinationWorkspaceId)) + try { + const controller = new AbortController() + controller.abort() + await expect( + proxySandboxResourceRequest( + new Request(`http://localhost:3000/api/mothership/sandbox/${token}/api/v2/files/copy`, { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify(body), + signal: controller.signal, + }), + token + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(await destinationRows()).toEqual([]) + const response = await callback() + expect(response.status, await response.clone().text()).toBe(201) + const result = v2CopyFileItemsContract.response.schema.parse(await response.json()).data + const [copied] = await destinationRows() + expect(copied).toMatchObject({ + id: result.files[0]?.id, + userId: f.editorId, + workspaceId: destinationWorkspaceId, + projectId: null, + }) + expect(await readFile(join(localStorageRoot, copied.key), 'utf8')).toBe( + 'callback copy bytes' + ) + const [provenance] = await db + .select() + .from(workspaceFileSecretProvenance) + .where(eq(workspaceFileSecretProvenance.fileId, copied.id)) + expect(provenance).toMatchObject({ + status: sourceProvenance.status, + entries: sourceProvenance.entries, + }) + const effects = (await redis.lrange(`${prefix}:inbox`, 0, -1)).map((entry) => + JSON.parse(entry) + ) + expect(effects).toContainEqual( + expect.objectContaining({ + op: 'upsert', + resource: expect.objectContaining({ id: copied.id, owner: destinationOwner }), + }) + ) + const [chat] = await db.select().from(copilotChats).where(eq(copilotChats.id, chatId)) + expect(JSON.stringify(chat)).toContain(copied.id) + const reverse = await callback({ + source: { owner: destinationOwner, fileIds: [copied.id], folderIds: [] }, + destination: { + owner: { entityType: 'project', entityId: destinationBinding.projectId }, + folderId: null, + }, + }) + expect(reverse.status, await reverse.clone().text()).toBe(201) + for (const input of [ + { + ...body, + destination: { + owner: { entityType: 'project', entityId: destinationBinding.projectId }, + folderId: null, + }, + }, + { + source: { owner: destinationOwner, fileIds: [copied.id], folderIds: [] }, + destination: { + owner: { entityType: 'workspace', entityId: f.workspaceId }, + folderId: null, + }, + }, + ]) { + const copiedResponse = await callback(input) + expect(copiedResponse.status, await copiedResponse.clone().text()).toBe(201) + } + await db + .update(copilotChats) + .set({ organizationId: null, workspaceId: f.workspaceId }) + .where(eq(copilotChats.id, chatId)) + await db + .update(copilotRuns) + .set({ organizationId: null, workspaceId: f.workspaceId }) + .where(eq(copilotRuns.id, scope.runId)) + await redis.set( + `${prefix}:context`, + JSON.stringify({ ...scope, organizationId: undefined, workspaceId: f.workspaceId }), + 'EX', + 120 + ) + expect((await callback()).status).toBe(201) + await db + .update(copilotChats) + .set({ organizationId: f.organizationId, workspaceId: null }) + .where(eq(copilotChats.id, chatId)) + await db + .update(copilotRuns) + .set({ organizationId: f.organizationId, workspaceId: null }) + .where(eq(copilotRuns.id, scope.runId)) + await redis.set(`${prefix}:context`, JSON.stringify(scope), 'EX', 120) + vi.stubEnv('PROJECTS', 'false') + featureFlagsMockFns.mockIsFeatureEnabled.mockResolvedValue(false) + expect((await callback()).status).toBe(503) + const deniedProjectRead = await proxySandboxResourceRequest( + new Request( + `http://localhost:3000/api/mothership/sandbox/${token}/api/v2/projects/${f.projectId}/files`, + { + headers: { + 'x-api-key': apiKey, + 'x-mothership-file-owner': JSON.stringify(sourceOwner), + }, + } + ), + token + ) + expect(deniedProjectRead.status).toBe(503) + vi.stubEnv('PROJECTS', 'true') + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation( + async (flag) => flag === 'projects' + ) + expect((await callback()).status).toBe(201) + const baseline = await destinationRows() + const invalidHeaders: Record[] = [ + { 'x-api-key': generateId() }, + { 'x-mothership-workspace-id': f.workspaceId }, + { 'x-mothership-file-owner': JSON.stringify(sourceOwner) }, + ] + for (const headers of invalidHeaders) + expect((await callback(body, headers)).status).toBe(403) + expect((await callback(body, {}, 'GET')).status).toBe(405) + expect((await callback({ ...body, workspaceId: f.workspaceId })).status).toBe(400) + await redis.set( + `${prefix}:context`, + JSON.stringify({ ...scope, fileOwnerProtocolVersion: undefined }), + 'EX', + 120 + ) + expect((await callback()).status).toBe(403) + await redis.set(`${prefix}:context`, JSON.stringify(scope), 'EX', 120) + for (const invalid of [ + { + ...body, + source: { + ...body.source, + owner: { entityType: 'project', entityId: foreign.projectId }, + }, + }, + { + ...body, + destination: { + ...body.destination, + owner: { entityType: 'workspace', entityId: foreign.workspaceId }, + }, + }, + { ...body, source: { ...body.source, fileIds: [generateId()] } }, + { ...body, destination: { ...body.destination, folderId: generateId() } }, + ]) + expect((await callback(invalid)).ok).toBe(false) + await db + .update(permissions) + .set({ permissionType: 'read' }) + .where(eq(permissions.id, destinationPermissionId)) + expect((await callback()).ok).toBe(false) + await db + .update(permissions) + .set({ permissionType: 'write' }) + .where(eq(permissions.id, destinationPermissionId)) + const [sourcePermission] = await db + .select() + .from(permissions) + .where(and(eq(permissions.userId, f.editorId), eq(permissions.entityId, f.workspaceId))) + await db.delete(permissions).where(eq(permissions.id, sourcePermission.id)) + expect((await callback()).ok).toBe(false) + await db.insert(permissions).values(sourcePermission) + const download = storage.downloadFile + let revoked = false + const read = vi.spyOn(storage, 'downloadFile').mockImplementation(async (options) => { + const bytes = await download(options) + if (!revoked) { + revoked = true + await db + .update(permissions) + .set({ permissionType: 'read' }) + .where(eq(permissions.id, destinationPermissionId)) + } + return bytes + }) + try { + expect((await callback()).ok).toBe(false) + expect(revoked).toBe(true) + } finally { + read.mockRestore() + } + await db + .update(permissions) + .set({ permissionType: 'write' }) + .where(eq(permissions.id, destinationPermissionId)) + await db + .update(copilotAsyncToolCalls) + .set({ executionRevokedAt: new Date() }) + .where(eq(copilotAsyncToolCalls.toolCallId, scope.toolCallId)) + expect((await callback()).status).toBe(403) + expect(await destinationRows()).toEqual(baseline) + } finally { + await redis.del(`${prefix}:context`, `${prefix}:inbox`, `${prefix}:seen`) + await db + .delete(workspaceFiles) + .where(eq(workspaceFiles.projectId, destinationBinding.projectId)) + await deleteWorkspaceFixture(db, eq(workspace.id, destinationWorkspaceId)) + } + } + ) +}) + +describe('private compound copy transport', () => { + check( + 'native paired copy dispatch returns the destination resource and persists the current actor bytes', + async () => { + const f = await fixture() + const original = await uploadWorkspaceFile( + f.workspaceId, + f.ownerId, + Buffer.from('native paired copy bytes'), + 'native-copy.txt', + 'text/plain' + ) + const destinationFolder = await createProjectFileFolder.execute({ + principal: f.principal, + input: { projectId: f.projectId, name: 'Architecture' }, + }) + const target = { + source: { + owner: { entityType: 'workspace' as const, entityId: f.workspaceId }, + fileIds: [original.id], + folderIds: [], + }, + destination: { + owner: { entityType: 'project' as const, entityId: f.projectId }, + folderId: destinationFolder.folder.id, + }, + } + const result = await executeAgentCliRequest( + { + invocation: { + kind: 'file-copy', + ...target, + argv: [ + 'files', + 'copy', + '--source', + JSON.stringify(target.source), + '--destination', + JSON.stringify(target.destination), + ], + }, + }, + { + userId: f.editorId, + workspaceId: f.workspaceId, + toolCallId: generateId(), + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: f.editorId, + invocation: { kind: 'workspace', workspaceId: f.workspaceId }, + }), + } + ) + expect(result, result.stderr).toMatchObject({ exitCode: 0 }) + const [copied] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.projectId, f.projectId)) + if (!copied) throw new Error('Native paired copy created no durable Project file') + expect(copied.id).not.toBe(original.id) + expect(copied.userId).toBe(f.editorId) + expect(copied.folderId).toBe(destinationFolder.folder.id) + expect(await storage.downloadFile({ key: copied.key, context: 'project' })).toEqual( + Buffer.from('native paired copy bytes') + ) + expect(result.resources).toContainEqual({ + op: 'upsert', + resource: { + type: 'file', + owner: target.destination.owner, + id: copied.id, + title: 'native-copy.txt', + path: `projects/${f.projectId}/files/Architecture/native-copy.txt`, + }, + }) + } + ) + + check( + 'private copy transport retains the acting user through the real v2 route and stored bytes', + async () => { + const f = await fixture() + const original = await createProjectFile.execute({ + principal: createSessionPrincipal({ userId: f.ownerId }), + input: createInput(f.projectId, 'private compound copy bytes'), + }) + const input = { + source: { + owner: { entityType: 'project' as const, entityId: f.projectId }, + fileIds: [original.file.id], + folderIds: [], + }, + destination: { + owner: { entityType: 'workspace' as const, entityId: f.workspaceId }, + folderId: null, + }, + } + const transport = createFileCopyCliTransport( + 'http://localhost:3000', + { + userId: f.editorId, + workspaceId: f.workspaceId, + toolCallId: generateId(), + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: f.editorId, + invocation: { kind: 'workspace', workspaceId: f.workspaceId }, + }), + }, + input + ) + const before = await ledger(f.organizationId) + const response = await transport('http://localhost:3000/api/v2/files/copy', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(input), + }) + expect(response.status).toBe(201) + const [copied] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.workspaceId, f.workspaceId)) + if (!copied || copied.sizeBytes === null) + throw new Error('Private copy created no sized durable file') + expect(copied.userId).toBe(f.editorId) + expect(copied.id).not.toBe(original.file.id) + expect(await storage.downloadFile({ key: copied.key, context: 'workspace' })).toEqual( + Buffer.from('private compound copy bytes') + ) + expect(await ledger(f.organizationId)).toBe(before + copied.sizeBytes) + } + ) + + check( + 'private copy transport binds immutable selections and refuses changed owners, routes and revoked access', + async () => { + const f = await fixture() + const original = await createProjectFile.execute({ + principal: f.principal, + input: createInput(f.projectId), + }) + const input = { + source: { + owner: { entityType: 'project' as const, entityId: f.projectId }, + fileIds: [original.file.id], + folderIds: [], + }, + destination: { + owner: { entityType: 'workspace' as const, entityId: f.workspaceId }, + folderId: null, + }, + } + const expected = structuredClone(input) + const transport = createFileCopyCliTransport( + 'http://localhost:3000', + { + userId: f.editorId, + workspaceId: f.workspaceId, + toolCallId: generateId(), + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: f.editorId, + invocation: { kind: 'workspace', workspaceId: f.workspaceId }, + }), + }, + input + ) + input.source.fileIds.push(generateId()) + for (const body of [ + input, + { ...expected, destination: { ...expected.destination, folderId: generateId() } }, + { + ...expected, + source: { ...expected.source, owner: { entityType: 'project', entityId: generateId() } }, + }, + ]) { + expect( + ( + await transport('http://localhost:3000/api/v2/files/copy', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + ).status + ).toBe(403) + } + expect( + (await transport(`http://localhost:3000/api/v2/projects/${f.projectId}/files`)).status + ).toBe(400) + expect( + ( + await transport('http://other.invalid/api/v2/files/copy', { + method: 'POST', + body: JSON.stringify(expected), + }) + ).status + ).toBe(400) + await db.delete(permissions).where(eq(permissions.userId, f.editorId)) + expect( + ( + await transport('http://localhost:3000/api/v2/files/copy', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(expected), + }) + ).status + ).toBe(403) + expect( + await db.select().from(workspaceFiles).where(eq(workspaceFiles.workspaceId, f.workspaceId)) + ).toEqual([]) + } + ) +}) + +describe('Project archive download and Markdown snapshot export', () => { + check( + 'Project Markdown snapshot export accepts parameterized alias MIME without a Markdown extension', + async () => { + const f = await fixture() + const source = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'draft'), + name: 'notes.txt', + contentType: 'Text/X-Markdown; charset=utf-8', + }, + }) + const exported = await exportProjectFileSnapshot.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id, content: '# Selected draft' }, + }) + expect(exported.buffer.toString('utf8')).toBe('# Selected draft') + } + ) +}) + +afterAll(async () => { + vi.restoreAllMocks() + for (const f of fixtures) { + await db.delete(idempotencyKey).where(sql`${idempotencyKey.key} LIKE ${`%${f.projectId}%`}`) + await db + .delete(outboxEvent) + .where( + sql`${outboxEvent.payload}::jsonb ->> 'key' LIKE ${`project/${f.projectId}/%`} OR ${outboxEvent.payload}::jsonb -> 'owner' ->> 'entityId' = ${f.projectId}` + ) + await db.delete(workspaceFiles).where(eq(workspaceFiles.projectId, f.projectId)) + await db.delete(folder).where(eq(folder.projectId, f.projectId)) + await deleteWorkspaceFixture(db, eq(workspace.id, f.workspaceId)) + await db.delete(subscription).where(eq(subscription.referenceId, f.organizationId)) + await db.delete(organization).where(eq(organization.id, f.organizationId)) + await db.delete(user).where(inArray(user.id, [f.ownerId, f.editorId])) + } + await rm(localStorageRoot, { recursive: true, force: true }) + const report = + process.env.PROJECT_FILE_CONTENT_PRODUCT_REPORT_PATH ?? + 'test-results/project-file-content-product.json' + await mkdir(dirname(report), { recursive: true }) + await writeFile(report, JSON.stringify({ checks }, null, 2)) + await closeRedisConnection() + await db.$client.end() +}) diff --git a/apps/sim/lib/projects/files/__integration__/uploads.integration.ts b/apps/sim/lib/projects/files/__integration__/uploads.integration.ts index 540aa167293..99757c40fc1 100644 --- a/apps/sim/lib/projects/files/__integration__/uploads.integration.ts +++ b/apps/sim/lib/projects/files/__integration__/uploads.integration.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import { mkdtempSync } from 'node:fs' import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -26,21 +27,32 @@ import { generateId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' import { and, eq, inArray, sql } from 'drizzle-orm' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) -await vi.hoisted(async () => { +const { redisUrl } = await vi.hoisted(async () => { const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') const redisUrl = readTestRedisUrl() if (redisUrl) process.env.REDIS_URL = redisUrl + return { redisUrl } }) vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock) -import { closeRedisConnection } from '@/lib/core/config/redis' +import { + v2CompleteProjectFileUploadContract, + v2CreateProjectFileUploadContract, +} from '@/lib/api/contracts/v2/project-file-uploads' +import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' +import { createProjectFileCliTransport } from '@/lib/mothership/agent-cli/project-file-transport' +import { createProjectFileUploadTransport } from '@/lib/mothership/agent-cli/project-file-upload-transport' +import { createWorkbenchFileProvenance } from '@/lib/mothership/agent-cli/workbench-file-provenance' +import { createCopilotResourceAdmission } from '@/lib/mothership/auth/application-delegation' import * as application from '@/lib/projects/files/application' import * as prefixCleanup from '@/lib/projects/files/prefix-cleanup' import * as manager from '@/lib/uploads/contexts/workspace/file-manager' +import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/file-secret-provenance' import { UPLOAD_URL_TTL_MS, writeLocalMultipartPart, @@ -49,6 +61,7 @@ import { import * as sessions from '@/lib/uploads/upload-session/service' import { simPageSourceEmbedBlock } from '@/lib/workspace-files/page-source-embed' +const redisKeys: string[] = [] const storageRoot = mkdtempSync(join(tmpdir(), 'sim-project-upload-')) setUploadDirServer(storageRoot) const fixtures: { @@ -629,6 +642,292 @@ describe('Project upload sessions with real leases, storage, and accounting', () ) }) +function workbench(f: Awaited>, bytes: Buffer) { + const machine = { providerId: 'daytona' as const, sandboxId: generateId() } + const sessionKey = generateId() + const namespace = createHash('sha256') + .update( + JSON.stringify([f.workspaceId, f.editorId, sessionKey, machine.providerId, machine.sandboxId]) + ) + .digest('hex') + const history = createHash('sha256') + .update(JSON.stringify([sessionKey, machine.providerId, machine.sandboxId])) + .digest('hex') + redisKeys.push( + `mothership:file-source:v1:${namespace}:${createHash('sha256').update(bytes).digest('hex')}`, + `mothership:workbench-provenance:v2:${history}` + ) + return { + machine, + files: createWorkbenchFileProvenance({ + workspaceId: f.workspaceId, + userId: f.editorId, + sessionKey, + }), + } +} + +async function privateUploadTransport( + f: Awaited>, + uploadProvenance: () => WorkspaceFileSecretProvenance, + signal?: AbortSignal +) { + const context = { + signal, + userId: f.editorId, + workspaceId: f.workspaceId, + toolCallId: generateId(), + copilotToolExecution: true, + copilotResourceAdmission: createCopilotResourceAdmission({ + userId: f.editorId, + invocation: { kind: 'workspace', workspaceId: f.workspaceId }, + }), + } + const endpoint = 'http://localhost:3000' + return createProjectFileUploadTransport({ + endpoint, + projectId: f.projectId, + context: { + ...context, + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry([], { + userId: f.editorId, + workspaceId: f.workspaceId, + }), + }, + fallback: createProjectFileCliTransport(endpoint, context, { projectId: f.projectId }), + uploadProvenance, + }) +} + +async function createPrivateUpload(transport: typeof fetch, projectId: string, bytes: Buffer) { + const response = await transport( + `http://localhost:3000/api/v2/projects/${projectId}/files/uploads`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: 'private.bin', + contentType: 'application/octet-stream', + size: bytes.length, + }), + } + ) + expect(response.status).toBe(201) + const created = v2CreateProjectFileUploadContract.response.schema.parse( + await response.json() + ).data + return { created, session: await sessions.verifyUploadSessionToken(created.uploadToken) } +} + +function completePrivateUpload( + transport: typeof fetch, + projectId: string, + session: sessions.UploadSessionRecord +) { + return transport( + `http://localhost:3000/api/v2/projects/${projectId}/files/uploads/${session.id}/complete`, + { + method: 'POST', + headers: { 'upload-token': session.uploadToken }, + } + ) +} + +describe.skipIf(!redisUrl)('private Project upload transport', () => { + check( + 'private Project upload seals actual streamed source evidence and retains original provenance attribution', + async () => { + const f = await fixture() + const bytes = Buffer.from([0, 255, 17, 6, 12]) + const evidence = { + status: 'exact' as const, + entries: [ + { + encryptedValue: 'upload-ciphertext-fixture', + sourceUserId: f.ownerId, + sourceWorkspaceId: f.workspaceId, + }, + ], + } + const { machine, files } = workbench(f, bytes) + const transport = await privateUploadTransport(f, files.uploadProvenance) + const downloaded = stream(bytes) + files.trackDownload(downloaded, evidence) + await new Response(files.observeDownload(machine, downloaded)).arrayBuffer() + const { session } = await createPrivateUpload(transport, f.projectId, bytes) + expect(session.metadata.projectFileSecretProvenance).toMatchObject({ + projectId: f.projectId, + pending: true, + }) + await writeLocalPutObject({ + uploadId: session.id, + key: session.finalKey, + body: files.observeUpload(machine, stream(bytes)), + expectedSize: bytes.length, + contentType: session.contentType, + metadata: sessions.uploadSessionObjectMetadata(session), + }) + const response = await completePrivateUpload(transport, f.projectId, session) + expect(response.status).toBe(200) + const completed = v2CompleteProjectFileUploadContract.response.schema.parse( + await response.json() + ).data + if (!completed.file) throw new Error('Private upload did not produce a file') + const [file] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.id, completed.file.id)) + const [provenance] = await db + .select() + .from(workspaceFileSecretProvenance) + .where(eq(workspaceFileSecretProvenance.fileId, file.id)) + expect(file).toMatchObject({ + projectId: f.projectId, + userId: f.editorId, + workspaceId: null, + }) + expect(await readFile(join(storageRoot, file.key))).toEqual(bytes) + expect(provenance).toMatchObject(evidence) + expect(await usage(f.organizationId)).toBe(bytes.length) + } + ) + + check( + 'private Project upload refuses incomplete evidence and never upgrades unknown or unrecorded streams to clean', + async () => { + const f = await fixture() + const bytes = Buffer.from('stream source') + for (const status of ['missing', 'partial', 'unknown', 'unrecorded'] as const) { + const { machine, files } = workbench(f, bytes) + const transport = await privateUploadTransport(f, files.uploadProvenance) + const { session } = await createPrivateUpload(transport, f.projectId, bytes) + if (status === 'missing' || status === 'partial') { + const partial = + status === 'partial' + ? files + .observeUpload( + machine, + new ReadableStream({ + start(controller) { + controller.enqueue(bytes) + }, + }) + ) + .getReader() + : undefined + if (partial) await partial.read() + await put(session, bytes) + const response = await completePrivateUpload(transport, f.projectId, session) + expect(response.status).toBe(503) + if (partial) await partial.cancel() + const current = await sessions.verifyUploadSessionToken(session.uploadToken) + expect(current.completedFileId).toBeNull() + expect(current.metadata.projectFileSecretProvenance).toMatchObject({ pending: true }) + } else { + const downloaded = stream(bytes) + files.trackDownload(downloaded, { status }) + await new Response(files.observeDownload(machine, downloaded)).arrayBuffer() + await writeLocalPutObject({ + uploadId: session.id, + key: session.finalKey, + body: files.observeUpload(machine, stream(bytes)), + expectedSize: bytes.length, + contentType: session.contentType, + metadata: sessions.uploadSessionObjectMetadata(session), + }) + const response = await completePrivateUpload(transport, f.projectId, session) + expect(response.status).toBe(200) + const current = await sessions.verifyUploadSessionToken(session.uploadToken) + if (!current.completedFileId) throw new Error('Classified upload did not produce a file') + const [provenance] = await db + .select() + .from(workspaceFileSecretProvenance) + .where(eq(workspaceFileSecretProvenance.fileId, current.completedFileId)) + expect(provenance).toMatchObject({ status, entries: [] }) + } + } + expect(await usage(f.organizationId)).toBe(bytes.length * 2) + } + ) + + check( + 'private Project upload binds control receipts to the invocation and current owner access', + async () => { + const f = await fixture() + const bytes = Buffer.from('private upload') + const evidence = () => ({ status: 'unknown' as const }) + const transport = await privateUploadTransport(f, evidence) + const { session } = await createPrivateUpload(transport, f.projectId, bytes) + await put(session, bytes) + const foreignTransport = await privateUploadTransport(f, evidence) + expect((await completePrivateUpload(foreignTransport, f.projectId, session)).status).toBe(403) + expect((await completePrivateUpload(transport, generateId(), session)).status).toBe(400) + expect( + ( + await transport(`http://localhost:3000/api/v2/projects/${f.projectId}/files/uploads`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: 'forged.bin', + contentType: 'application/octet-stream', + size: 1, + secretProvenance: { status: 'exact', entries: [] }, + }), + }) + ).status + ).toBe(400) + await db.delete(permissions).where(eq(permissions.userId, f.editorId)) + expect((await completePrivateUpload(transport, f.projectId, session)).status).toBe(403) + expect( + await db.select().from(workspaceFiles).where(eq(workspaceFiles.projectId, f.projectId)) + ).toEqual([]) + expect(await usage(f.organizationId)).toBe(0) + const current = await sessions.verifyUploadSessionToken(session.uploadToken) + expect(current.completedFileId).toBeNull() + expect(current.metadata.projectFileSecretProvenance).toMatchObject({ pending: true }) + } + ) + + for (const signalOwner of ['request', 'invocation'] as const) { + check( + `private Project upload never registers a file when the ${signalOwner} is cancelled while its provenance resolves`, + async () => { + const f = await fixture() + const bytes = Buffer.from('cancelled upload') + const controller = new AbortController() + /** Resolves cleanly after the abort, so only a recheck before completion can stop it. */ + const cancelling = () => { + controller.abort(new Error('cancelled during upload provenance')) + return { status: 'exact' as const, entries: [] } + } + const transport = await privateUploadTransport( + f, + cancelling, + signalOwner === 'invocation' ? controller.signal : undefined + ) + const { session } = await createPrivateUpload(transport, f.projectId, bytes) + await put(session, bytes) + await expect( + transport( + `http://localhost:3000/api/v2/projects/${f.projectId}/files/uploads/${session.id}/complete`, + { + method: 'POST', + headers: { 'upload-token': session.uploadToken }, + signal: signalOwner === 'request' ? controller.signal : undefined, + } + ) + ).rejects.toThrow('cancelled during upload provenance') + const current = await sessions.verifyUploadSessionToken(session.uploadToken) + expect(current.completedFileId).toBeNull() + expect( + await db.select().from(workspaceFiles).where(eq(workspaceFiles.projectId, f.projectId)) + ).toEqual([]) + expect(await usage(f.organizationId)).toBe(0) + } + ) + } +}) + afterAll(async () => { for (const f of fixtures) { await db.delete(uploadSession).where(inArray(uploadSession.userId, [f.ownerId, f.editorId])) @@ -642,6 +941,7 @@ afterAll(async () => { await db.delete(user).where(inArray(user.id, [f.ownerId, f.editorId])) } await rm(storageRoot, { recursive: true, force: true }) + if (redisKeys.length) await getRedisClient()?.del(...redisKeys) await closeRedisConnection() const report = process.env.PROJECT_UPLOAD_REPORT_PATH ?? 'test-results/project-upload.json' await mkdir(dirname(report), { recursive: true }) diff --git a/apps/sim/lib/projects/files/api/create-input.ts b/apps/sim/lib/projects/files/api/create-input.ts new file mode 100644 index 00000000000..05d5dac4275 --- /dev/null +++ b/apps/sim/lib/projects/files/api/create-input.ts @@ -0,0 +1,19 @@ +import type { ContractBody } from '@/lib/api/contracts/types' +import type { v2CreateProjectFileContract } from '@/lib/api/contracts/v2/project-files' +import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file' + +/** One mapping from the public create body to the use case, shared by the v2 route and the CLI. */ +export function toCreateProjectFileInput( + projectId: string, + body: ContractBody +) { + return { + projectId, + name: body.name, + contentType: body.contentType ?? getMimeTypeFromExtension(getFileExtension(body.name)), + content: body.content, + encoding: body.encoding, + folderPath: body.folderPath ?? '/', + exactName: true, + } +} diff --git a/apps/sim/lib/projects/files/api/index.ts b/apps/sim/lib/projects/files/api/index.ts index 0ab5b3517d8..d1e95468e91 100644 --- a/apps/sim/lib/projects/files/api/index.ts +++ b/apps/sim/lib/projects/files/api/index.ts @@ -1,5 +1,6 @@ export { toFileVersion as toProjectFileVersion } from '@/lib/workspace-files/api/version-presenters' export { presentProjectFileContent } from './content-presenter' +export { toCreateProjectFileInput } from './create-input' export { toV2ProjectFile } from './presenters' export { presentProjectFileUpload } from './upload-presenter' export { presentProjectFileVersionContent } from './version-presenters' diff --git a/apps/sim/lib/projects/files/application/artifacts.ts b/apps/sim/lib/projects/files/application/artifacts.ts index b356ebda15e..7be5134b2a8 100644 --- a/apps/sim/lib/projects/files/application/artifacts.ts +++ b/apps/sim/lib/projects/files/application/artifacts.ts @@ -4,6 +4,7 @@ import { workspaceFiles } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { and, asc, inArray, isNull } from 'drizzle-orm' +import type { AuthorizingUseCase } from '@/lib/core/application' import { requireAdmittedContent } from '@/lib/core/application/content-delivery' import { isDocSandboxEnabled } from '@/lib/core/config/env-flags' import { runWithOutboundOrganization } from '@/lib/core/network/context.server' @@ -41,7 +42,19 @@ import { import { fileDocumentInputIdentity } from '@/lib/uploads/documents/input-identity' import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative' import { getWorkspaceFileSize, MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { + MAX_TEXT_EXTRACTION_BYTES, + needsRenderedArtifact, + observedMediaType, +} from '@/lib/uploads/utils/file' import { reportWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' +import { + type ExtractedFileText, + extractFileTextFromBytes, + renderedTextTooLargeMessage, + textExtractionTooLargeError, + textExtractionUnsupportedError, +} from '@/lib/workspace-files/application/read-workspace-file-text' import { fileOwnerCondition } from '@/lib/workspace-files/ownership-query' import { isSimPageSource, @@ -61,6 +74,7 @@ import { type FileSearchArtifactCompletion, requeueFileSearchForRenderedArtifact, } from '@/lib/workspace-files/search/artifact-ready' +import { workspaceFileTextFormat } from '@/lib/workspace-files/text-format' const logger = createLogger('ProjectFileArtifacts') @@ -75,6 +89,10 @@ interface ArtifactInput extends ProjectFileTarget { maxBytes: number forModel?: boolean preview?: boolean + /** Bytes are only extracted as text, so an unextractable or oversized source is refused unread. */ + text?: boolean + /** The caller never reads `file.folderPath`, so the owner's folder tree is not loaded for it. */ + skipFolderPath?: boolean } interface SourceSnapshot { file: WorkspaceFileRow @@ -229,77 +247,97 @@ async function downloadDependencies( } } -const snapshotSource = defineAuthorizedProjectFileUseCase< - typeof projectFileOperations.readArtifact, - ArtifactInput, - SourceSnapshot ->({ - operation: projectFileOperations.readArtifact, - async execute({ - context, - tx, - }: { - context: ProjectFileAuthorizationContext - tx: DbTransaction - input: ArtifactInput - }) { - const file = context.file - if (!file) throw new OrchestrationError('not_found', 'File not found') - const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [file]) - return { - file, - owner: context.owner, - organizationId: context.organizationId, - secretProvenance: evidence.get(file.id), - } - }, - afterSuccess: ({ principal, result }) => - requireAdmittedContent(principal, { - kind: 'workspace-file', - id: result.file.id, - provenance: result.secretProvenance ?? { status: 'unknown' }, - view: 'opaque', - }), -}) +/** + * How a read's bytes leave: `opaque` bytes reach a caller as is, so no redaction can mask a secret + * in them; `complete` bytes are only ever turned into text, which the registry redacts. Every + * admission in one read uses the same view, so it is fixed per read rather than chosen per call. + */ +type ArtifactView = 'opaque' | 'complete' + +const defineSnapshotSource = (view: ArtifactView) => + defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.readArtifact, + ArtifactInput, + SourceSnapshot + >({ + operation: projectFileOperations.readArtifact, + async execute({ + context, + tx, + }: { + context: ProjectFileAuthorizationContext + tx: DbTransaction + input: ArtifactInput + }) { + const file = context.file + if (!file) throw new OrchestrationError('not_found', 'File not found') + const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [file]) + return { + file, + owner: context.owner, + organizationId: context.organizationId, + secretProvenance: evidence.get(file.id), + } + }, + afterSuccess: ({ principal, result }) => + requireAdmittedContent(principal, { + kind: 'workspace-file', + id: result.file.id, + provenance: result.secretProvenance ?? { status: 'unknown' }, + view, + }), + }) /** Locks the named inputs against the unchanged source and admits their merged provenance before any input read. */ -const snapshotDependencies = defineAuthorizedProjectFileUseCase< - typeof projectFileOperations.readArtifact, - ArtifactInput & { source: WorkspaceFileRow; dependencyIds: string[] }, - { dependencies: WorkspaceFileRow[]; secretProvenance: WorkspaceFileSecretProvenance } ->({ - operation: projectFileOperations.readArtifact, - async execute({ input, context, tx }) { - requireSameRevision(context.file, input.source) - const dependencies = await loadDependencies(tx, context.owner, input.dependencyIds) - const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [ - input.source, - ...dependencies, - ]) - return { - dependencies, - secretProvenance: mergeWorkspaceFileSecretProvenance(...evidence.values()), - } - }, - afterSuccess: ({ principal, input, result }) => - requireAdmittedContent(principal, { - kind: 'workspace-file', - id: input.source.id, - provenance: result.secretProvenance, - view: 'opaque', - }), -}) +const defineSnapshotDependencies = (view: ArtifactView) => + defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.readArtifact, + ArtifactInput & { source: WorkspaceFileRow; dependencyIds: string[] }, + { dependencies: WorkspaceFileRow[]; secretProvenance: WorkspaceFileSecretProvenance } + >({ + operation: projectFileOperations.readArtifact, + async execute({ input, context, tx }) { + requireSameRevision(context.file, input.source) + const dependencies = await loadDependencies(tx, context.owner, input.dependencyIds) + const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [ + input.source, + ...dependencies, + ]) + return { + dependencies, + secretProvenance: mergeWorkspaceFileSecretProvenance(...evidence.values()), + } + }, + afterSuccess: ({ principal, input, result }) => + requireAdmittedContent(principal, { + kind: 'workspace-file', + id: input.source.id, + provenance: result.secretProvenance, + view, + }), + }) +type SnapshotDependencies = ReturnType async function renderArtifact( principal: Principal, input: ArtifactInput, snapshot: SourceSnapshot, + snapshotDependencies: SnapshotDependencies, signal?: AbortSignal ): Promise { const { file, owner } = snapshot const artifact = await resolveDocumentRender( file.originalName, - { maxBytes: input.maxBytes, signal }, + { + maxBytes: input.maxBytes, + signal, + ...(input.text + ? { + tooLargeMessage: (limit: string) => + renderedTextTooLargeMessage(file.originalName, limit), + } + : {}), + }, async (): Promise => { const content = await downloadFile({ key: file.key, @@ -375,6 +413,19 @@ async function renderArtifact( return { ...artifact, ...derivative } } +/** + * Refuses a text read before any byte is fetched, as the workspace text path does. The stored size + * is authoritative only for an ordinary upload; a generation source is bounded by the render. + */ +function assertTextExtractable(file: WorkspaceFileRow, maxBytes: number) { + const name = file.originalName + if (!workspaceFileTextFormat({ name, type: observedMediaType(file.contentType, name) })) + throw textExtractionUnsupportedError(name) + const size = getWorkspaceFileSize(file) + if (!needsRenderedArtifact(file.contentType, name) && size > maxBytes) + throw textExtractionTooLargeError(name, size, maxBytes) +} + /** Indexing is best effort: a contended or failed requeue never fails a delivered read. */ async function requeueRenderedSearchArtifact(completion: FileSearchArtifactCompletion) { try { @@ -388,84 +439,173 @@ async function requeueRenderedSearchArtifact(completion: FileSearchArtifactCompl } /** Renders only source-owned dependencies, then rechecks the source, every input, and current authority. */ -export const readProjectFileArtifact = defineAuthorizedProjectFileUseCase< +function defineArtifactRead(view: ArtifactView) { + const snapshotSource = defineSnapshotSource(view) + const snapshotDependencies = defineSnapshotDependencies(view) + return defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.readArtifact, + ArtifactInput, + ArtifactResult, + PreparedArtifact + >({ + operation: projectFileOperations.readArtifact, + async prepareFromSnapshot({ principal, input, request }) { + if ( + !Number.isSafeInteger(input.maxBytes) || + input.maxBytes <= 0 || + input.maxBytes > MAX_BUFFERED_TRANSFER_BYTES + ) { + throw new OrchestrationError('validation', 'Invalid document byte limit') + } + const snapshot = await snapshotSource.execute({ principal, input }) + if (input.text) assertTextExtractable(snapshot.file, input.maxBytes) + return runWithOutboundOrganization(snapshot.organizationId, () => + renderArtifact(principal, input, snapshot, snapshotDependencies, request?.signal) + ) + }, + async execute({ input, context, tx, prepared }) { + if (!prepared) throw new Error('Prepared document is unavailable') + requireSameRevision(context.file, prepared.source) + const dependencies = await loadDependencies( + tx, + context.owner, + prepared.dependencies.map((file) => file.id) + ) + const currentById = new Map(dependencies.map((file) => [file.id, file])) + for (const expected of prepared.dependencies) + requireSameRevision(currentById.get(expected.id), expected) + const source = context.file + if (!source) throw new OrchestrationError('not_found', 'File not found') + const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [ + source, + ...dependencies, + ]) + const secretProvenance = mergeWorkspaceFileSecretProvenance(...evidence.values()) + if ( + input.forModel && + (secretProvenance.status !== 'exact' || secretProvenance.entries.length !== 0) + ) { + throw new OrchestrationError('forbidden', MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE) + } + const folders = + source.folderId && !input.skipFolderPath + ? await listFileFolders(context.owner, { scope: 'all' }, tx) + : [] + const result = { + file: mapFileRecord(source, context.owner, buildWorkspaceFileFolderPathMap(folders)), + buffer: prepared.buffer, + contentType: prepared.contentType, + dependsOnReferencedFiles: prepared.dependsOnReferencedFiles, + secretProvenance, + receipt: createFileReadReceipt(context.owner, [source, ...dependencies]), + } + if (prepared.artifactKey) + renderedSearchArtifacts.set(result, { + owner: context.owner, + file: { fileId: source.id, key: source.key, contentUpdatedAt: source.contentUpdatedAt }, + dependencies: dependencies.map((file) => ({ + fileId: file.id, + key: file.key, + sourceContentUpdatedAt: file.contentUpdatedAt, + })), + artifactKey: prepared.artifactKey, + }) + return result + }, + async afterSuccess({ principal, result }) { + await requireAdmittedContent(principal, { + kind: 'workspace-file', + id: result.file.id, + provenance: result.secretProvenance, + view, + }) + await reportWorkspaceFileDelivery(result.secretProvenance, result.file.id) + const completion = renderedSearchArtifacts.get(result) + renderedSearchArtifacts.delete(result) + if (completion) await requeueRenderedSearchArtifact(completion) + }, + }) +} + +/** A Project file's stored or rendered bytes. No redaction reaches them, so a secret refuses the read. */ +export const readProjectFileArtifact = defineArtifactRead('opaque') + +/** Admitted as text, so these bytes never leave except as extracted text. */ +const readProjectFileArtifactAsText = defineArtifactRead('complete') + +interface ProjectFileTextInput extends ProjectFileTarget { + fileId: string + maxBytes?: number + offset?: number + limit?: number + skipFolderPath?: boolean +} + +interface ProjectFileTextResult extends ExtractedFileText { + file: OwnedFileRecord + contentType: string + secretProvenance: WorkspaceFileSecretProvenance +} + +/** + * Reads a Project file's text the way a workspace file's text is read: the source and every input + * a render embeds are admitted as `complete`, the registry masks their secrets, and only text + * leaves. A generated document is read from its rendered artifact, like the workspace path. + * + * Only a ranged read is redacted here, before the window is cut. Whole-file text comes back raw: + * the caller must import `secretProvenance` into its egress registry before the text reaches an + * agent, as `importProjectFileProvenance` does. + */ +export const readProjectFileText: AuthorizingUseCase< typeof projectFileOperations.readArtifact, - ArtifactInput, - ArtifactResult, - PreparedArtifact ->({ - operation: projectFileOperations.readArtifact, - async prepareFromSnapshot({ principal, input, request }) { - if ( - !Number.isSafeInteger(input.maxBytes) || - input.maxBytes <= 0 || - input.maxBytes > MAX_BUFFERED_TRANSFER_BYTES - ) { - throw new OrchestrationError('validation', 'Invalid document byte limit') - } - const snapshot = await snapshotSource.execute({ principal, input }) - return runWithOutboundOrganization(snapshot.organizationId, () => - renderArtifact(principal, input, snapshot, request?.signal) + ProjectFileTextInput, + ProjectFileTextResult +> = { + operation: readProjectFileArtifactAsText.operation, + delegationAudience: readProjectFileArtifactAsText.delegationAudience, + authorize: (args) => + readProjectFileArtifactAsText.authorize({ + ...args, + input: { + projectId: args.input.projectId, + fileId: args.input.fileId, + maxBytes: MAX_TEXT_EXTRACTION_BYTES, + }, + }), + async execute(args) { + const { input, principal } = args + const maxBytes = Math.min( + input.maxBytes ?? MAX_TEXT_EXTRACTION_BYTES, + MAX_TEXT_EXTRACTION_BYTES ) - }, - async execute({ input, context, tx, prepared }) { - if (!prepared) throw new Error('Prepared document is unavailable') - requireSameRevision(context.file, prepared.source) - const dependencies = await loadDependencies( - tx, - context.owner, - prepared.dependencies.map((file) => file.id) + const artifact = await readProjectFileArtifactAsText.execute({ + ...args, + input: { + projectId: input.projectId, + fileId: input.fileId, + maxBytes, + text: true, + skipFolderPath: input.skipFolderPath, + }, + }) + const format = workspaceFileTextFormat({ name: artifact.file.name, type: artifact.contentType }) + if (!format) throw textExtractionUnsupportedError(artifact.file.name) + const extracted = await extractFileTextFromBytes( + artifact.buffer, + format, + artifact.file.name, + input, + { + principal, + kind: 'workspace-file', + signal: args.request?.signal, + } ) - const currentById = new Map(dependencies.map((file) => [file.id, file])) - for (const expected of prepared.dependencies) - requireSameRevision(currentById.get(expected.id), expected) - const source = context.file - if (!source) throw new OrchestrationError('not_found', 'File not found') - const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [ - source, - ...dependencies, - ]) - const secretProvenance = mergeWorkspaceFileSecretProvenance(...evidence.values()) - if ( - input.forModel && - (secretProvenance.status !== 'exact' || secretProvenance.entries.length !== 0) - ) { - throw new OrchestrationError('forbidden', MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE) - } - const folders = source.folderId - ? await listFileFolders(context.owner, { scope: 'all' }, tx) - : [] - const result = { - file: mapFileRecord(source, context.owner, buildWorkspaceFileFolderPathMap(folders)), - buffer: prepared.buffer, - contentType: prepared.contentType, - dependsOnReferencedFiles: prepared.dependsOnReferencedFiles, - secretProvenance, - receipt: createFileReadReceipt(context.owner, [source, ...dependencies]), + return { + ...extracted, + file: artifact.file, + contentType: artifact.contentType, + secretProvenance: artifact.secretProvenance, } - if (prepared.artifactKey) - renderedSearchArtifacts.set(result, { - owner: context.owner, - file: { fileId: source.id, key: source.key, contentUpdatedAt: source.contentUpdatedAt }, - dependencies: dependencies.map((file) => ({ - fileId: file.id, - key: file.key, - sourceContentUpdatedAt: file.contentUpdatedAt, - })), - artifactKey: prepared.artifactKey, - }) - return result }, - async afterSuccess({ principal, result }) { - await requireAdmittedContent(principal, { - kind: 'workspace-file', - id: result.file.id, - provenance: result.secretProvenance, - view: 'opaque', - }) - await reportWorkspaceFileDelivery(result.secretProvenance, result.file.id) - const completion = renderedSearchArtifacts.get(result) - renderedSearchArtifacts.delete(result) - if (completion) await requeueRenderedSearchArtifact(completion) - }, -}) +} diff --git a/apps/sim/lib/projects/files/application/authorization.ts b/apps/sim/lib/projects/files/application/authorization.ts index 3c2986c6554..fb6c8a7c6b6 100644 --- a/apps/sim/lib/projects/files/application/authorization.ts +++ b/apps/sim/lib/projects/files/application/authorization.ts @@ -32,6 +32,7 @@ import { PROJECT_FILE_DELEGATION_TTL_MS, type ProjectFileOperation, type ProjectFilePrincipal, + projectFileDelegationScope, projectFileOperations, } from '@/lib/projects/files/application/operations' import { requireProjectApiEnabled } from '@/lib/projects/rollout.server' @@ -87,12 +88,7 @@ function requirePrincipal( entityType: 'project', entityId: input.projectId, } - : { - kind: 'entity', - entityType: 'project', - entityId: input.projectId, - ...(input.fileId ? { fileId: input.fileId } : {}), - }, + : projectFileDelegationScope(input), maxTtlMs: PROJECT_FILE_DELEGATION_TTL_MS, }) } diff --git a/apps/sim/lib/projects/files/application/downloads.ts b/apps/sim/lib/projects/files/application/downloads.ts index 2cdeee6a553..9c9042e5085 100644 --- a/apps/sim/lib/projects/files/application/downloads.ts +++ b/apps/sim/lib/projects/files/application/downloads.ts @@ -47,6 +47,7 @@ import { MAX_RENDERED_DOCUMENT_BYTES, needsRenderedArtifact, } from '@/lib/uploads/utils/file' +import { normalizeMimeType } from '@/lib/uploads/utils/mime' import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' import { observeWorkspaceFileDelivery, @@ -318,7 +319,7 @@ const snapshotMarkdownAssets = defineAuthorizedProjectFileUseCase< if (!source) throw new OrchestrationError('not_found', 'File not found') if ( !isMarkdownFile({ name: source.originalName, type: source.contentType }) && - source.contentType !== 'text/x-markdown' + normalizeMimeType(source.contentType) !== 'text/x-markdown' ) throw new OrchestrationError('validation', 'Only Markdown files support snapshot export') if ( diff --git a/apps/sim/lib/projects/files/application/index.ts b/apps/sim/lib/projects/files/application/index.ts index 4ea278da821..cdbf5cbead3 100644 --- a/apps/sim/lib/projects/files/application/index.ts +++ b/apps/sim/lib/projects/files/application/index.ts @@ -1,4 +1,4 @@ -export { readProjectFileArtifact } from './artifacts' +export { readProjectFileArtifact, readProjectFileText } from './artifacts' export { getProjectFileListAccess } from './collection-access' export { createProjectFile, @@ -18,6 +18,7 @@ export { export { projectFileOperations } from './operations' export { readProjectFileCsvPreview, readProjectInlineFile } from './previews' export { + getProjectFileCapabilities, getProjectFileMetadata, listProjectFileItems, listProjectFiles, diff --git a/apps/sim/lib/projects/files/application/operations.ts b/apps/sim/lib/projects/files/application/operations.ts index fa3e6032f21..c9497e8aed4 100644 --- a/apps/sim/lib/projects/files/application/operations.ts +++ b/apps/sim/lib/projects/files/application/operations.ts @@ -1,4 +1,8 @@ -import type { Principal, ResourceDelegatedPrincipal } from '@sim/auth/principal' +import type { + Principal, + ResourceDelegatedPrincipal, + ResourceDelegationScope, +} from '@sim/auth/principal' import { type ApplicationOperation, assertOperationCapability, @@ -27,6 +31,19 @@ interface ProjectFilePolicy extends ApplicationOperation { export const PROJECT_FILE_DELEGATION_TTL_MS = 60_000 +/** The grant a delegated Project file call carries: the Project, narrowed to one file when targeted. */ +export function projectFileDelegationScope(target: { + projectId: string + fileId?: string +}): ResourceDelegationScope { + return { + kind: 'entity', + entityType: 'project', + entityId: target.projectId, + ...(target.fileId ? { fileId: target.fileId } : {}), + } +} + function defineProjectFileOperation(operation: O): O { assertOperationCapability(operation) assertOperationOAuthPolicy(operation) diff --git a/apps/sim/lib/projects/files/application/read.ts b/apps/sim/lib/projects/files/application/read.ts index 7702d991975..ab14ffd5ca4 100644 --- a/apps/sim/lib/projects/files/application/read.ts +++ b/apps/sim/lib/projects/files/application/read.ts @@ -38,6 +38,14 @@ interface ListProjectFilesResult { capabilities: { canRead: true; canWrite: boolean } } +/** Capability hints describe this request's authority; later operations always reauthorize. */ +export const getProjectFileCapabilities = defineAuthorizedProjectFileUseCase({ + operation: projectFileOperations.list, + async execute({ context }) { + return { owner: context.owner, canRead: true as const, canWrite: context.canWrite } + }, +}) + export const listProjectFiles = defineAuthorizedProjectFileUseCase< typeof projectFileOperations.list, ListProjectFilesInput, diff --git a/apps/sim/lib/uploads/documents/compile.test.ts b/apps/sim/lib/uploads/documents/compile.test.ts index 698ea4479b1..08bf263e77a 100644 --- a/apps/sim/lib/uploads/documents/compile.test.ts +++ b/apps/sim/lib/uploads/documents/compile.test.ts @@ -1,5 +1,9 @@ import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { + readWorkspaceFileMetadataMock, + readWorkspaceFileMetadataMockFns, +} from '@sim/testing/mocks/read-workspace-file-metadata.mock' import { remoteSandboxMock, remoteSandboxMockFns } from '@sim/testing/mocks/remote-sandbox.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -7,7 +11,6 @@ const { loadCompiledDocMock, publishCompiledDocArtifactMock, readWorkspaceFileContentMock, - readWorkspaceFileMetadataMock, storeCompiledDocMock, runSandboxTaskMock, } = vi.hoisted(() => ({ @@ -15,7 +18,6 @@ const { loadCompiledDocMock: vi.fn(), publishCompiledDocArtifactMock: vi.fn(), readWorkspaceFileContentMock: vi.fn(), - readWorkspaceFileMetadataMock: vi.fn(), storeCompiledDocMock: vi.fn(), })) @@ -26,9 +28,10 @@ vi.mock('@/lib/execution/languages', () => ({ vi.mock('@/lib/workspace-files/application/read-workspace-file-content', () => ({ readWorkspaceFileContent: { execute: readWorkspaceFileContentMock }, })) -vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', () => ({ - readWorkspaceFileMetadata: { execute: readWorkspaceFileMetadataMock }, -})) +vi.mock( + '@/lib/workspace-files/application/read-workspace-file-metadata', + () => readWorkspaceFileMetadataMock +) vi.mock('@/lib/execution/sandbox/run-task', () => ({ runSandboxTask: runSandboxTaskMock })) vi.mock('@/lib/uploads/documents/compiled-store', async (importOriginal) => ({ ...(await importOriginal()), @@ -49,6 +52,7 @@ import { import { DocCompileUserError } from '@/lib/uploads/documents/compile-error' const executeInSandboxMock = remoteSandboxMockFns.mockExecuteInSandbox +const readMetadataMock = readWorkspaceFileMetadataMockFns.mockReadWorkspaceFileMetadata const ID = '550e8400-e29b-41d4-a716-446655440000' const FILE_PRINCIPAL = createSessionPrincipal() @@ -194,7 +198,7 @@ describe('collectReferencedFileIds', () => { filePrincipal: FILE_PRINCIPAL, }) ).rejects.toThrow('More than 500 referenced input files; maximum is 500') - expect(readWorkspaceFileMetadataMock).not.toHaveBeenCalled() + expect(readMetadataMock).not.toHaveBeenCalled() expect(executeInSandboxMock).not.toHaveBeenCalled() }) @@ -214,7 +218,7 @@ describe('collectReferencedFileIds', () => { contentUpdatedAt, storageContext: 'workspace' as const, } - readWorkspaceFileMetadataMock.mockResolvedValue({ file: record, share: null }) + readMetadataMock.mockResolvedValue({ file: record, share: null }) loadCompiledDocMock.mockResolvedValue(null) readWorkspaceFileContentMock.mockResolvedValue({ file: record, content: Buffer.from('image') }) executeInSandboxMock.mockResolvedValue({ @@ -270,7 +274,7 @@ describe('collectReferencedFileIds', () => { contentUpdatedAt, storageContext: 'workspace', } - readWorkspaceFileMetadataMock.mockResolvedValue({ file: record, share: null }) + readMetadataMock.mockResolvedValue({ file: record, share: null }) loadCompiledDocMock.mockResolvedValue(Buffer.from('%PDF-cached')) await expect( @@ -348,7 +352,7 @@ function buildReadingReferencedFile() { _input: unknown, options: { onWorkspaceFileAccess?: (identity: unknown) => void } ) => { - const { file } = await readWorkspaceFileMetadataMock() + const { file } = await readMetadataMock() options.onWorkspaceFileAccess?.({ fileId: ID, key: file.key, @@ -386,7 +390,7 @@ describe('isolated-VM compile for Chat', () => { filePrincipal: chat, }) ) - readWorkspaceFileMetadataMock.mockResolvedValue({ + readMetadataMock.mockResolvedValue({ file: referencedFileVersion('2026-08-05T00:00:00.000Z'), share: null, }) @@ -394,7 +398,7 @@ describe('isolated-VM compile for Chat', () => { await compile() expect(runSandboxTaskMock).toHaveBeenCalledOnce() - readWorkspaceFileMetadataMock.mockResolvedValue({ + readMetadataMock.mockResolvedValue({ file: referencedFileVersion('2026-08-06T00:00:00.000Z'), share: null, }) @@ -418,7 +422,7 @@ describe('isolated-VM compile for a session', () => { it('rebuilds a cached build whose referenced file was edited, reporting the current version', async () => { buildReadingReferencedFile() - readWorkspaceFileMetadataMock.mockResolvedValue({ + readMetadataMock.mockResolvedValue({ file: referencedFileVersion('2026-08-05T00:00:00.000Z'), share: null, }) @@ -427,7 +431,7 @@ describe('isolated-VM compile for a session', () => { expect(runSandboxTaskMock).toHaveBeenCalledOnce() const edited = referencedFileVersion('2026-08-06T00:00:00.000Z') - readWorkspaceFileMetadataMock.mockResolvedValue({ file: edited, share: null }) + readMetadataMock.mockResolvedValue({ file: edited, share: null }) const rebuilt = await compile('session-edited') expect(runSandboxTaskMock).toHaveBeenCalledTimes(2) expect(rebuilt.contributingFiles).toEqual([ @@ -437,14 +441,12 @@ describe('isolated-VM compile for a session', () => { it('rebuilds a cached build whose referenced file was deleted', async () => { buildReadingReferencedFile() - readWorkspaceFileMetadataMock.mockResolvedValue({ + readMetadataMock.mockResolvedValue({ file: referencedFileVersion('2026-08-05T00:00:00.000Z'), share: null, }) await compile('session-deleted') - readWorkspaceFileMetadataMock.mockRejectedValueOnce( - new OrchestrationError('not_found', 'File not found') - ) + readMetadataMock.mockRejectedValueOnce(new OrchestrationError('not_found', 'File not found')) await compile('session-deleted') expect(runSandboxTaskMock).toHaveBeenCalledTimes(2) }) diff --git a/apps/sim/lib/uploads/documents/servable.test.ts b/apps/sim/lib/uploads/documents/servable.test.ts index d33cd737616..e85f6aecdfc 100644 --- a/apps/sim/lib/uploads/documents/servable.test.ts +++ b/apps/sim/lib/uploads/documents/servable.test.ts @@ -1,5 +1,9 @@ import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { + readWorkspaceFileMetadataMock, + readWorkspaceFileMetadataMockFns, +} from '@sim/testing/mocks/read-workspace-file-metadata.mock' import { remoteSandboxMock, remoteSandboxMockFns } from '@sim/testing/mocks/remote-sandbox.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -8,7 +12,6 @@ const { mockLoadPublishedCompiledDoc, mockPublishCompiledDocArtifact, mockReadWorkspaceFileContent, - mockReadWorkspaceFileMetadata, mockRunSandboxTask, mockStoreCompiledDoc, } = vi.hoisted(() => ({ @@ -16,7 +19,6 @@ const { mockLoadPublishedCompiledDoc: vi.fn(), mockPublishCompiledDocArtifact: vi.fn(), mockReadWorkspaceFileContent: vi.fn(), - mockReadWorkspaceFileMetadata: vi.fn(), mockRunSandboxTask: vi.fn(), mockStoreCompiledDoc: vi.fn(), })) @@ -31,9 +33,10 @@ vi.mock('@/lib/execution/sandbox/run-task', () => ({ vi.mock('@/lib/workspace-files/application/read-workspace-file-content', () => ({ readWorkspaceFileContent: { execute: mockReadWorkspaceFileContent }, })) -vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', () => ({ - readWorkspaceFileMetadata: { execute: mockReadWorkspaceFileMetadata }, -})) +vi.mock( + '@/lib/workspace-files/application/read-workspace-file-metadata', + () => readWorkspaceFileMetadataMock +) vi.mock('@/lib/uploads/documents/compiled-store', async (importOriginal) => ({ ...(await importOriginal()), loadCompiledDoc: mockLoadCompiledDoc, @@ -50,6 +53,7 @@ import { import { DocCompileUserError } from '@/lib/uploads/documents/compile-error' const { mockExecuteInSandbox } = remoteSandboxMockFns +const { mockReadWorkspaceFileMetadata } = readWorkspaceFileMetadataMockFns const WORKSPACE_ID = '550e8400-e29b-41d4-a716-446655440000' const FILE_PRINCIPAL = createSessionPrincipal() diff --git a/apps/sim/lib/uploads/utils/file.test.ts b/apps/sim/lib/uploads/utils/file.test.ts index aca271c52cf..286d067b7f2 100644 --- a/apps/sim/lib/uploads/utils/file.test.ts +++ b/apps/sim/lib/uploads/utils/file.test.ts @@ -5,6 +5,7 @@ import { extractWorkspaceIdFromStorageKey, inferContextFromKey, isInternalFileUrl, + observedMediaType, processSingleFileToUserFile, resolveEffectiveMimeType, resolveFileType, @@ -169,3 +170,17 @@ describe('resolveEffectiveMimeType', () => { expect(resolveFileType({ type: 'audio/webm', name: 'clip.webm' })).toBe('audio/webm') }) }) + +describe('observedMediaType', () => { + it('reads a generated document as the type it renders to, not its source type', () => { + expect(observedMediaType('text/x-python-pdf', 'report.pdf')).toBe('application/pdf') + expect(observedMediaType('text/x-docxjs', 'brief.docx')).toBe( + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ) + }) + + it('reads a stored file as its own base media type', () => { + expect(observedMediaType('text/plain; charset=utf-8', 'notes.txt')).toBe('text/plain') + expect(observedMediaType('IMAGE/PNG', 'photo.png')).toBe('image/png') + }) +}) diff --git a/apps/sim/lib/uploads/utils/file.ts b/apps/sim/lib/uploads/utils/file.ts index 43b15b67df9..50c977e451f 100644 --- a/apps/sim/lib/uploads/utils/file.ts +++ b/apps/sim/lib/uploads/utils/file.ts @@ -269,6 +269,21 @@ export function needsRenderedArtifact( : isRenderableDocumentName(fileName) } +/** + * The base media type a reader sees for a stored file: a generated document's rendered type, as + * its name promises, otherwise its own effective type. Readers choose a representation from this. + */ +export function observedMediaType( + contentType: string | null | undefined, + fileName: string +): string { + return normalizeMimeType( + needsRenderedArtifact(contentType, fileName) + ? getMimeTypeFromExtension(getFileExtension(fileName)) + : resolveEffectiveMimeType(contentType, fileName) + ) +} + const ARCHIVE_EXTENSIONS = new Set(SUPPORTED_ARCHIVE_EXTENSIONS) /** diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-text.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-text.ts index f5bdf03a108..d672dfe5a5b 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-text.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-text.ts @@ -1,6 +1,6 @@ import type { Principal } from '@sim/auth/principal' import { getErrorMessage } from '@sim/utils/errors' -import type { AuthorizedWorkspaceUseCaseContext } from '@/lib/core/application' +import type { AuthorizedWorkspaceUseCaseContext, ContentSourceKind } from '@/lib/core/application' import { redactBeforeCutting } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' @@ -48,8 +48,8 @@ export interface ReadWorkspaceFileTextInput { allowPlainText?: boolean } -export interface ReadWorkspaceFileTextResult { - file: WorkspaceFileRecord +/** Text extracted from one file's bytes, whichever owner stores them. */ +export interface ExtractedFileText { text: string /** True when a parser limit stopped extraction before the input was exhausted. */ truncated: boolean @@ -71,6 +71,39 @@ export interface ReadWorkspaceFileTextResult { } } +export interface ReadWorkspaceFileTextResult extends ExtractedFileText { + file: WorkspaceFileRecord +} + +/** Refuses a text read of a file whose format has no extractor. */ +export function textExtractionUnsupportedError(name: string): OrchestrationError { + return new OrchestrationError( + 'validation', + `Text extraction is not supported for "${name}"; download the raw bytes instead of extracting text` + ) +} + +/** + * Refuses a text read whose stored bytes exceed the limit. Sizes render with `includeBytes` + * because a caller-supplied `maxBytes` is routinely under 1 KB, and the default formatting collapses + * every sub-1 KB value to "0 Bytes" — naming neither the real size nor the limit to raise. + */ +export function textExtractionTooLargeError( + name: string, + size: number, + maxBytes: number +): OrchestrationError { + return new OrchestrationError( + 'payload_too_large', + `"${name}" is ${formatFileSize(size, { includeBytes: true })}, above the ${formatFileSize(maxBytes, { includeBytes: true })} text-extraction limit; download the raw bytes instead of extracting text` + ) +} + +/** The refusal for a generated document whose render exceeds the text-extraction limit. */ +export function renderedTextTooLargeMessage(name: string, limit: string): string { + return `"${name}" renders to more than ${limit}, above the text-extraction limit; download the raw bytes instead of extracting text` +} + /** * Reads an ordinary uploaded file's bytes. * @@ -83,17 +116,7 @@ async function readSourceBuffer( maxBytes: number, signal?: AbortSignal ): Promise { - if (file.size > maxBytes) { - /** - * Sizes render with `includeBytes` because a caller-supplied `maxBytes` is - * routinely under 1 KB, and the default formatting collapses every sub-1 KB - * value to "0 Bytes" — naming neither the real size nor the limit to raise. - */ - throw new OrchestrationError( - 'payload_too_large', - `"${file.name}" is ${formatFileSize(file.size, { includeBytes: true })}, above the ${formatFileSize(maxBytes, { includeBytes: true })} text-extraction limit; download the raw bytes instead of extracting text` - ) - } + if (file.size > maxBytes) throw textExtractionTooLargeError(file.name, file.size, maxBytes) return fetchWorkspaceFileBuffer(file, { maxBytes, signal }) } @@ -134,12 +157,7 @@ export async function extractWorkspaceFileRecordText( const extension = input.allowPlainText ? (workspaceFileTextFormat(file) ?? getFileExtension(file.name)) : getFileExtension(file.name) - if (!isSupportedFileType(extension)) { - throw new OrchestrationError( - 'validation', - `Text extraction is not supported for "${file.name}"; download the raw bytes instead of extracting text` - ) - } + if (!isSupportedFileType(extension)) throw textExtractionUnsupportedError(file.name) const maxBytes = Math.min(input.maxBytes ?? MAX_TEXT_EXTRACTION_BYTES, MAX_TEXT_EXTRACTION_BYTES) @@ -156,8 +174,7 @@ export async function extractWorkspaceFileRecordText( const rendered = await resolveRenderedWorkspaceArtifact(file, principal, { maxBytes, signal, - tooLargeMessage: (limit) => - `"${file.name}" renders to more than ${limit}, above the text-extraction limit; download the raw bytes instead of extracting text`, + tooLargeMessage: (limit) => renderedTextTooLargeMessage(file.name, limit), }) await requireAdmittedContributors( principal, @@ -169,22 +186,41 @@ export async function extractWorkspaceFileRecordText( } else { content = await readSourceBuffer(file, maxBytes, signal) } - const parsed = await parseFileText(content, extension, file.name, signal) - const metadata = parsed.metadata ?? {} + return { + file, + ...(await extractFileTextFromBytes(content, extension, file.name, input, { + principal, + kind: 'workspace-file', + signal, + })), + } +} +/** + * Turns admitted bytes into text for every file owner. The caller admits the content as `complete` + * first; a line window is cut only after redaction, so it can never split a secret the registry + * would otherwise mask whole. + */ +export async function extractFileTextFromBytes( + content: Buffer, + format: string, + fileName: string, + window: { offset?: number; limit?: number }, + options: { principal?: Principal; kind: ContentSourceKind; signal?: AbortSignal } +): Promise { + const parsed = await parseFileText(content, format, fileName, options.signal) + const metadata = parsed.metadata ?? {} const truncated = metadata.truncated === true - const ranged = input.offset !== undefined || input.limit !== undefined + const ranged = window.offset !== undefined || window.limit !== undefined const { text, lineRange } = sliceFileTextLines( ranged - ? await redactBeforeCutting(principal, parsed.content, 'workspace-file') + ? await redactBeforeCutting(options.principal, parsed.content, options.kind) : parsed.content, - input.offset, - input.limit, + window.offset, + window.limit, truncated ) - return { - file, text, truncated, degraded: metadata.degraded === true, @@ -209,7 +245,7 @@ export async function extractWorkspaceFileRecordText( * well formed, it is the stored bytes that cannot become the representation * being asked for, and the caller needs to know that retrying will not help. */ -export async function parseFileText( +async function parseFileText( content: Buffer, extension: string, fileName: string, diff --git a/apps/sim/lib/workspace-files/ownership.ts b/apps/sim/lib/workspace-files/ownership.ts index c1618254b3c..97aea752618 100644 --- a/apps/sim/lib/workspace-files/ownership.ts +++ b/apps/sim/lib/workspace-files/ownership.ts @@ -7,6 +7,12 @@ export interface FileOwner { entityId: string } +/** An asserted file target; application authorization verifies its persisted owner. */ +export interface OwnedFileTarget { + owner: FileOwner + fileId: string +} + /** Owners whose durable files support the shared Files editor and folder lifecycle. */ export interface EditableFileOwner { entityType: 'workspace' | 'project' diff --git a/apps/sim/stores/panel/types.ts b/apps/sim/stores/panel/types.ts index 483a705b530..2eb7b91cec9 100644 --- a/apps/sim/stores/panel/types.ts +++ b/apps/sim/stores/panel/types.ts @@ -1,5 +1,6 @@ import type { MothershipTableViewContext } from '@/lib/api/contracts/mothership-resources' import type { ManagedMcpConnectorId } from '@/lib/credential-groups/managed-mcp-connectors' +import type { FileOperationOwner } from '@/lib/mothership/generated/file-owner' /** * Available panel tabs @@ -80,10 +81,11 @@ export type ChatContext = */ columnIds?: string[] } & WorkspaceOwned) - | ({ kind: 'file'; fileId: string; label: string } & WorkspaceOwned) + | ({ kind: 'file'; fileId: string; label: string; owner?: FileOperationOwner } & WorkspaceOwned) | ({ kind: 'dashboard'; dashboardId: string; label: string } & WorkspaceOwned) | ({ kind: 'file_selection' + owner?: FileOperationOwner fileId: string label: string /** Name of the file the selection came from. See `tableName` above. */ @@ -106,6 +108,7 @@ export type ChatContext = | ({ kind: 'filefolder'; fileFolderId: string; label: string } & WorkspaceOwned) /** A whole workspace in an organization chat: "I'm working in this one". */ | { kind: 'workspace'; workspaceId: string; label: string } + | { kind: 'project'; projectId: string; label: string } | { kind: 'docs'; label: string } /** * A tab in the desktop browser or terminal panel, dragged into the input to diff --git a/packages/sim-cli/scripts/print-command-inventory.ts b/packages/sim-cli/scripts/print-command-inventory.ts index 37f562b115c..7fe511d2080 100644 --- a/packages/sim-cli/scripts/print-command-inventory.ts +++ b/packages/sim-cli/scripts/print-command-inventory.ts @@ -21,7 +21,7 @@ import { type ReferenceDocument, } from '#sim-cli/contract/reference' import { V2_OPERATIONS, type V2OperationName } from '#sim-cli/generated/v2-api' -import { buildProgram } from '#sim-cli/program' +import { buildProgram, UNPUBLISHED_GROUPS } from '#sim-cli/program' import { calledOperations, operationOf } from '#sim-cli/runtime/called-operations' import { camel } from '#sim-cli/runtime/derive' import { cursorSlot, flagNameFor } from '#sim-cli/runtime/request' @@ -58,28 +58,58 @@ interface InventoryCommand extends CommandReference { * Absent when every call is admitted or the command calls no known operation. */ mothershipUnavailable?: true + /** In a group hidden from `--help` until launch; the worker exposes it only when Sim enables it. */ + unpublished?: true } function isHiddenCommand(command: Command): boolean { return (command as Command & { _hidden?: boolean })._hidden === true } -function subcommands(command: Command): Command[] { +/** Hidden commands stay out, except a whole top-level group that is only unpublished until launch. */ +function subcommands(command: Command, atRoot: boolean): Command[] { return command.commands.filter( - (child) => child.name() !== HELP_COMMAND && !isHiddenCommand(child) + (child) => + child.name() !== HELP_COMMAND && + (!isHiddenCommand(child) || (atRoot && UNPUBLISHED_GROUPS.has(child.name()))) ) } function collectLeaves(command: Command, prefix: string[]): { path: string[]; command: Command }[] { - const children = subcommands(command) + const children = subcommands(command, prefix.length === 0) if (children.length === 0) return [{ path: prefix, command }] return children.flatMap((child) => collectLeaves(child, [...prefix, child.name()])) } -const OPENAPI_DOCS: ReferenceDocument[] = fs - .readdirSync(path.join(ROOT, 'apps/docs')) - .filter((name) => /^openapi-v2-.*\.json$/.test(name)) - .map((name) => JSON.parse(fs.readFileSync(path.join(ROOT, 'apps/docs', name), 'utf8'))) +/** + * Unpublished groups are documented in launch-ready definitions kept out of the public reference, + * so their response shapes come from those definitions, as the CLI's own `--help` text does. + */ +async function unpublishedOpenApiDocuments(): Promise { + const [ + { generateOpenApiDocument }, + { filesAuditOpenApiDocument }, + { unpublishedProjectFileDocumentation }, + ] = await Promise.all([ + import(path.join(ROOT, 'scripts/openapi/generator.ts')), + import(path.join(ROOT, 'apps/sim/lib/api/contracts/v2/openapi/files-audit.ts')), + import(path.join(ROOT, 'apps/sim/lib/api/contracts/v2/openapi/unpublished-project-files.ts')), + ]) + return [ + generateOpenApiDocument({ + ...filesAuditOpenApiDocument, + routes: unpublishedProjectFileDocumentation.routes, + }), + ] +} + +const OPENAPI_DOCS: ReferenceDocument[] = [ + ...fs + .readdirSync(path.join(ROOT, 'apps/docs')) + .filter((name) => /^openapi-v2-.*\.json$/.test(name)) + .map((name) => JSON.parse(fs.readFileSync(path.join(ROOT, 'apps/docs', name), 'utf8'))), + ...(await unpublishedOpenApiDocuments()), +] /** * `METHOD path` of every route Sim refuses a Mothership caller, as the route inventory @@ -150,6 +180,7 @@ const inventory: InventoryCommand[] = collectLeaves(program, []).map( options, ...reference, ...(mothershipRefuses(called) ? { mothershipUnavailable: true as const } : {}), + ...(UNPUBLISHED_GROUPS.has(cmdPath[0] ?? '') ? { unpublished: true as const } : {}), } } ) diff --git a/packages/sim-cli/src/contract/reference.test.ts b/packages/sim-cli/src/contract/reference.test.ts index 99d3bd3e68f..478d9e185f7 100644 --- a/packages/sim-cli/src/contract/reference.test.ts +++ b/packages/sim-cli/src/contract/reference.test.ts @@ -8,6 +8,7 @@ import { } from '#sim-cli/contract/reference' import { runEmbeddedCli } from '#sim-cli/embed' import { V2_OPERATIONS } from '#sim-cli/generated/v2-api' +import { buildProgram, UNPUBLISHED_GROUPS } from '#sim-cli/program' import { cursorSlot } from '#sim-cli/runtime/request' const ROOT = fileURLToPath(new URL('../../../../', import.meta.url)) @@ -17,6 +18,7 @@ interface InventoryEntry { shape?: string body?: string mothershipUnavailable?: true + unpublished?: true } let produced: InventoryEntry[] | undefined @@ -47,6 +49,19 @@ function document(schema: ReferenceSchema): ReferenceDocument { const operation = { method: 'POST', path: '/items/[id]' } describe('CLI reference producer', () => { + it('lists unpublished groups for the worker, marked, while --help keeps hiding them', () => { + const commands = inventory() + for (const group of UNPUBLISHED_GROUPS) { + const members = commands.filter((entry) => entry.path[0] === group) + expect(members.length).toBeGreaterThan(0) + expect(members.every((entry) => entry.unpublished === true)).toBe(true) + expect(buildProgram().helpInformation()).not.toMatch(new RegExp(`^\\s+${group}\\b`, 'm')) + } + expect( + commands.filter((entry) => entry.unpublished && !UNPUBLISHED_GROUPS.has(entry.path[0] ?? '')) + ).toEqual([]) + }) + it('publishes creation ids and preserves single/batch row choices from the real API', () => { const commands = inventory() const find = (path: string) => { diff --git a/packages/sim-cli/src/program.ts b/packages/sim-cli/src/program.ts index 92d2025ab50..fda98e468dc 100644 --- a/packages/sim-cli/src/program.ts +++ b/packages/sim-cli/src/program.ts @@ -62,8 +62,12 @@ best match rather than repeating similar searches, then run === END AGENT COMMAND DISCOVERY === ` -/** Executable but unpublished until launch, so kept out of `--help` and `sim cli` search. */ -const UNPUBLISHED_GROUPS: ReadonlySet = new Set(['projects']) +/** + * Executable but unpublished until launch, so kept out of `--help` and `sim cli` search. The + * Mothership command inventory still lists them, marked `unpublished`, because the worker routes + * to them only when Sim enables the feature for a turn. + */ +export const UNPUBLISHED_GROUPS: ReadonlySet = new Set(['projects']) /** The root's own value-taking options, which consume the token after them. */ const ROOT_VALUE_FLAGS: ReadonlySet = new Set([ diff --git a/packages/testing/src/mocks/index.ts b/packages/testing/src/mocks/index.ts index f32c784e589..ef66b75a5a0 100644 --- a/packages/testing/src/mocks/index.ts +++ b/packages/testing/src/mocks/index.ts @@ -691,6 +691,10 @@ export { reactQueryMock, reactQueryMockFns, } from './react-query.mock' +export { + readWorkspaceFileMetadataMock, + readWorkspaceFileMetadataMockFns, +} from './read-workspace-file-metadata.mock' export { realtimeNotifyMock, realtimeNotifyMockFns, diff --git a/packages/testing/src/mocks/read-workspace-file-metadata.mock.ts b/packages/testing/src/mocks/read-workspace-file-metadata.mock.ts new file mode 100644 index 00000000000..ba1dc3c1dbc --- /dev/null +++ b/packages/testing/src/mocks/read-workspace-file-metadata.mock.ts @@ -0,0 +1,69 @@ +import { vi } from 'vitest' + +/** A copy of `fileOperations.readMetadata` from `@/lib/workspace-files/application/operations`. */ +const FILES_READ_METADATA_OPERATION = { + id: 'files.read_metadata', + oauthScope: 'api:read', + minimumRole: 'read', + workspaceApiKey: 'allow', + capability: 'files.use', + contentDelivery: 'none', + principalKinds: [ + 'session', + 'personal_api_key', + 'oauth_access_token', + 'workspace_api_key', + 'delegated', + ], + delegatedServices: ['copilot', 'executor'], +} as const + +/** `WORKSPACE_FILES_DELEGATION_AUDIENCE`, exposed by every workspace-file use case. */ +const WORKSPACE_FILES_DELEGATION_AUDIENCE = 'sim:workspace-files' + +/** + * Controllable mock functions for `@/lib/workspace-files/application/read-workspace-file-metadata`. + * Every function is a bare `vi.fn()`. Each use case is exposed as its `execute` knob + * (`mock`) and its `authorize` knob (`mockAuthorize`). + * + * @example + * ```ts + * import { readWorkspaceFileMetadataMockFns } from '@sim/testing/mocks/read-workspace-file-metadata.mock' + * + * readWorkspaceFileMetadataMockFns.mockReadWorkspaceFileMetadata.mockResolvedValue({ file, share: null }) + * ``` + */ +export const readWorkspaceFileMetadataMockFns = { + mockReadWorkspaceFileMetadata: vi.fn(), + mockReadWorkspaceFileMetadataAuthorize: vi.fn(), + mockReadWorkspaceFileMetadataWithVersion: vi.fn(), + mockReadWorkspaceFileMetadataWithVersionAuthorize: vi.fn(), +} + +const fns = readWorkspaceFileMetadataMockFns + +/** + * Static mock module for `@/lib/workspace-files/application/read-workspace-file-metadata`. Each use + * case is `{ operation, delegationAudience, authorize, execute }` with a copy of the real + * `files.read_metadata` operation; a test whose code checks operation identity through a Copilot + * adapter spreads this mock and substitutes the real operation. + * + * @example + * ```ts + * vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', () => readWorkspaceFileMetadataMock) + * ``` + */ +export const readWorkspaceFileMetadataMock = { + readWorkspaceFileMetadata: { + operation: FILES_READ_METADATA_OPERATION, + delegationAudience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + authorize: fns.mockReadWorkspaceFileMetadataAuthorize, + execute: fns.mockReadWorkspaceFileMetadata, + }, + readWorkspaceFileMetadataWithVersion: { + operation: FILES_READ_METADATA_OPERATION, + delegationAudience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + authorize: fns.mockReadWorkspaceFileMetadataWithVersionAuthorize, + execute: fns.mockReadWorkspaceFileMetadataWithVersion, + }, +} diff --git a/scripts/check-source-conventions.baseline.json b/scripts/check-source-conventions.baseline.json index 83492b0745a..84203fe59fe 100644 --- a/scripts/check-source-conventions.baseline.json +++ b/scripts/check-source-conventions.baseline.json @@ -203,7 +203,6 @@ "apps/sim/lib/mothership/chat/persisted-message.test.ts": 1, "apps/sim/lib/mothership/chat/post.test.ts": 1, "apps/sim/lib/mothership/chat/process-contents.ts": 1, - "apps/sim/lib/mothership/chat/selection-clipboard.test.ts": 1, "apps/sim/lib/mothership/chat/selection-context.test.ts": 1, "apps/sim/lib/mothership/chat/sim-key-redaction.test.ts": 1, "apps/sim/lib/mothership/chat/terminal-state.test.ts": 1, diff --git a/scripts/check-test-types.baseline.json b/scripts/check-test-types.baseline.json index e656a8d0318..b8cc7b107e7 100644 --- a/scripts/check-test-types.baseline.json +++ b/scripts/check-test-types.baseline.json @@ -549,8 +549,7 @@ "TS2304": 1, "TS2322": 4, "TS2353": 1, - "TS2550": 1, - "TS2741": 4 + "TS2550": 1 }, "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.test.tsx": { "TS2322": 1 @@ -1318,15 +1317,15 @@ "TS2345": 3, "TS2739": 8 }, - "apps/sim/lib/execution/remote-sandbox/spec.test.ts": { - "TS2345": 2 - }, "apps/sim/lib/execution/remote-sandbox/session-file-snapshot.test.ts": { "TS2322": 1 }, "apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts": { "TS2322": 1 }, + "apps/sim/lib/execution/remote-sandbox/spec.test.ts": { + "TS2345": 2 + }, "apps/sim/lib/file-parsers/odf-text.test.ts": { "TS2739": 4 }, @@ -1642,12 +1641,12 @@ "apps/sim/lib/knowledge/chunks/keyset-sql.test.ts": { "TS2345": 2 }, - "apps/sim/lib/knowledge/connectors/error.test.ts": { - "TS2345": 1 - }, "apps/sim/lib/knowledge/connectors/directory-queue.test.ts": { "TS2322": 1 }, + "apps/sim/lib/knowledge/connectors/error.test.ts": { + "TS2345": 1 + }, "apps/sim/lib/knowledge/connectors/google-company-scheduler.test.ts": { "TS2322": 2 }, @@ -1677,9 +1676,6 @@ "TS2345": 2, "TS2739": 2 }, - "apps/sim/lib/knowledge/documents/processing-source.test.ts": { - "TS2345": 4 - }, "apps/sim/lib/knowledge/documents/processing-lane.test.ts": { "TS2322": 1 }, @@ -1699,6 +1695,9 @@ "apps/sim/lib/knowledge/documents/processing-recovery-queue.test.ts": { "TS2540": 2 }, + "apps/sim/lib/knowledge/documents/processing-source.test.ts": { + "TS2345": 4 + }, "apps/sim/lib/knowledge/documents/retry-processing-grace.test.ts": { "TS2554": 1, "TS2739": 2