fix(file-search): stop requeueing skipped artifacts, and fix public share, live-doc persist and cleanup edge cases

apps/sim/app/api/files/public/[token]/content/route.ts

@@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server'
 import { getClientIp } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import {
+  logPublicFileError,
   publicFileAuthDenied,
   publicFileBinaryErrorResponse,
   publicFileShareCredential,
@@ -58,7 +59,7 @@ export const GET = withRouteHandler(
         request.headers.get('if-none-match')
       )
     } catch (error) {
-      logger.error('Error serving public shared file:', error)
+      logPublicFileError(logger, 'Error serving public shared file:', error)
       return publicFileBinaryErrorResponse(error, 'Failed to serve file')
     }
   }

apps/sim/app/api/files/public/[token]/inline/route.ts

@@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server'
 import { getClientIp } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import {
+  logPublicFileError,
   publicFileAuthDenied,
   publicFileBinaryErrorResponse,
   publicFileShareCredential,
@@ -48,7 +49,7 @@ export const GET = withRouteHandler(
         request.headers.get('if-none-match')
       )
     } catch (error) {
-      logger.error('Error serving public inline image:', error)
+      logPublicFileError(logger, 'Error serving public inline image:', error)
       return publicFileBinaryErrorResponse(error, 'Failed to serve file')
     }
   }

apps/sim/app/api/files/public/[token]/otp/route.ts

@@ -11,7 +11,11 @@ import { OTP_IP_RATE_LIMIT } from '@/lib/core/security/otp'
 import { afterResponse } from '@/lib/core/utils/after-response'
 import { getClientIp } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
-import { publicFileAuthDenied, publicFileErrorResponse } from '@/lib/public-shares/api'
+import {
+  logPublicFileError,
+  publicFileAuthDenied,
+  publicFileErrorResponse,
+} from '@/lib/public-shares/api'
 import {
   requestPublicFileShareOtp,
   verifyPublicFileShareOtp,
@@ -48,7 +52,7 @@ export const POST = withRouteHandler(
       afterResponse(accepted.deliver)
       return NextResponse.json({ message: 'Verification code sent' })
     } catch (error) {
-      logger.error('Error processing OTP request:', error)
+      logPublicFileError(logger, 'Error processing OTP request:', error)
       return publicFileErrorResponse(error, 'Failed to process request')
     }
   }
@@ -69,7 +73,7 @@ export const PUT = withRouteHandler(
       response.cookies.set(auth.cookie)
       return response
     } catch (error) {
-      logger.error('Error verifying OTP:', error)
+      logPublicFileError(logger, 'Error verifying OTP:', error)
       return publicFileErrorResponse(error, 'Failed to process request')
     }
   }

apps/sim/app/api/files/public/[token]/route.ts

@@ -9,6 +9,7 @@ import { parseRequest } from '@/lib/api/server'
 import { getClientIp } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import {
+  logPublicFileError,
   publicFileAuthDenied,
   publicFileErrorResponse,
   publicFileShareCredential,
@@ -48,7 +49,7 @@ export const GET = withRouteHandler(
         ownerName,
       })
     } catch (error) {
-      logger.error('Error fetching public file metadata:', error)
+      logPublicFileError(logger, 'Error fetching public file metadata:', error)
       return publicFileErrorResponse(error, 'Failed to fetch file')
     }
   }
@@ -70,7 +71,7 @@ export const POST = withRouteHandler(
       response.cookies.set(auth.cookie)
       return response
     } catch (error) {
-      logger.error('Error authenticating public file share:', error)
+      logPublicFileError(logger, 'Error authenticating public file share:', error)
       return publicFileErrorResponse(error, 'Failed to authenticate')
     }
   }

apps/sim/app/api/files/public/[token]/sso/route.ts

@@ -6,7 +6,11 @@ import { parseRequest } from '@/lib/api/server'
 import { RateLimiter, type TokenBucketConfig } from '@/lib/core/rate-limiter'
 import { getClientIp } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
-import { publicFileAuthDenied, publicFileErrorResponse } from '@/lib/public-shares/api'
+import {
+  logPublicFileError,
+  publicFileAuthDenied,
+  publicFileErrorResponse,
+} from '@/lib/public-shares/api'
 import { getPublicFileShareSsoEligibility } from '@/lib/public-shares/application'
 
 export const dynamic = 'force-dynamic'
@@ -47,7 +51,7 @@ export const POST = withRouteHandler(
         ? NextResponse.json({ eligible: result.eligible })
         : publicFileAuthDenied(result)
     } catch (error) {
-      logger.error('Error checking public file SSO eligibility:', error)
+      logPublicFileError(logger, 'Error checking public file SSO eligibility:', error)
       return publicFileErrorResponse(error, 'Failed to process request')
     }
   }

apps/sim/app/api/internal/file-doc/persist/route.ts

@@ -3,8 +3,10 @@ import { getErrorMessage } from '@sim/utils/errors'
 import type { NextRequest } from 'next/server'
 import { NextResponse } from 'next/server'
 import { persistFileDocContract } from '@/lib/api/contracts/file-doc'
+import { isOffAppHost } from '@/lib/api/mcp/host-routing'
 import { parseRequest } from '@/lib/api/server'
 import { persistFileDoc } from '@/lib/collab-doc/persist'
+import { MAX_COLLAB_DOC_PERSIST_BODY_BYTES } from '@/lib/collab-doc/state'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { checkInternalApiKey, createUnauthorizedResponse } from '@/lib/mothership/request/http'
 
@@ -16,12 +18,19 @@ const logger = createLogger('FileDocPersistAPI')
  * the shared `x-api-key: INTERNAL_API_SECRET` secret, matching the header the realtime relay sends.
  * The relay owns the live doc but not the conversion engine or blob/DB access, so it ships the current
  * doc state here — the server-authoritative durable path that replaces the editor's client autosave.
+ * Excluded from the proxy matcher, so the route refuses the dedicated MCP host before auth itself.
  */
 export const POST = withRouteHandler(async (request: NextRequest) => {
+  if (isOffAppHost(request)) return new NextResponse(null, { status: 404 })
   const auth = checkInternalApiKey(request)
   if (!auth.success) return createUnauthorizedResponse()
 
-  const parsed = await parseRequest(persistFileDocContract, request, {})
+  const parsed = await parseRequest(
+    persistFileDocContract,
+    request,
+    {},
+    { maxBodyBytes: MAX_COLLAB_DOC_PERSIST_BODY_BYTES, bypassesProxy: true }
+  )
   if (!parsed.success) return parsed.response
   const { workspaceId, fileId, userId, docState, expectedVersion } = parsed.data.body
 

apps/sim/app/api/internal/project-file-doc/[projectId]/[fileId]/persist/route.ts

@@ -7,12 +7,9 @@ import {
   internalRateLimits,
 } from '@/lib/api/server/routes'
 import { realtimeProjectFileAuth } from '@/lib/auth/realtime-file-delegation'
-import { MAX_COLLAB_DOC_STATE_BYTES } from '@/lib/collab-doc/state'
+import { MAX_COLLAB_DOC_PERSIST_BODY_BYTES } from '@/lib/collab-doc/state'
 import { persistProjectFileDoc } from '@/lib/projects/files/application/documents'
 
-/** A base64-encoded maximum snapshot plus 1 MiB for its JSON envelope (17 MiB). */
-const PERSIST_MAX_BODY_BYTES = Math.ceil(MAX_COLLAB_DOC_STATE_BYTES / 3) * 4 + 1024 * 1024
-
 const persist = defineInternalJsonRoute({
   contract: projectFileDocPersistContract,
   operation: persistProjectFileDoc.operation,
@@ -27,7 +24,7 @@ const persist = defineInternalJsonRoute({
     expectedVersion: body.expectedVersion,
   }),
   useCase: persistProjectFileDoc,
-  parseOptions: { maxBodyBytes: PERSIST_MAX_BODY_BYTES, bypassesProxy: true },
+  parseOptions: { maxBodyBytes: MAX_COLLAB_DOC_PERSIST_BODY_BYTES, bypassesProxy: true },
 })
 
 /** Excluded from the proxy matcher, so the route refuses the dedicated MCP host before auth itself. */

apps/sim/lib/collab-doc/state.ts

@@ -7,6 +7,13 @@ import type { DbOrTx, DbTransaction } from '@/lib/db/types'
 /** Matches the decoded size of the persist endpoint's 16 MiB base64 snapshot limit. */
 export const MAX_COLLAB_DOC_STATE_BYTES = 12 * 1024 * 1024
 
+/**
+ * Body cap of the relay's persist routes: a base64-encoded maximum snapshot plus 1 MiB for its JSON
+ * envelope (17 MiB). Both routes bypass the proxy, whose 10 MB buffer would truncate it.
+ */
+export const MAX_COLLAB_DOC_PERSIST_BODY_BYTES =
+  Math.ceil(MAX_COLLAB_DOC_STATE_BYTES / 3) * 4 + 1024 * 1024
+
 /** Reject oversized snapshots before copying, decoding, or writing them. */
 export function assertCollabDocStateSize(docState: Uint8Array): void {
   if (docState.byteLength > MAX_COLLAB_DOC_STATE_BYTES) {

apps/sim/lib/mothership/memories/cleanup-outbox.ts

@@ -29,9 +29,15 @@ const logger = createLogger('MemoryCleanup')
 const UNAVAILABLE_RETRY_MS = 60 * 60 * 1000
 /**
  * Statuses meaning the worker does not serve cleanup yet, not that this request failed: a worker
- * that predates the route answers 404 to the hosted key and 401 to a self-hosted customer key.
+ * that predates the route answers 404 to the hosted key.
  */
-const ENDPOINT_MISSING_STATUSES = new Set([401, 404, 405, 501])
+const ENDPOINT_MISSING_STATUSES = new Set([404, 405, 501])
+/**
+ * A worker that predates the route also answers this to a self-hosted customer key, but so does a
+ * worker that revoked the key for good: it waits like a missing route but spends an attempt, so a
+ * revoked key dead-letters after the default attempts (about ten hours), past a worker rollout.
+ */
+const UNAUTHORIZED_STATUS = 401
 
 /** Non-strict so rows enqueued with the retired `credentialFingerprint` field still parse. */
 const cleanupPayloadSchema = z.object({
@@ -74,16 +80,21 @@ export async function enqueueMemoryCleanup(
   }
 }
 
-function waitForWorker(reason: string, eventId: string, detail?: Record<string, string>) {
+function waitForWorker(
+  reason: string,
+  eventId: string,
+  { consumeAttempt = false, ...detail }: { baseUrl?: string; consumeAttempt?: boolean } = {}
+) {
   logger.warn(`Memory cleanup waiting: ${reason}`, { eventId, ...detail })
-  return deferOutboxHandler(reason, UNAVAILABLE_RETRY_MS, false)
+  return deferOutboxHandler(reason, UNAVAILABLE_RETRY_MS, consumeAttempt)
 }
 
 /**
  * Sends the processor's current key only to a destination the processor itself configures, and
  * replays the enqueuing deployment's source env so the worker erases that deployment's notes.
  * A missing key, destination, or worker endpoint waits without consuming attempts: erasure must
- * not dead-letter while configuration or the worker rollout catches up.
+ * not dead-letter while configuration or the worker rollout catches up. A rejected key waits too,
+ * but within the retry budget.
  */
 export const memoryCleanupOutboxHandlers = {
   [MEMORY_CLEANUP_EVENT]: async (rawPayload, context) => {
@@ -109,6 +120,10 @@ export const memoryCleanupOutboxHandlers = {
       await response.body?.cancel().catch(() => {})
       if (ENDPOINT_MISSING_STATUSES.has(response.status))
         return waitForWorker(`endpoint unavailable (${response.status})`, context.eventId)
+      if (response.status === UNAUTHORIZED_STATUS)
+        return waitForWorker('worker rejected the key (401)', context.eventId, {
+          consumeAttempt: true,
+        })
       throw new Error(`Memory cleanup failed (${response.status})`)
     }
     MemoryCleanupResult.parse(await response.json())

apps/sim/lib/mothership/memories/cleanup.integration.ts

@@ -139,10 +139,10 @@ it('rolls cleanup back with deletion and retries committed account/target erasur
   expect(received.every((request) => request.key === 'memory-cleanup-fixture-key')).toBe(true)
 
   const first = events[0]
-  const replay = async (payload: unknown = first.payload) => {
+  const replay = async (payload: unknown = first.payload, attempts = 9) => {
     await db
       .update(outboxEvent)
-      .set({ status: 'pending', availableAt: new Date(0), attempts: 9, payload })
+      .set({ status: 'pending', availableAt: new Date(0), attempts, payload })
       .where(eq(outboxEvent.id, first.id))
     await runtime.outbox.processOutboxEventById(
       first.id,
@@ -161,11 +161,18 @@ it('rolls cleanup back with deletion and retries committed account/target erasur
   waitsAnHour(await replay())
   expect(received).toHaveLength(before + 1)
 
+  status = 401
+  const rejected = await replay(first.payload, 5)
+  expect(rejected).toMatchObject({ status: 'pending', attempts: 6 })
+  expect(rejected.availableAt.getTime()).toBeGreaterThan(Date.now() + 50 * 60_000)
+  expect(await replay()).toMatchObject({ status: 'dead_letter', attempts: 10 })
+  expect(received).toHaveLength(before + 3)
+
   status = 200
   waitsAnHour(await replay({ ...toRecord(first.payload), baseUrl: 'https://unconfigured.test' }))
-  expect(received).toHaveLength(before + 1)
+  expect(received).toHaveLength(before + 3)
 
   const legacy = await replay({ ...toRecord(first.payload), credentialFingerprint: 'retired' })
   expect(legacy.status).toBe('completed')
-  expect(received).toHaveLength(before + 2)
+  expect(received).toHaveLength(before + 4)
 }, 30_000)

apps/sim/lib/public-shares/api/index.ts

@@ -1,5 +1,6 @@
 export { publicFileShareCredential } from './credentials'
 export {
+  logPublicFileError,
   publicFileAuthDenied,
   publicFileBinaryErrorResponse,
   publicFileErrorResponse,

apps/sim/lib/public-shares/api/responses.ts

@@ -1,3 +1,4 @@
+import type { Logger } from '@sim/logger'
 import { NextResponse } from 'next/server'
 import {
   asOrchestrationError,
@@ -39,6 +40,12 @@ function classifyPublicFileError(error: unknown, fallback: string) {
   }
 }
 
+/** A classified client failure, such as a missing share or the wrong auth mode, is only a warning. */
+export function logPublicFileError(logger: Logger, message: string, error: unknown) {
+  if (classifyPublicFileError(error, '').status < 500) logger.warn(message, error)
+  else logger.error(message, error)
+}
+
 /** Conceal unavailable targets and driver failures while preserving classified domain failures. */
 export function publicFileErrorResponse(error: unknown, fallback: string) {
   const result = classifyPublicFileError(error, fallback)

apps/sim/lib/public-shares/application/read.ts

@@ -37,7 +37,11 @@ import { resolveEffectiveMimeType } from '@/lib/uploads/utils/file'
 import { sniffImageContentType } from '@/lib/uploads/utils/validation'
 import { reportWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer'
 import { fileOwnerCondition } from '@/lib/workspace-files/ownership-query'
-import { isSimPageSource, SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile'
+import {
+  isSimPageSource,
+  ProjectPageForeignEmbedError,
+  SIM_PAGE_CONTENT_TYPE,
+} from '@/lib/workspace-files/page-compile'
 import { renderSimPageDocument } from '@/lib/workspace-files/page-document'
 import {
   collectSimPageFileReferences,
@@ -70,6 +74,16 @@ function present(snapshot: PublicFileShareSnapshot) {
     },
   }
 }
+/** A Project page embedding another owner's file reads as a missing input, as for members. */
+function renderSharedPage(source: string, snapshot: PublicFileShareSnapshot) {
+  try {
+    return renderSimPageDocument(source, snapshot.adapter.pageOptions(snapshot.owner.entityId))
+  } catch (error) {
+    if (error instanceof ProjectPageForeignEmbedError)
+      throw new OrchestrationError('not_found', 'Document input not found')
+    throw error
+  }
+}
 function requireRevision(current: WorkspaceFileRow, expected: WorkspaceFileRow) {
   if (
     current.id !== expected.id ||
@@ -287,7 +301,7 @@ export async function readPublicFileShareContent({
           (snapshot.file.contentType === SIM_PAGE_CONTENT_TYPE ||
             snapshot.file.originalName.toLowerCase().endsWith('.html')) &&
           isSimPageSource(source)
-            ? renderSimPageDocument(source, snapshot.adapter.pageOptions(snapshot.owner.entityId))
+            ? renderSharedPage(source, snapshot)
             : undefined
         const references = pageHtml ? collectSimPageFileReferences(pageHtml) : []
         if (
@@ -386,9 +400,7 @@ export async function readPublicFileShareInline({
       operation,
       async (tx, snapshot): Promise<PublicManifest> => {
         requireRevision(snapshot.file, initial.file)
-        const pageHtml = isSimPageSource(source)
-          ? renderSimPageDocument(source, snapshot.adapter.pageOptions(snapshot.owner.entityId))
-          : undefined
+        const pageHtml = isSimPageSource(source) ? renderSharedPage(source, snapshot) : undefined
         const reference = fileId ? { fileId } : key ? { key } : null
         const references = pageHtml ? collectSimPageFileReferences(pageHtml) : []
         const embedded =

apps/sim/lib/workspace-files/search/artifact-ready.ts

@@ -4,7 +4,7 @@ import {
   workspaceFileSearchBuild,
   workspaceFileSearchRevision,
 } from '@sim/db/schema'
-import { and, eq, or, sql } from 'drizzle-orm'
+import { and, eq, inArray, or, sql } from 'drizzle-orm'
 import type { DbTransaction } from '@/lib/db/types'
 import type { EditableFileOwner } from '@/lib/workspace-files/ownership'
 import { FILE_SEARCH_ARTIFACT_READY_LOCK_TIMEOUT_MS } from '@/lib/workspace-files/search/constants'
@@ -51,7 +51,8 @@ function artifactRevisionCondition(input: FileSearchArtifactCompletion, owner: E
 
 /**
  * Lock-free form of {@link markFileSearchArtifactReadyInTx}'s no-op test: the file's current
- * revision is already built from this artifact, ready or still under a live build lease.
+ * revision is already built from this artifact, published (ready or skipped) or still under a live
+ * build lease.
  */
 async function isFileSearchArtifactCurrent(
   input: FileSearchArtifactCompletion,
@@ -69,7 +70,7 @@ async function isFileSearchArtifactCurrent(
         artifactRevisionCondition(input, owner),
         eq(workspaceFileSearchBuild.artifactKey, input.artifactKey),
         or(
-          eq(workspaceFileSearchRevision.status, 'ready'),
+          inArray(workspaceFileSearchRevision.status, ['ready', 'skipped']),
           and(
             eq(workspaceFileSearchRevision.status, 'pending'),
             sql`${workspaceFileSearchBuild.expiresAt} > clock_timestamp()`
@@ -128,7 +129,9 @@ export async function markFileSearchArtifactReadyInTx(
       .for('update')
     if (
       build?.artifactKey === input.artifactKey &&
-      (state.status === 'ready' || (state.status === 'pending' && build.live))
+      (state.status === 'ready' ||
+        state.status === 'skipped' ||
+        (state.status === 'pending' && build.live))
     )
       return false
     await tx