From 190b815487193f85753d1337578a0e3e66707116 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 19:19:51 -0700 Subject: [PATCH 1/4] fix(realtime): end the socket attempt when the token mint fails The Socket.IO auth callback never invoked cb when POST /api/auth/socket-token failed (429, 5xx, network), so the open transport sat until the realtime server's 45 s connectTimeout while the UI showed Reconnecting, and Retry-After was ignored. The client `timeout` cannot help: the Manager clears it when the transport opens, and resets its backoff on every open, so each retry ran at the 1 s floor. On a retryable mint failure, disconnect the socket and reconnect it on the existing auth backoff (1 s to 30 s, matching the reconnection config), never before the response's Retry-After. 401 keeps its cb({ token: null }) path. --- .../workspace/providers/socket-provider.tsx | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/apps/sim/app/workspace/providers/socket-provider.tsx b/apps/sim/app/workspace/providers/socket-provider.tsx index 020505ae161..c16f86b7b84 100644 --- a/apps/sim/app/workspace/providers/socket-provider.tsx +++ b/apps/sim/app/workspace/providers/socket-provider.tsx @@ -26,7 +26,7 @@ import type { WorkflowUpdatedBroadcast, } from '@sim/realtime-protocol/events' import { generateId } from '@sim/utils/id' -import { backoffWithJitter } from '@sim/utils/retry' +import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { useQueryClient } from '@tanstack/react-query' import { useParams } from 'next/navigation' import type { Socket } from 'socket.io-client' @@ -70,6 +70,22 @@ const MAX_AUTH_RETRY_ATTEMPTS = 5 const AUTH_RETRY_BASE_MS = 1000 const AUTH_RETRY_MAX_MS = 30000 +/** + * Upper bound on an honored socket-token `Retry-After`. The token route's IP + * bucket refills over 60 s, so the default 30 s cap would retry into a 429. + */ +const SOCKET_TOKEN_RETRY_AFTER_MAX_MS = 5 * 60_000 + +/** A socket-token mint that failed for a retryable reason, with the server's `Retry-After`. */ +class SocketTokenUnavailableError extends Error { + readonly retryAfterMs: number | null + + constructor(retryAfterMs: number | null) { + super('Failed to generate socket token') + this.retryAfterMs = retryAfterMs + } +} + function getTabSessionId(): string { if (typeof window === 'undefined') return '' @@ -185,6 +201,7 @@ export function SocketProvider({ children, user }: SocketProviderProps) { const joinRetryTimeoutRef = useRef | null>(null) const authRetryAttemptsRef = useRef(0) const authRetryTimeoutRef = useRef | null>(null) + const tokenRetryAttemptsRef = useRef(0) const sessionRejectedRef = useRef(false) const connectFailureCountRef = useRef(0) const connectFailureReportedRef = useRef(false) @@ -367,7 +384,9 @@ export function SocketProvider({ children, user }: SocketProviderProps) { if (res.status === 401) { throw new Error('Authentication required') } - throw new Error('Failed to generate socket token') + throw new SocketTokenUnavailableError( + parseRetryAfter(res.headers.get('retry-after'), SOCKET_TOKEN_RETRY_AFTER_MAX_MS) + ) } const body = await res.json().catch(() => ({})) const token = body?.token @@ -428,8 +447,36 @@ export function SocketProvider({ children, user }: SocketProviderProps) { // True auth failure - pass null token, server will reject with "Authentication required" sessionRejectedRef.current = true cb({ token: null }) + return } - // For server errors, don't call cb - connection will timeout and Socket.IO will retry + if (!socketInstance.active || socketInstance.connected) return + + /* Withholding `cb` would hold the open transport until the realtime + server's 45 s connect timeout, and the Manager resets its backoff on + every transport open, so it cannot pace this retry. End the attempt + and reconnect on our own backoff, never before `Retry-After`. */ + tokenRetryAttemptsRef.current += 1 + const retryAfterMs = + error instanceof SocketTokenUnavailableError ? error.retryAfterMs : null + const delayMs = Math.max( + backoffWithJitter(tokenRetryAttemptsRef.current, null, { + baseMs: AUTH_RETRY_BASE_MS, + maxMs: AUTH_RETRY_MAX_MS, + }), + retryAfterMs ?? 0 + ) + socketInstance.disconnect() + setIsConnecting(false) + setIsReconnecting(true) + logger.warn('Socket token unavailable, reconnecting after delay', { + attempt: tokenRetryAttemptsRef.current, + delayMs: Math.round(delayMs), + }) + clearAuthRetryTimeout() + authRetryTimeoutRef.current = setTimeout(() => { + authRetryTimeoutRef.current = null + socketInstance.connect() + }, delayMs) } }, }) @@ -478,6 +525,7 @@ export function SocketProvider({ children, user }: SocketProviderProps) { setIsConnecting(false) setIsReconnecting(false) authRetryAttemptsRef.current = 0 + tokenRetryAttemptsRef.current = 0 connectFailureCountRef.current = 0 connectFailureReportedRef.current = false clearAuthRetryTimeout() From fe7e14735e2dce0a5da6da6d96b75760e6649d14 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 19:23:45 -0700 Subject: [PATCH 2/4] fix(realtime): clear connected state and jitter Retry-After on token retry disconnect() on a socket that has not connected yet emits no 'disconnect' event. After a native reconnect is followed by a failed token mint, isConnected stayed true for the whole outage. Clear it, and the socket id, exactly as the 'disconnect' handler does. Spread a Retry-After-driven delay by up to +20% above the floor, so tabs behind one IP do not retry in the same second. --- .../workspace/providers/socket-provider.tsx | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/apps/sim/app/workspace/providers/socket-provider.tsx b/apps/sim/app/workspace/providers/socket-provider.tsx index c16f86b7b84..59111a3b9f5 100644 --- a/apps/sim/app/workspace/providers/socket-provider.tsx +++ b/apps/sim/app/workspace/providers/socket-provider.tsx @@ -26,6 +26,7 @@ import type { WorkflowUpdatedBroadcast, } from '@sim/realtime-protocol/events' import { generateId } from '@sim/utils/id' +import { randomFloat } from '@sim/utils/random' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { useQueryClient } from '@tanstack/react-query' import { useParams } from 'next/navigation' @@ -76,6 +77,9 @@ const AUTH_RETRY_MAX_MS = 30000 */ const SOCKET_TOKEN_RETRY_AFTER_MAX_MS = 5 * 60_000 +/** Up to +20% spread above a `Retry-After` floor, matching `backoffWithJitter`'s band. */ +const SOCKET_TOKEN_RETRY_AFTER_JITTER = 0.2 + /** A socket-token mint that failed for a retryable reason, with the server's `Retry-After`. */ class SocketTokenUnavailableError extends Error { readonly retryAfterMs: number | null @@ -458,15 +462,22 @@ export function SocketProvider({ children, user }: SocketProviderProps) { tokenRetryAttemptsRef.current += 1 const retryAfterMs = error instanceof SocketTokenUnavailableError ? error.retryAfterMs : null - const delayMs = Math.max( - backoffWithJitter(tokenRetryAttemptsRef.current, null, { - baseMs: AUTH_RETRY_BASE_MS, - maxMs: AUTH_RETRY_MAX_MS, - }), - retryAfterMs ?? 0 - ) + const backoffMs = backoffWithJitter(tokenRetryAttemptsRef.current, null, { + baseMs: AUTH_RETRY_BASE_MS, + maxMs: AUTH_RETRY_MAX_MS, + }) + // Spread tabs behind one IP that share the same 429 window. + const delayMs = + retryAfterMs === null + ? backoffMs + : Math.max( + backoffMs, + retryAfterMs * (1 + SOCKET_TOKEN_RETRY_AFTER_JITTER * randomFloat()) + ) socketInstance.disconnect() + setIsConnected(false) setIsConnecting(false) + setCurrentSocketId(null) setIsReconnecting(true) logger.warn('Socket token unavailable, reconnecting after delay', { attempt: tokenRetryAttemptsRef.current, From ccd81bac34427128ffd177ac70f82b59e59113de Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 19:24:22 -0700 Subject: [PATCH 3/4] fix(realtime): tighten the token-retry comment --- apps/sim/app/workspace/providers/socket-provider.tsx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/sim/app/workspace/providers/socket-provider.tsx b/apps/sim/app/workspace/providers/socket-provider.tsx index 59111a3b9f5..25b7efcd46a 100644 --- a/apps/sim/app/workspace/providers/socket-provider.tsx +++ b/apps/sim/app/workspace/providers/socket-provider.tsx @@ -455,10 +455,8 @@ export function SocketProvider({ children, user }: SocketProviderProps) { } if (!socketInstance.active || socketInstance.connected) return - /* Withholding `cb` would hold the open transport until the realtime - server's 45 s connect timeout, and the Manager resets its backoff on - every transport open, so it cannot pace this retry. End the attempt - and reconnect on our own backoff, never before `Retry-After`. */ + // Withholding `cb` would park the transport until the server's 45 s connect + // timeout, and the Manager resets its backoff on every open, so pace it here. tokenRetryAttemptsRef.current += 1 const retryAfterMs = error instanceof SocketTokenUnavailableError ? error.retryAfterMs : null From 7adc8a371fe43b403003585fc2aff40f3b25d79c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 19:35:21 -0700 Subject: [PATCH 4/4] fix(realtime): ignore token failures from a superseded handshake and report persistent token outages --- .../workspace/providers/socket-provider.tsx | 95 +++++++++++-------- 1 file changed, 55 insertions(+), 40 deletions(-) diff --git a/apps/sim/app/workspace/providers/socket-provider.tsx b/apps/sim/app/workspace/providers/socket-provider.tsx index 25b7efcd46a..3188a98d5d8 100644 --- a/apps/sim/app/workspace/providers/socket-provider.tsx +++ b/apps/sim/app/workspace/providers/socket-provider.tsx @@ -25,6 +25,7 @@ import type { WorkflowRevertedBroadcast, WorkflowUpdatedBroadcast, } from '@sim/realtime-protocol/events' +import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { randomFloat } from '@sim/utils/random' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' @@ -430,6 +431,51 @@ export function SocketProvider({ children, user }: SocketProviderProps) { timestamp: new Date().toISOString(), }) + /** + * Reports a realtime connection that is not coming back. + * + * A socket that never connects raises no exception anywhere — every + * failure path here is handled, so error tracking sees nothing and the + * only user-visible trace is a "Reconnecting…" toast. Socket.IO then + * retries the same URL forever, so the failure is both permanent and + * silent. This is the one signal that distinguishes "the realtime + * service is down" from "this client resolved the wrong URL", which is + * why the origin is reported alongside the reason. + * + * Fires at most once per socket instance, at the point the toast + * appears, rather than once per retry. + * + * Called once per failed attempt: from `connect_error`, and from the auth + * callback when it ends an attempt itself because no token could be minted + * (that attempt never reaches `connect_error`). `connect_error` is the only + * socket handler that sees exactly one event per attempt: a failed + * *reconnect* also emits the manager's `reconnect_error` (`manager.reconnect()` calls + * `open()`, whose error path emits `error` — which the socket re-emits as + * `connect_error` — and then emits `reconnect_error` itself), so counting + * in both would advance twice per try and trip the threshold early. + */ + const reportPersistentConnectFailure = (reason: string) => { + connectFailureCountRef.current += 1 + if ( + connectFailureReportedRef.current || + connectFailureCountRef.current < CONNECT_FAILURES_BEFORE_REPORT + ) { + return + } + connectFailureReportedRef.current = true + + captureClientEvent('realtime_connection_failing', { + socket_origin: socketOrigin, + expected_socket_origin_configured: Boolean(getEnv('NEXT_PUBLIC_SOCKET_URL')?.trim()), + attempts: connectFailureCountRef.current, + reason, + }) + } + + // Each auth callback owns one handshake; a token request that outlives it must + // not end the attempt that replaced it. + let authAttemptGeneration = 0 + const socketInstance = io(socketUrl, { transports: ['websocket', 'polling'], withCredentials: true, @@ -442,6 +488,7 @@ export function SocketProvider({ children, user }: SocketProviderProps) { // otherwise one early 401 would still be set when a later, unrelated // denial exhausts the retry budget. sessionRejectedRef.current = false + const attempt = ++authAttemptGeneration try { const freshToken = await generateSocketToken() cb({ token: freshToken }) @@ -453,7 +500,13 @@ export function SocketProvider({ children, user }: SocketProviderProps) { cb({ token: null }) return } - if (!socketInstance.active || socketInstance.connected) return + if ( + attempt !== authAttemptGeneration || + !socketInstance.active || + socketInstance.connected + ) { + return + } // Withholding `cb` would park the transport until the server's 45 s connect // timeout, and the Manager resets its backoff on every open, so pace it here. @@ -477,6 +530,7 @@ export function SocketProvider({ children, user }: SocketProviderProps) { setIsConnecting(false) setCurrentSocketId(null) setIsReconnecting(true) + reportPersistentConnectFailure(getErrorMessage(error)) logger.warn('Socket token unavailable, reconnecting after delay', { attempt: tokenRetryAttemptsRef.current, delayMs: Math.round(delayMs), @@ -490,45 +544,6 @@ export function SocketProvider({ children, user }: SocketProviderProps) { }, }) - /** - * Reports a realtime connection that is not coming back. - * - * A socket that never connects raises no exception anywhere — every - * failure path here is handled, so error tracking sees nothing and the - * only user-visible trace is a "Reconnecting…" toast. Socket.IO then - * retries the same URL forever, so the failure is both permanent and - * silent. This is the one signal that distinguishes "the realtime - * service is down" from "this client resolved the wrong URL", which is - * why the origin is reported alongside the reason. - * - * Fires at most once per socket instance, at the point the toast - * appears, rather than once per retry. - * - * Called from `connect_error` and nowhere else. That is the only handler - * that sees exactly one event per attempt: a failed *reconnect* also - * emits the manager's `reconnect_error` (`manager.reconnect()` calls - * `open()`, whose error path emits `error` — which the socket re-emits as - * `connect_error` — and then emits `reconnect_error` itself), so counting - * in both would advance twice per try and trip the threshold early. - */ - const reportPersistentConnectFailure = (reason: string) => { - connectFailureCountRef.current += 1 - if ( - connectFailureReportedRef.current || - connectFailureCountRef.current < CONNECT_FAILURES_BEFORE_REPORT - ) { - return - } - connectFailureReportedRef.current = true - - captureClientEvent('realtime_connection_failing', { - socket_origin: socketOrigin, - expected_socket_origin_configured: Boolean(getEnv('NEXT_PUBLIC_SOCKET_URL')?.trim()), - attempts: connectFailureCountRef.current, - reason, - }) - } - socketInstance.on('connect', () => { setIsConnected(true) setIsConnecting(false)