fix(executor): add exponential backoff retry for transient LLM provid…

apps/sim/executor/execution/block-executor.ts

@@ -36,6 +36,7 @@ import {
 } from '@/executor/types'
 import { streamingResponseFormatProcessor } from '@/executor/utils'
 import { buildBlockExecutionError, normalizeError } from '@/executor/utils/errors'
+import { withRetry } from '@/executor/utils/retry'
 import {
   buildUnifiedParentIterations,
   getIterationContext,
@@ -120,9 +121,11 @@ export class BlockExecutor {
     cleanupSelfReference?.()
 
     try {
-      const output = handler.executeWithNode
-        ? await handler.executeWithNode(ctx, block, resolvedInputs, nodeMetadata)
-        : await handler.execute(ctx, block, resolvedInputs)
+      const output = await withRetry(
+        () => handler.executeWithNode
+          ? handler.executeWithNode(ctx, block, resolvedInputs, nodeMetadata)
+          : handler.execute(ctx, block, resolvedInputs)
+      )
 
       const isStreamingExecution =
         output && typeof output === 'object' && 'stream' in output && 'execution' in output

apps/sim/executor/utils/retry.test.ts

@@ -0,0 +1,74 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+import { withRetry } from './retry'
+
+describe('withRetry', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  it('returns result immediately on success', async () => {
+    const fn = vi.fn().mockResolvedValue('ok')
+    const result = await withRetry(fn)
+    expect(result).toBe('ok')
+    expect(fn).toHaveBeenCalledTimes(1)
+  })
+
+  it('retries on 429 and succeeds on second attempt', async () => {
+    const error = Object.assign(new Error('rate limited'), { status: 429 })
+    const fn = vi.fn()
+      .mockRejectedValueOnce(error)
+      .mockResolvedValue('ok')
+    const promise = withRetry(fn, { initialDelayMs: 10, maxRetries: 3 })
+    await vi.runAllTimersAsync()
+    const result = await promise
+    expect(result).toBe('ok')
+    expect(fn).toHaveBeenCalledTimes(2)
+  })
+
+  it('retries up to maxRetries times then throws', async () => {
+    const error = Object.assign(new Error('service unavailable'), { status: 503 })
+    const fn = vi.fn().mockRejectedValue(error)
+    const promise = withRetry(fn, { maxRetries: 3, initialDelayMs: 10 })
+    await vi.runAllTimersAsync()
+    await expect(promise).rejects.toThrow('service unavailable')
+    expect(fn).toHaveBeenCalledTimes(4)
+  })
+
+  it('does NOT retry on 400 bad request', async () => {
+    const error = Object.assign(new Error('bad request'), { status: 400 })
+    const fn = vi.fn().mockRejectedValue(error)
+    await expect(withRetry(fn, { maxRetries: 3 })).rejects.toThrow('bad request')
+    expect(fn).toHaveBeenCalledTimes(1)
+  })
+
+  it('does NOT retry on 401 unauthorized', async () => {
+    const error = Object.assign(new Error('unauthorized'), { status: 401 })
+    const fn = vi.fn().mockRejectedValue(error)
+    await expect(withRetry(fn, { maxRetries: 3 })).rejects.toThrow('unauthorized')
+    expect(fn).toHaveBeenCalledTimes(1)
+  })
+
+  it('retries on network error with no status code', async () => {
+    const error = new Error('network failure')
+    const fn = vi.fn()
+      .mockRejectedValueOnce(error)
+      .mockResolvedValue('recovered')
+    const promise = withRetry(fn, { maxRetries: 3, initialDelayMs: 10 })
+    await vi.runAllTimersAsync()
+    const result = await promise
+    expect(result).toBe('recovered')
+    expect(fn).toHaveBeenCalledTimes(2)
+  })
+
+  it('respects Retry-After header on 429', async () => {
+    const headers = new Headers({ 'retry-after': '2' })
+    const error = Object.assign(new Error('rate limited'), { status: 429, headers })
+    const fn = vi.fn()
+      .mockRejectedValueOnce(error)
+      .mockResolvedValue('ok')
+    const promise = withRetry(fn, { maxRetries: 3, initialDelayMs: 100 })
+    await vi.runAllTimersAsync()
+    await promise
+    expect(fn).toHaveBeenCalledTimes(2)
+  })
+})

apps/sim/executor/utils/retry.ts

@@ -0,0 +1,95 @@
+import { createLogger } from '@/lib/logging/client'
+
+const logger = createLogger('retry')
+
+export interface RetryOptions {
+  maxRetries?: number
+  initialDelayMs?: number
+  maxDelayMs?: number
+  retryableStatusCodes?: number[]
+}
+
+const RETRY_DEFAULTS = {
+  MAX_RETRIES: 3,
+  INITIAL_DELAY_MS: 1000,
+  MAX_DELAY_MS: 30000,
+  RETRYABLE_STATUS_CODES: [429, 503, 529],
+} as const
+
+function calculateBackoffDelay(attempt: number, initialDelayMs: number, maxDelayMs: number): number {
+  const exponential = initialDelayMs * Math.pow(2, attempt)
+  const capped = Math.min(maxDelayMs, exponential)
+  const jitter = Math.random() * capped * 0.2
+  return Math.floor(capped + jitter)
+}
+
+function parseRetryAfterHeader(headers: Headers): number | null {
+  const retryAfter = headers.get('retry-after')
+  if (!retryAfter) return null
+  const seconds = parseFloat(retryAfter)
+  if (!isNaN(seconds)) return Math.ceil(seconds * 1000)
+  const date = new Date(retryAfter)
+  if (!isNaN(date.getTime())) {
+    const delayMs = date.getTime() - Date.now()
+    return delayMs > 0 ? delayMs : null
+  }
+  return null
+}
+
+/**
+ * Wraps an async function with retry logic using exponential backoff and jitter.
+ * Respects Retry-After headers from LLM providers on 429 responses.
+ * Only retries on transient errors (429, 503, 529) — never on user errors (4xx).
+ */
+export async function withRetry<T>(
+  fn: () => Promise<T>,
+  options: RetryOptions = {}
+): Promise<T> {
+  const maxRetries = options.maxRetries ?? RETRY_DEFAULTS.MAX_RETRIES
+  const initialDelayMs = options.initialDelayMs ?? RETRY_DEFAULTS.INITIAL_DELAY_MS
+  const maxDelayMs = options.maxDelayMs ?? RETRY_DEFAULTS.MAX_DELAY_MS
+  const retryableStatusCodes = options.retryableStatusCodes ?? RETRY_DEFAULTS.RETRYABLE_STATUS_CODES
+
+  let lastError: unknown
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await fn()
+    } catch (error) {
+      lastError = error
+
+      if (attempt === maxRetries) break
+
+      const status = (error as any)?.status ?? (error as any)?.statusCode ?? null
+      const responseHeaders: Headers | null = (error as any)?.headers ?? null
+
+      const isRetryable =
+        status === null ||
+        retryableStatusCodes.includes(status)
+
+      if (!isRetryable) {
+        logger.warn('Non-retryable error, aborting retry loop', { status, attempt })
+        throw error
+      }
+
+      let delayMs: number
+      if (responseHeaders && status === 429) {
+        const retryAfterMs = parseRetryAfterHeader(responseHeaders)
+        delayMs = retryAfterMs ?? calculateBackoffDelay(attempt, initialDelayMs, maxDelayMs)
+      } else {
+        delayMs = calculateBackoffDelay(attempt, initialDelayMs, maxDelayMs)
+      }
+
+      logger.warn('Retrying after transient LLM provider error', {
+        attempt: attempt + 1,
+        maxRetries,
+        status,
+        delayMs,
+      })
+
+      await new Promise((resolve) => setTimeout(resolve, delayMs))
+    }
+  }
+
+  throw lastError
+}