[BR-1794]: Add server error retry with exponential backoff

src/app/core/constants.ts

@@ -4,6 +4,10 @@ export const HTTP_CODES = {
   FORBIDDEN: 403,
   NOT_FOUND: 404,
   TOO_MANY_REQUESTS: 429,
+  INTERNAL_SERVER_ERROR: 500,
+  BAD_GATEWAY: 502,
+  SERVICE_UNAVAILABLE: 503,
+  GATEWAY_TIMEOUT: 504,
 };
 export enum ErrorMessages {
   ServerUnavailable = 'Server Unavailable',
@@ -12,4 +16,5 @@ export enum ErrorMessages {
   NetworkError = 'Network Error',
   ConnectionLost = 'Connection lost',
   FilePickerCancelled = 'File picker was canceled or failed',
+  CORS = 'cors',
 }

src/app/network/retry-with-rate-limit.test.ts → src/app/network/retry-with-backoff.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { retryWithBackoff } from './retry-with-rate-limit';
+import { retryWithBackoff, RetryReason } from './retry-with-backoff';
 import * as timeUtils from 'utils/timeUtils';
+import { ConnectionLostError } from './requests';
 
 vi.mock('utils/timeUtils', () => ({
   wait: vi.fn(() => Promise.resolve()),
@@ -39,16 +40,16 @@ describe('retryWithBackoff', () => {
     expect(timeUtils.wait).toHaveBeenCalledWith(5000);
   });
 
-  it('when error is not rate limit then throws immediately without retry', async () => {
-    const mockFn = vi.fn().mockRejectedValue({ status: 500, message: 'Internal Server Error' });
+  it('when error is not retryable then throws immediately without retry', async () => {
+    const mockFn = vi.fn().mockRejectedValue({ status: 400, message: 'Bad Request' });
 
-    await expect(retryWithBackoff(mockFn)).rejects.toEqual({ status: 500, message: 'Internal Server Error' });
+    await expect(retryWithBackoff(mockFn)).rejects.toEqual({ status: 400, message: 'Bad Request' });
 
     expect(mockFn).toHaveBeenCalledTimes(1);
     expect(timeUtils.wait).not.toHaveBeenCalled();
   });
 
-  it('when rate limited multiple times then calls onRetry for each retry', async () => {
+  it('when rate limited multiple times then calls onRetry for each retry with reason', async () => {
     const error = createRateLimitError('1000');
     const mockFn = vi.fn().mockRejectedValueOnce(error).mockRejectedValueOnce(error).mockResolvedValueOnce('success');
 
@@ -59,8 +60,8 @@ describe('retryWithBackoff', () => {
     expect(result).toBe('success');
     expect(mockFn).toHaveBeenCalledTimes(3);
     expect(onRetry).toHaveBeenCalledTimes(2);
-    expect(onRetry).toHaveBeenNthCalledWith(1, 1, 1000);
-    expect(onRetry).toHaveBeenNthCalledWith(2, 2, 1000);
+    expect(onRetry).toHaveBeenNthCalledWith(1, 1, 1000, RetryReason.RateLimit);
+    expect(onRetry).toHaveBeenNthCalledWith(2, 2, 1000, RetryReason.RateLimit);
   });
 
   it('when headers missing or invalid then throws error', async () => {
@@ -89,16 +90,59 @@ describe('retryWithBackoff', () => {
     expect(timeUtils.wait).toHaveBeenCalledTimes(2);
   });
 
-  it('when error is not an object then throws immediately without retry', async () => {
-    const testCases = ['string error', null];
+  it('when server error occurs then retries and succeeds', async () => {
+    const mockFn = vi
+      .fn()
+      .mockRejectedValueOnce({ status: 500, message: 'Server Error' })
+      .mockResolvedValueOnce('success');
 
-    for (const error of testCases) {
-      const mockFn = vi.fn().mockRejectedValue(error);
-      await expect(retryWithBackoff(mockFn)).rejects.toBe(error);
-      expect(mockFn).toHaveBeenCalledTimes(1);
-      vi.clearAllMocks();
-    }
+    const result = await retryWithBackoff(mockFn);
 
-    expect(timeUtils.wait).not.toHaveBeenCalled();
+    expect(result).toBe('success');
+    expect(mockFn).toHaveBeenCalledTimes(2);
+    expect(timeUtils.wait).toHaveBeenCalledTimes(1);
+  });
+
+  it('when CORS error occurs then retries and succeeds', async () => {
+    const mockFn = vi.fn().mockRejectedValueOnce({ message: 'cors error' }).mockResolvedValueOnce('success');
+
+    const result = await retryWithBackoff(mockFn);
+
+    expect(result).toBe('success');
+    expect(mockFn).toHaveBeenCalledTimes(2);
+    expect(timeUtils.wait).toHaveBeenCalledTimes(1);
+  });
+
+  it('when network error occurs then retries and succeeds', async () => {
+    const mockFn = vi.fn().mockRejectedValueOnce(new ConnectionLostError()).mockResolvedValueOnce('success');
+
+    const result = await retryWithBackoff(mockFn);
+
+    expect(result).toBe('success');
+    expect(mockFn).toHaveBeenCalledTimes(2);
+    expect(timeUtils.wait).toHaveBeenCalledTimes(1);
+  });
+
+  it('when server error occurs then notifies with correct reason', async () => {
+    const mockFn = vi
+      .fn()
+      .mockRejectedValueOnce({ status: 503, message: 'server unavailable' })
+      .mockResolvedValueOnce('success');
+    const onRetry = vi.fn();
+
+    const result = await retryWithBackoff(mockFn, { onRetry });
+
+    expect(result).toBe('success');
+    expect(onRetry).toHaveBeenCalledTimes(1);
+    expect(onRetry).toHaveBeenCalledWith(1, expect.any(Number), RetryReason.ServerError);
+  });
+
+  it('when server error persists then throws after max retries', async () => {
+    const mockFn = vi.fn().mockRejectedValue({ status: 500, message: 'Server Error' });
+
+    await expect(retryWithBackoff(mockFn, { maxRetries: 2 })).rejects.toEqual({ status: 500, message: 'Server Error' });
+
+    expect(mockFn).toHaveBeenCalledTimes(3);
+    expect(timeUtils.wait).toHaveBeenCalledTimes(2);
   });
 });

src/app/network/retry-with-backoff.ts

@@ -0,0 +1,142 @@
+import { wait } from 'utils/timeUtils';
+import { HTTP_CODES, ErrorMessages } from 'app/core/constants';
+import errorService from 'services/error.service';
+import { ConnectionLostError } from './requests';
+import { randomBytes } from 'crypto';
+
+export enum RetryReason {
+  RateLimit = 'RATE-LIMIT',
+  ServerError = 'SERVER-ERROR',
+}
+
+export interface RetryOptions {
+  maxRetries?: number;
+  onRetry?: (attempt: number, delayMs: number, reason: RetryReason) => void;
+}
+
+interface ErrorWithStatus {
+  status?: number;
+  headers?: Record<string, string>;
+}
+
+const hasStatusProperty = (error: unknown): error is ErrorWithStatus => {
+  return typeof error === 'object' && error !== null;
+};
+
+const isRateLimitError = (error: unknown): boolean => {
+  return hasStatusProperty(error) && error.status === HTTP_CODES.TOO_MANY_REQUESTS;
+};
+
+const NETWORK_ERROR_MESSAGES = new Set([
+  ErrorMessages.ConnectionLost.toLowerCase(),
+  ErrorMessages.NetworkError.toLowerCase(),
+]);
+
+const SERVER_ERROR_MESSAGES = new Set([
+  ErrorMessages.ServerUnavailable.toLowerCase(),
+  ErrorMessages.ServerError.toLowerCase(),
+  ErrorMessages.InternalServerError.toLowerCase(),
+]);
+
+const isNetworkError = (error: unknown, message: string): boolean => {
+  if (error instanceof ConnectionLostError) return true;
+  return NETWORK_ERROR_MESSAGES.has(message);
+};
+
+const is5xxServerError = (message: string): boolean => {
+  return SERVER_ERROR_MESSAGES.has(message);
+};
+
+const isCORSError = (message: string): boolean => {
+  return message.includes(ErrorMessages.CORS);
+};
+
+const isRetryableServerError = (error: unknown): boolean => {
+  const castedError = errorService.castError(error);
+  const lowerCaseMessage = castedError.message.toLowerCase();
+
+  return isNetworkError(error, lowerCaseMessage) || is5xxServerError(lowerCaseMessage) || isCORSError(lowerCaseMessage);
+};
+
+const INITIAL_BACKOFF_MS = 1000;
+const MAX_BACKOFF_MS = 10000;
+const MAX_UINT32 = 0xffffffff;
+
+const calculateExponentialBackoff = (attemptNumber: number): number => {
+  const exponentialDelay = INITIAL_BACKOFF_MS * Math.pow(2, attemptNumber - 1);
+  const cappedDelay = Math.min(exponentialDelay, MAX_BACKOFF_MS);
+  const randomValue = randomBytes(4).readUInt32BE(0) / MAX_UINT32;
+  return randomValue * cappedDelay;
+};
+
+const extractRateLimitDelay = (error: ErrorWithStatus): number | null => {
+  const resetHeader = error.headers?.['x-internxt-ratelimit-reset'];
+  if (!resetHeader) {
+    return null;
+  }
+
+  const delayMs = Number.parseInt(resetHeader, 10);
+  return Number.isNaN(delayMs) ? null : delayMs;
+};
+
+/**
+ * Retries a function when it encounters retryable errors.
+ *
+ * Handles two types of retryable errors:
+ * - Rate limit errors (429): Waits for the time specified in server's rate limit header
+ * - Server/network errors (500s, CORS, network failures): Uses exponential backoff with jitter
+ *
+ * @param fn - The async function to retry on failure
+ * @param options - Retry configuration
+ * @param options.maxRetries - Maximum number of retry attempts (default: 3)
+ * @param options.onRetry - Callback invoked before each retry with attempt number and delay
+ *
+ * @returns The result of the function if successful
+ * @throws The original error if it's not retryable or max retries exceeded
+ *
+ * @example
+ * ```typescript
+ * const result = await retryWithBackoff(
+ *   () => apiCall(),
+ *   { maxRetries: 5, onRetry: (attempt, delay) => console.log(`Retry ${attempt}`) }
+ * );
+ * ```
+ */
+export const retryWithBackoff = async <T>(fn: () => Promise<T>, options: RetryOptions = {}): Promise<T> => {
+  const opts = {
+    maxRetries: 5,
+    onRetry: () => {},
+    ...options,
+  };
+
+  for (let attempt = 0; attempt < opts.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error: unknown) {
+      const currentAttempt = attempt + 1;
+
+      if (isRateLimitError(error)) {
+        const delayMs = extractRateLimitDelay(error as ErrorWithStatus);
+
+        if (!delayMs) {
+          throw error;
+        }
+
+        opts.onRetry(currentAttempt, delayMs, RetryReason.RateLimit);
+        await wait(delayMs);
+        continue;
+      }
+
+      if (isRetryableServerError(error)) {
+        const delayMs = calculateExponentialBackoff(currentAttempt);
+        opts.onRetry(currentAttempt, delayMs, RetryReason.ServerError);
+        await wait(delayMs);
+        continue;
+      }
+
+      throw error;
+    }
+  }
+
+  return await fn();
+};