Retry Strategies in Distributed Systems

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158

// HTTP Status Code Retry Classification Framework
 
interface RetryDecision {
  shouldRetry: boolean;
  reason: string;
  maxRetries?: number;
  useBackoff?: boolean;
}
 
function classifyHttpStatusForRetry(status: number): RetryDecision {
  // ============================================
  // DEFINITE NO-RETRY: Client errors (4xx)
  // ============================================
  
  // 400 Bad Request - Client sent malformed request
  // Retrying will always produce the same result
  if (status === 400) {
    return {
      shouldRetry: false,
      reason: "Malformed request - fix client code",
    };
  }
  
  // 401 Unauthorized - Authentication required/failed
  // Retrying without new credentials is pointless
  if (status === 401) {
    return {
      shouldRetry: false,
      reason: "Authentication required - obtain valid credentials",
    };
  }
  
  // 403 Forbidden - Authorized but not permitted
  // No amount of retrying grants permission
  if (status === 403) {
    return {
      shouldRetry: false,
      reason: "Permission denied - requires authorization change",
    };
  }
  
  // 404 Not Found - Resource doesn't exist
  // Unless expecting eventual consistency, don't retry
  if (status === 404) {
    return {
      shouldRetry: false,
      reason: "Resource not found - verify resource exists",
    };
  }
  
  // 409 Conflict - Request conflicts with current state
  // Often requires reading current state before retry
  if (status === 409) {
    return {
      shouldRetry: false,  // Usually - but read-modify-write patterns may retry
      reason: "Conflict - resolve state conflict first",
    };
  }
  
  // 422 Unprocessable Entity - Semantic error
  // Request is syntactically valid but semantically wrong
  if (status === 422) {
    return {
      shouldRetry: false,
      reason: "Validation failed - fix request payload",
    };
  }
  
  // ============================================
  // SPECIAL CASE: Rate limiting (429)
  // ============================================
  
  // 429 Too Many Requests - Rate limited
  // SHOULD retry, but with significant delay
  if (status === 429) {
    return {
      shouldRetry: true,
      reason: "Rate limited - respect Retry-After header",
      maxRetries: 3,
      useBackoff: true,  // Or use Retry-After if provided
    };
  }
  
  // ============================================
  // DEFINITE RETRY: Server errors indicating transient issues
  // ============================================
  
  // 502 Bad Gateway - Upstream server error
  // Usually transient - gateway couldn't reach backend
  if (status === 502) {
    return {
      shouldRetry: true,
      reason: "Upstream unavailable - likely transient",
      maxRetries: 3,
      useBackoff: true,
    };
  }
  
  // 503 Service Unavailable - Server temporarily overloaded
  // Explicitly designed for temporary conditions
  if (status === 503) {
    return {
      shouldRetry: true,
      reason: "Service unavailable - temporary condition",
      maxRetries: 3,
      useBackoff: true,
    };
  }
  
  // 504 Gateway Timeout - Upstream timed out
  // Network timing issue - retry with backoff
  if (status === 504) {
    return {
      shouldRetry: true,
      reason: "Gateway timeout - transient network issue",
      maxRetries: 3,
      useBackoff: true,
    };
  }
  
  // ============================================
  // AMBIGUOUS: 500 Internal Server Error
  // ============================================
  
  // 500 is the tricky one - could be anything
  if (status === 500) {
    return {
      shouldRetry: true,  // Default to retry with limits
      reason: "Server error - may be transient or permanent",
      maxRetries: 2,  // Lower limit due to ambiguity
      useBackoff: true,
    };
  }
  
  // Default for unhandled 4xx - don't retry
  if (status >= 400 && status < 500) {
    return {
      shouldRetry: false,
      reason: "Client error - fix request",
    };
  }
  
  // Default for unhandled 5xx - cautious retry
  if (status >= 500) {
    return {
      shouldRetry: true,
      reason: "Server error - cautious retry",
      maxRetries: 2,
      useBackoff: true,
    };
  }
  
  // 2xx and 3xx not really "failures"
  return {
    shouldRetry: false,
    reason: "Not a failure status",
  };
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134

// Network-Level Failure Classification
 
type FailureType = 
  | "transient"      // Definitely retry
  | "permanent"      // Never retry
  | "ambiguous"      // Retry cautiously with limits
  | "dangerous";     // Retry only if idempotent
 
interface FailureClassification {
  type: FailureType;
  retryable: boolean;
  description: string;
  recommendedAction: string;
}
 
function classifyNetworkError(error: Error): FailureClassification {
  const errorCode = (error as NodeJS.ErrnoException).code;
  const errorMessage = error.message.toLowerCase();
  
  // ============================================
  // Connection Establishment Failures
  // ============================================
  
  if (errorCode === "ECONNREFUSED") {
    return {
      type: "ambiguous",
      retryable: true,
      description: "Connection refused - server not accepting connections",
      recommendedAction: "Retry with backoff; may indicate service down",
    };
  }
  
  if (errorCode === "ETIMEDOUT" || errorCode === "ECONNABORTED") {
    return {
      type: "transient",
      retryable: true,
      description: "Connection timeout - network congestion or overload",
      recommendedAction: "Retry with exponential backoff",
    };
  }
  
  if (errorCode === "ECONNRESET") {
    return {
      type: "transient",
      retryable: true,
      description: "Connection reset - peer unexpectedly closed connection",
      recommendedAction: "Retry immediately or with minimal delay",
    };
  }
  
  // ============================================
  // DNS Failures
  // ============================================
  
  if (errorCode === "ENOTFOUND" || errorCode === "EAI_AGAIN") {
    return {
      type: "ambiguous",
      retryable: true,
      description: "DNS resolution failed",
      recommendedAction: "Retry with backoff; verify hostname if persists",
    };
  }
  
  // ============================================
  // Read/Write Failures (DANGEROUS)
  // ============================================
  
  if (errorMessage.includes("socket hang up") ||
      errorMessage.includes("read econnreset")) {
    return {
      type: "dangerous",
      retryable: true, // Only if operation is idempotent!
      description: "Connection closed during data transfer",
      recommendedAction: "Retry ONLY if operation is idempotent",
    };
  }
  
  if (errorCode === "EPIPE" || errorCode === "ENOTCONN") {
    return {
      type: "dangerous",
      retryable: true,
      description: "Connection lost while writing",
      recommendedAction: "Retry ONLY if operation is idempotent",
    };
  }
  
  // ============================================
  // SSL/TLS Failures
  // ============================================
  
  if (errorMessage.includes("certificate") ||
      errorMessage.includes("ssl") ||
      errorCode === "CERT_HAS_EXPIRED") {
    return {
      type: "permanent",
      retryable: false,
      description: "SSL/TLS certificate error",
      recommendedAction: "Fix certificate configuration",
    };
  }
  
  if (errorMessage.includes("handshake")) {
    return {
      type: "ambiguous",
      retryable: true, // Sometimes transient (protocol negotiation)
      description: "TLS handshake failed",
      recommendedAction: "Retry once; investigate if persists",
    };
  }
  
  // ============================================
  // Resource Exhaustion
  // ============================================
  
  if (errorCode === "EMFILE" || errorCode === "ENFILE") {
    return {
      type: "transient",
      retryable: true,
      description: "Too many open files - local resource exhaustion",
      recommendedAction: "Retry with delay; may indicate leak",
    };
  }
  
  // ============================================
  // Unknown/Default
  // ============================================
  
  return {
    type: "ambiguous",
    retryable: true,
    description: "Unknown network error",
    recommendedAction: "Retry with limits and monitoring",
  };
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

// Comprehensive Retry Safety Evaluation
 
type OperationType = 
  | "read"           // Pure read, no side effects
  | "idempotent"     // Safe to retry any number of times
  | "non-idempotent" // Executing twice has different effects
  | "unknown";       // Cannot determine safety
 
interface RetrySafetyResult {
  safeToRetry: boolean;
  reason: string;
  recommendation: string;
  requiresIdempotencyKey: boolean;
}
 
class RetrySafetyEvaluator {
  /**
   * Evaluates whether an operation is safe to retry.
   * This is the CRITICAL decision point before any retry.
   */
  evaluate(
    method: string,
    hasIdempotencyKey: boolean,
    operationType: OperationType,
    errorType: "connection" | "timeout" | "response"
  ): RetrySafetyResult {
    
    // =========================================
    // Case 1: Connection failures (request never sent)
    // =========================================
    if (errorType === "connection") {
      // Server never received request — always safe to retry
      return {
        safeToRetry: true,
        reason: "Connection failed before request transmission",
        recommendation: "Retry with exponential backoff",
        requiresIdempotencyKey: false,
      };
    }
    
    // =========================================
    // Case 2: Response-based failures (got response)
    // =========================================
    if (errorType === "response") {
      // Server responded with error — request was processed
      // Safe to retry if operation is idempotent
      const isMethodIdempotent = ["GET", "HEAD", "OPTIONS", "PUT", "DELETE"]
        .includes(method.toUpperCase());
      
      if (isMethodIdempotent || hasIdempotencyKey) {
        return {
          safeToRetry: true,
          reason: "Operation is idempotent or has idempotency key",
          recommendation: "Retry with backoff based on status code",
          requiresIdempotencyKey: false,
        };
      }
      
      return {
        safeToRetry: false,
        reason: "Non-idempotent operation already reached server",
        recommendation: "Do not retry; return error to caller",
        requiresIdempotencyKey: true,
      };
    }
    
    // =========================================
    // Case 3: Timeout failures (UNKNOWN outcome)
    // =========================================
    if (errorType === "timeout") {
      // This is the dangerous case: we don't know if server
      // received and processed the request or not
      
      // Read operations are always safe
      if (operationType === "read") {
        return {
          safeToRetry: true,
          reason: "Read-only operation",
          recommendation: "Retry safely",
          requiresIdempotencyKey: false,
        };
      }
      
      // Idempotent operations are safe
      if (operationType === "idempotent" || hasIdempotencyKey) {
        return {
          safeToRetry: true,
          reason: "Operation is idempotent",
          recommendation: "Retry with backoff",
          requiresIdempotencyKey: false,
        };
      }
      
      // Non-idempotent without key: DANGER
      return {
        safeToRetry: false,
        reason: "Timeout on non-idempotent operation — duplicate risk",
        recommendation: "Log for investigation; alert user of unknown state",
        requiresIdempotencyKey: true,
      };
    }
    
    // Unknown error type — be conservative
    return {
      safeToRetry: false,
      reason: "Cannot determine safety",
      recommendation: "Manual investigation required",
      requiresIdempotencyKey: true,
    };
  }
}
 
// Usage Example
const evaluator = new RetrySafetyEvaluator();
 
// Safe to retry: GET request that timed out
const getRetry = evaluator.evaluate("GET", false, "read", "timeout");
console.log(getRetry);
// { safeToRetry: true, reason: "Read-only operation", ... }
 
// UNSAFE to retry: POST payment that timed out
const paymentRetry = evaluator.evaluate("POST", false, "non-idempotent", "timeout");
console.log(paymentRetry);
// { safeToRetry: false, reason: "Timeout on non-idempotent operation — duplicate risk", ... }
 
// Safe with idempotency key: POST payment with key
const paymentWithKey = evaluator.evaluate("POST", true, "non-idempotent", "timeout");
console.log(paymentWithKey);
// { safeToRetry: true, reason: "Operation is idempotent", ... }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

                    ┌─────────────────────┐
                    │   REQUEST FAILED   │
                    └─────────┬───────────┘
                              │
                              ▼
              ┌───────────────────────────────┐
              │  Is failure clearly PERMANENT? │
              │  (404, 401, 400, 422, etc.)   │
              └───────────────┬───────────────┘
                              │
           ┌──────────────────┼──────────────────┐
           │ YES              │ NO / UNCLEAR     │
           ▼                  ▼                  │
   ┌───────────────┐  ┌─────────────────────┐   │
   │ DON'T RETRY   │  │ Is operation SAFE   │   │
   │ Return error  │  │ to retry?           │   │
   └───────────────┘  │ (idempotent/has key)│   │
                      └─────────┬───────────┘   │
                                │               │
              ┌─────────────────┼───────────────┤
              │ YES             │ NO            │
              ▼                 ▼               │
   ┌───────────────────┐  ┌─────────────────┐  │
   │ Was this a        │  │ Was it CONN     │  │
   │ TIMEOUT failure?  │  │ failure (never  │  │
   │                   │  │ sent)?          │  │
   └─────────┬─────────┘  └───────┬─────────┘  │
             │                    │             │
    ┌────────┼────────┐    ┌──────┼──────┐     │
    │ YES    │ NO     │    │ YES  │ NO   │     │
    ▼        ▼        │    ▼      ▼      │     │
┌────────┐ ┌────────┐ │ ┌────────┐ ┌────────┐ │
│ RETRY  │ │ RETRY  │ │ │ RETRY  │ │ DON'T  │ │
│with    │ │with    │ │ │(safe)  │ │RETRY   │ │
│backoff │ │backoff │ │ └────────┘ │(risky) │ │
└────────┘ └────────┘ │            └────────┘ │
                      │                       │
                      ▼                       ▼
              ┌───────────────────────────────┐
              │    BEFORE RETRYING:           │
              │    ✓ Check retry budget       │
              │    ✓ Apply backoff delay      │
              │    ✓ Log retry attempt        │
              │    ✓ Increment retry counter  │
              └───────────────────────────────┘

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268

/**
 * Production-grade Retry Decision Engine
 * 
 * This engine encapsulates the complete decision logic for whether
 * to retry a failed request, unifying failure classification,
 * safety evaluation, and policy enforcement.
 */
 
interface RetryableError {
  type: "http" | "network" | "timeout";
  statusCode?: number;
  errorCode?: string;
  message: string;
}
 
interface RequestContext {
  method: string;
  path: string;
  hasIdempotencyKey: boolean;
  isExplicitlyIdempotent: boolean;  // Marked by developer
  retryCount: number;
  maxRetries: number;
  retryAfterHeader?: number;  // Seconds, from response
}
 
interface RetryDecision {
  shouldRetry: boolean;
  delayMs: number;
  reason: string;
  final: boolean;  // If true, this is definitive; don't ask again
}
 
class RetryDecisionEngine {
  private maxRetriesHard = 5;  // Never exceed this regardless of config
  private minDelayMs = 100;
  private maxDelayMs = 30000;
  
  /**
   * Main entry point: Should we retry this failed request?
   */
  decide(error: RetryableError, context: RequestContext): RetryDecision {
    // =========================================
    // Gate 1: Hard limits
    // =========================================
    if (context.retryCount >= Math.min(context.maxRetries, this.maxRetriesHard)) {
      return {
        shouldRetry: false,
        delayMs: 0,
        reason: `Retry limit reached (${context.retryCount}/${context.maxRetries})`,
        final: true,
      };
    }
    
    // =========================================
    // Gate 2: Failure classification
    // =========================================
    const failureClass = this.classifyFailure(error);
    
    if (failureClass.type === "permanent") {
      return {
        shouldRetry: false,
        delayMs: 0,
        reason: failureClass.reason,
        final: true,
      };
    }
    
    // =========================================
    // Gate 3: Safety check
    // =========================================
    const safetyResult = this.checkSafety(error, context);
    
    if (!safetyResult.safe) {
      return {
        shouldRetry: false,
        delayMs: 0,
        reason: safetyResult.reason,
        final: true,
      };
    }
    
    // =========================================
    // Approved for retry: Calculate delay
    // =========================================
    let delayMs = this.calculateDelay(error, context);
    
    // Respect Retry-After if provided
    if (context.retryAfterHeader) {
      delayMs = Math.max(delayMs, context.retryAfterHeader * 1000);
    }
    
    // Clamp to limits
    delayMs = Math.max(this.minDelayMs, Math.min(this.maxDelayMs, delayMs));
    
    return {
      shouldRetry: true,
      delayMs,
      reason: `Transient failure, retry ${context.retryCount + 1}/${context.maxRetries}`,
      final: false,
    };
  }
  
  private classifyFailure(error: RetryableError): { type: "transient" | "permanent" | "ambiguous"; reason: string } {
    // HTTP errors
    if (error.type === "http" && error.statusCode) {
      // 4xx client errors are usually permanent
      if (error.statusCode >= 400 && error.statusCode < 500) {
        // Exception: 429 is transient (rate limiting)
        if (error.statusCode === 429) {
          return { type: "transient", reason: "Rate limited" };
        }
        // Exception: 408 is transient (request timeout)
        if (error.statusCode === 408) {
          return { type: "transient", reason: "Request timeout (server side)" };
        }
        return { type: "permanent", reason: `Client error: ${error.statusCode}` };
      }
      
      // 5xx server errors are usually transient
      if (error.statusCode >= 500) {
        // 501 Not Implemented is permanent
        if (error.statusCode === 501) {
          return { type: "permanent", reason: "Not implemented" };
        }
        // 505 HTTP Version Not Supported is permanent
        if (error.statusCode === 505) {
          return { type: "permanent", reason: "HTTP version not supported" };
        }
        return { type: "transient", reason: `Server error: ${error.statusCode}` };
      }
    }
    
    // Network errors
    if (error.type === "network") {
      // Most network errors are transient
      if (error.errorCode === "CERT_HAS_EXPIRED" ||
          error.message.includes("certificate")) {
        return { type: "permanent", reason: "Certificate error" };
      }
      return { type: "transient", reason: "Network error" };
    }
    
    // Timeouts are transient by nature
    if (error.type === "timeout") {
      return { type: "transient", reason: "Request timed out" };
    }
    
    return { type: "ambiguous", reason: "Unknown failure type" };
  }
  
  private checkSafety(
    error: RetryableError,
    context: RequestContext
  ): { safe: boolean; reason: string } {
    const method = context.method.toUpperCase();
    
    // GET, HEAD, OPTIONS are always safe (read-only)
    if (["GET", "HEAD", "OPTIONS"].includes(method)) {
      return { safe: true, reason: "Read-only method" };
    }
    
    // PUT and DELETE are idempotent by HTTP spec
    if (["PUT", "DELETE"].includes(method)) {
      return { safe: true, reason: "Idempotent method by spec" };
    }
    
    // POST and PATCH require explicit idempotency guarantee
    if (["POST", "PATCH"].includes(method)) {
      // If developer marked as idempotent
      if (context.isExplicitlyIdempotent) {
        return { safe: true, reason: "Explicitly marked idempotent" };
      }
      
      // If idempotency key is present
      if (context.hasIdempotencyKey) {
        return { safe: true, reason: "Has idempotency key" };
      }
      
      // For network/connection errors (request never sent), safe to retry
      if (error.type === "network" && 
          (error.errorCode === "ECONNREFUSED" ||
           error.errorCode === "ETIMEDOUT")) {
        return { safe: true, reason: "Request never transmitted" };
      }
      
      // Timeout on POST without idempotency key: UNSAFE
      if (error.type === "timeout") {
        return {
          safe: false,
          reason: "Timeout on non-idempotent POST/PATCH without idempotency key - risk of duplicate execution",
        };
      }
      
      // HTTP error means server received request; need idempotency
      if (error.type === "http") {
        return {
          safe: false,
          reason: "Server error on non-idempotent operation - requires idempotency key for safe retry",
        };
      }
    }
    
    return { safe: false, reason: "Unknown method safety" };
  }
  
  private calculateDelay(error: RetryableError, context: RequestContext): number {
    // Base: exponential backoff
    const base = 100;  // Start at 100ms
    const exponentialDelay = base * Math.pow(2, context.retryCount);
    
    // Add jitter (±25%) to prevent thundering herd
    const jitter = exponentialDelay * 0.25 * (Math.random() * 2 - 1);
    
    return Math.round(exponentialDelay + jitter);
  }
}
 
// =========================================
// Usage Example
// =========================================
 
const engine = new RetryDecisionEngine();
 
// Example 1: GET request that timed out
const decision1 = engine.decide(
  { type: "timeout", message: "Request timed out after 5000ms" },
  {
    method: "GET",
    path: "/api/users/123",
    hasIdempotencyKey: false,
    isExplicitlyIdempotent: false,
    retryCount: 0,
    maxRetries: 3,
  }
);
console.log("GET timeout:", decision1);
// { shouldRetry: true, delayMs: ~100, reason: "Transient failure, retry 1/3" }
 
// Example 2: POST payment without idempotency key that timed out
const decision2 = engine.decide(
  { type: "timeout", message: "Request timed out" },
  {
    method: "POST",
    path: "/api/payments",
    hasIdempotencyKey: false,
    isExplicitlyIdempotent: false,
    retryCount: 0,
    maxRetries: 3,
  }
);
console.log("POST payment timeout (no key):", decision2);
// { shouldRetry: false, reason: "Timeout on non-idempotent POST/PATCH..." }
 
// Example 3: POST payment WITH idempotency key that got 503
const decision3 = engine.decide(
  { type: "http", statusCode: 503, message: "Service Unavailable" },
  {
    method: "POST",
    path: "/api/payments",
    hasIdempotencyKey: true,
    isExplicitlyIdempotent: false,
    retryCount: 1,
    maxRetries: 3,
    retryAfterHeader: 5,
  }
);
console.log("POST payment 503 (with key):", decision3);
// { shouldRetry: true, delayMs: 5000, reason: "Transient failure, retry 2/3" }