The Network Is Reliable (It's Not)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

/**
 * Example: Resilient HTTP client with timeout, retry, and circuit breaker
 * This illustrates the patterns needed for network unreliability
 */
 
interface RetryConfig {
    maxRetries: number;
    baseDelayMs: number;
    maxDelayMs: number;
}
 
interface CircuitBreakerConfig {
    failureThreshold: number;    // Opens after this many failures
    recoveryTimeMs: number;      // Try again after this interval
}
 
class ResilientHttpClient {
    private failureCount = 0;
    private lastFailureTime = 0;
    private circuitOpen = false;
 
    constructor(
        private retryConfig: RetryConfig = { 
            maxRetries: 3, 
            baseDelayMs: 100, 
            maxDelayMs: 10000 
        },
        private circuitConfig: CircuitBreakerConfig = { 
            failureThreshold: 5, 
            recoveryTimeMs: 30000 
        },
        private timeoutMs: number = 5000
    ) {}
 
    async request<T>(url: string, options: RequestInit = {}): Promise<T> {
        // Check circuit breaker state
        if (this.isCircuitOpen()) {
            throw new Error('Circuit breaker open - failing fast');
        }
 
        let lastError: Error | null = null;
 
        for (let attempt = 0; attempt <= this.retryConfig.maxRetries; attempt++) {
            try {
                const response = await this.executeWithTimeout(url, options);
                
                // Success - reset failure count
                this.onSuccess();
                return response as T;
                
            } catch (error) {
                lastError = error as Error;
                this.onFailure();
 
                // Don't retry if circuit opened
                if (this.isCircuitOpen()) {
                    throw new Error('Circuit breaker opened during retries');
                }
 
                // Don't retry on client errors (4xx) - these won't resolve
                if (error instanceof HttpError && error.status >= 400 && error.status < 500) {
                    throw error;
                }
 
                // Wait before retry with exponential backoff + jitter
                if (attempt < this.retryConfig.maxRetries) {
                    const delay = this.calculateBackoff(attempt);
                    await this.sleep(delay);
                }
            }
        }
 
        throw lastError || new Error('Request failed after retries');
    }
 
    private async executeWithTimeout(url: string, options: RequestInit): Promise<unknown> {
        const controller = new AbortController();
        const timeoutId = setTimeout(() => controller.abort(), this.timeoutMs);
 
        try {
            const response = await fetch(url, { ...options, signal: controller.signal });
            
            if (!response.ok) {
                throw new HttpError(response.status, await response.text());
            }
            
            return await response.json();
        } finally {
            clearTimeout(timeoutId);
        }
    }
 
    private calculateBackoff(attempt: number): number {
        // Exponential backoff: baseDelay * 2^attempt
        const exponentialDelay = this.retryConfig.baseDelayMs * Math.pow(2, attempt);
        
        // Cap at maximum delay
        const cappedDelay = Math.min(exponentialDelay, this.retryConfig.maxDelayMs);
        
        // Add jitter (±25%) to prevent thundering herd
        const jitter = cappedDelay * 0.25 * (Math.random() * 2 - 1);
        
        return Math.floor(cappedDelay + jitter);
    }
 
    private isCircuitOpen(): boolean {
        if (!this.circuitOpen) return false;
 
        // Check if recovery period has passed
        const timeSinceFailure = Date.now() - this.lastFailureTime;
        if (timeSinceFailure >= this.circuitConfig.recoveryTimeMs) {
            // Half-open state: allow one request to test recovery
            return false;
        }
 
        return true;
    }
 
    private onSuccess(): void {
        this.failureCount = 0;
        this.circuitOpen = false;
    }
 
    private onFailure(): void {
        this.failureCount++;
        this.lastFailureTime = Date.now();
 
        if (this.failureCount >= this.circuitConfig.failureThreshold) {
            this.circuitOpen = true;
            console.warn(`Circuit breaker opened after ${this.failureCount} failures`);
        }
    }
 
    private sleep(ms: number): Promise<void> {
        return new Promise(resolve => setTimeout(resolve, ms));
    }
}
 
class HttpError extends Error {
    constructor(public status: number, message: string) {
        super(`HTTP ${status}: ${message}`);
        this.name = 'HttpError';
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

/**
 * Server-side idempotency implementation
 * Tracks idempotency keys to prevent duplicate processing
 */
 
interface IdempotencyRecord {
    key: string;
    response: unknown;
    createdAt: Date;
    expiresAt: Date;
}
 
class IdempotencyService {
    // In production: use Redis or database with TTL
    private records = new Map<string, IdempotencyRecord>();
    private readonly TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
 
    /**
     * Execute an operation idempotently
     * Returns cached result for duplicate keys
     */
    async executeOnce<T>(
        idempotencyKey: string,
        operation: () => Promise<T>
    ): Promise<T> {
        // Check for existing record
        const existing = this.records.get(idempotencyKey);
        
        if (existing) {
            if (existing.expiresAt > new Date()) {
                console.log(`Returning cached result for key: ${idempotencyKey}`);
                return existing.response as T;
            } else {
                // Expired - remove and allow re-execution
                // Be careful: in production, consider if re-execution is safe
                this.records.delete(idempotencyKey);
            }
        }
 
        // Execute the operation
        // NOTE: In production, use distributed locking to prevent
        // concurrent executions with the same idempotency key
        const result = await operation();
 
        // Cache the result
        const now = new Date();
        this.records.set(idempotencyKey, {
            key: idempotencyKey,
            response: result,
            createdAt: now,
            expiresAt: new Date(now.getTime() + this.TTL_MS),
        });
 
        return result;
    }
}
 
// Example API endpoint using idempotency
class PaymentController {
    constructor(private idempotencyService: IdempotencyService) {}
    
    async chargeCustomer(request: {
        idempotencyKey: string;
        customerId: string;
        amount: number;
    }): Promise<PaymentResult> {
        // Same idempotency key = same result, even if called multiple times
        return this.idempotencyService.executeOnce(
            request.idempotencyKey,
            async () => {
                // Actual payment processing happens here
                // This will only execute once per unique idempotency key
                const charge = await this.paymentGateway.charge(
                    request.customerId,
                    request.amount
                );
                
                return {
                    success: true,
                    chargeId: charge.id,
                    amount: charge.amount,
                };
            }
        );
    }
    
    private paymentGateway = {
        charge: async (customerId: string, amount: number) => ({
            id: `ch_${Date.now()}`,
            customerId,
            amount,
        })
    };
}
 
interface PaymentResult {
    success: boolean;
    chargeId: string;
    amount: number;
}