Retry with Backoff: Building Resilient Distributed Systems

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162

// Server-side idempotency key implementation
import { v4 as uuidv4 } from 'uuid';
 
interface IdempotencyRecord {
    key: string;
    status: 'processing' | 'completed' | 'failed';
    response?: {
        statusCode: number;
        body: unknown;
        headers: Record<string, string>;
    };
    createdAt: Date;
    expiresAt: Date;
}
 
class IdempotencyStore {
    private store = new Map<string, IdempotencyRecord>();
    private readonly defaultTTLMs = 24 * 60 * 60 * 1000; // 24 hours
    
    /**
     * Check if we have a result for this idempotency key.
     * Sets to 'processing' if not found (atomic operation in production).
     */
    async getOrSetProcessing(key: string): Promise<IdempotencyRecord | null> {
        const existing = this.store.get(key);
        
        if (existing) {
            // Key found - return existing record
            return existing;
        }
        
        // Key not found - create new record in 'processing' state
        // In production: atomic compare-and-set operation
        const newRecord: IdempotencyRecord = {
            key,
            status: 'processing',
            createdAt: new Date(),
            expiresAt: new Date(Date.now() + this.defaultTTLMs),
        };
        
        this.store.set(key, newRecord);
        return null; // null indicates this is the first request
    }
    
    /**
     * Store completed result for replay
     */
    async setCompleted(
        key: string,
        response: IdempotencyRecord['response']
    ): Promise<void> {
        const record = this.store.get(key);
        if (record) {
            record.status = 'completed';
            record.response = response;
        }
    }
    
    /**
     * Mark as failed if operation fails
     */
    async setFailed(key: string): Promise<void> {
        const record = this.store.get(key);
        if (record) {
            record.status = 'failed';
        }
    }
    
    /**
     * Remove processing lock (for cleanup on failure)
     */
    async remove(key: string): Promise<void> {
        this.store.delete(key);
    }
}
 
// Express-style middleware for idempotency
async function idempotencyMiddleware(
    req: Request,
    operation: () => Promise<Response>
): Promise<Response> {
    const idempotencyKey = req.headers.get('Idempotency-Key');
    
    // If no key provided, execute without idempotency protection
    // (Consider requiring keys for all POST/PATCH/DELETE)
    if (!idempotencyKey) {
        console.warn('No idempotency key provided for state-changing request');
        return await operation();
    }
    
    // Validate key format (UUID)
    if (!isValidUUID(idempotencyKey)) {
        return new Response(JSON.stringify({
            error: 'invalid_idempotency_key',
            message: 'Idempotency-Key must be a valid UUID',
        }), { status: 400 });
    }
    
    const store = getIdempotencyStore();
    const existing = await store.getOrSetProcessing(idempotencyKey);
    
    if (existing) {
        // We've seen this key before
        switch (existing.status) {
            case 'completed':
                // Return cached response (idempotent replay)
                console.log(`Idempotency replay: ${idempotencyKey}`);
                return new Response(
                    JSON.stringify(existing.response?.body),
                    {
                        status: existing.response?.statusCode || 200,
                        headers: new Headers({
                            'Content-Type': 'application/json',
                            'Idempotent-Replayed': 'true',
                            ...existing.response?.headers,
                        }),
                    }
                );
            
            case 'processing':
                // Another request is currently processing
                // Return 409 Conflict or 503 with Retry-After
                return new Response(JSON.stringify({
                    error: 'request_in_progress',
                    message: 'Another request with this idempotency key is in progress',
                }), { 
                    status: 409,
                    headers: { 'Retry-After': '1' },
                });
            
            case 'failed':
                // Previous attempt failed - allow retry
                await store.remove(idempotencyKey);
                // Fall through to execute
        }
    }
    
    // Execute the operation
    try {
        const response = await operation();
        
        // Store successful response for replay
        const body = await response.clone().json();
        await store.setCompleted(idempotencyKey, {
            statusCode: response.status,
            body,
            headers: Object.fromEntries(response.headers.entries()),
        });
        
        return response;
    } catch (error) {
        await store.setFailed(idempotencyKey);
        throw error;
    }
}
 
function isValidUUID(str: string): boolean {
    const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
    return uuidRegex.test(str);
}
 
declare function getIdempotencyStore(): IdempotencyStore;

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162

// Client-side idempotency key management with retry logic
import { v4 as uuidv4 } from 'uuid';
 
interface RequestWithIdempotency<T> {
    url: string;
    method: 'POST' | 'PUT' | 'PATCH' | 'DELETE';
    body?: T;
    headers?: Record<string, string>;
}
 
interface IdempotentRequestContext<TReq, TRes> {
    request: RequestWithIdempotency<TReq>;
    idempotencyKey: string;
    execute(): Promise<TRes>;
}
 
/**
 * Create an idempotent request context that can be retried safely.
 * The idempotency key is generated once and reused for all retry attempts.
 */
function createIdempotentRequest<TReq, TRes>(
    request: RequestWithIdempotency<TReq>
): IdempotentRequestContext<TReq, TRes> {
    // Generate key once at request creation time
    const idempotencyKey = uuidv4();
    
    return {
        request,
        idempotencyKey,
        
        async execute(): Promise<TRes> {
            const response = await fetch(request.url, {
                method: request.method,
                headers: {
                    'Content-Type': 'application/json',
                    'Idempotency-Key': idempotencyKey,
                    ...request.headers,
                },
                body: request.body ? JSON.stringify(request.body) : undefined,
            });
            
            if (!response.ok) {
                throw new HttpError(response.status, await response.text());
            }
            
            return response.json();
        },
    };
}
 
/**
 * Execute a request with automatic retry and preserved idempotency key
 */
async function executeWithIdempotentRetry<TReq, TRes>(
    request: RequestWithIdempotency<TReq>,
    retryConfig: {
        maxAttempts: number;
        baseDelayMs: number;
        maxDelayMs: number;
        isRetryable: (error: Error) => boolean;
    }
): Promise<TRes> {
    // Create context with idempotency key BEFORE any attempts
    const context = createIdempotentRequest<TReq, TRes>(request);
    
    console.log(`Starting idempotent request: key=${context.idempotencyKey}`);
    
    let lastError: Error | undefined;
    
    for (let attempt = 1; attempt <= retryConfig.maxAttempts; attempt++) {
        try {
            const result = await context.execute();
            
            console.log(
                `Request succeeded on attempt ${attempt}: key=${context.idempotencyKey}`
            );
            
            return result;
        } catch (error) {
            lastError = error as Error;
            
            if (!retryConfig.isRetryable(lastError)) {
                throw lastError;
            }
            
            if (attempt < retryConfig.maxAttempts) {
                const delay = Math.min(
                    retryConfig.baseDelayMs * Math.pow(2, attempt - 1),
                    retryConfig.maxDelayMs
                );
                
                console.log(
                    `Attempt ${attempt} failed, retrying in ${delay}ms: ` +
                                        `key=${context.idempotencyKey}`
                );
                
                await sleep(delay);
            }
        }
    }
    
    throw new Error(
        `Request failed after ${retryConfig.maxAttempts} attempts ` +
                                    `(idempotencyKey: ${context.idempotencyKey}): ${lastError?.message}`
    );
}
 
// Usage example: Creating a payment with safe retries
async function createPayment(amount: number, customerId: string) {
    return executeWithIdempotentRetry(
        {
            url: 'https://api.example.com/v1/payments',
            method: 'POST',
            body: {
                amount,
                currency: 'usd',
                customer: customerId,
            },
        },
        {
            maxAttempts: 5,
            baseDelayMs: 200,
            maxDelayMs: 10000,
            isRetryable: (error) => {
                if (error instanceof HttpError) {
                    // Retry on transient errors only
                    return [408, 429, 500, 502, 503, 504].includes(error.status);
                }
                return error.name === 'NetworkError';
            },
        }
    );
}
 
// IMPORTANT: Each user action should create a new idempotency key
// DO NOT reuse keys across distinct business operations
 
// Correct: User clicks "Pay" - new key for this payment attempt
const payButton = document.getElementById('pay-button');
payButton?.addEventListener('click', async () => {
    // Each click is a new user intent, gets a new key
    try {
        const result = await createPayment(100, 'cust_123');
        showSuccess(result);
    } catch (error) {
        showError(error);
    }
});
 
class HttpError extends Error {
    constructor(public status: number, message: string) {
        super(`HTTP ${status}: ${message}`);
        this.name = 'HttpError';
    }
}
 
function sleep(ms: number): Promise<void> {
    return new Promise(resolve => setTimeout(resolve, ms));
}
 
declare function showSuccess(result: unknown): void;
declare function showError(error: unknown): void;

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166

// Pattern 1: Making counter increments idempotent
interface IncrementRequest {
    idempotencyKey: string;
    incrementBy: number;
}
 
class IdempotentCounter {
    private counter: number = 0;
    private processedIncrements = new Set<string>();
    
    increment(request: IncrementRequest): number {
        // Check if we've already processed this increment
        if (this.processedIncrements.has(request.idempotencyKey)) {
            console.log(`Increment ${request.idempotencyKey} already processed, returning current value`);
            return this.counter;
        }
        
        // Process the increment
        this.counter += request.incrementBy;
        this.processedIncrements.add(request.idempotencyKey);
        
        return this.counter;
    }
}
 
// Pattern 2: Auto-increment IDs → Client-generated IDs
// WRONG: Server generates ID on each request
async function createUserWrong(name: string) {
    // If retried, creates duplicate users with different IDs
    const id = await generateAutoIncrementId();
    await db.users.create({ id, name });
    return { id, name };
}
 
// RIGHT: Client provides ID, server rejects duplicates
async function createUserRight(request: { id: string; name: string }) {
    // If exists, return existing (idempotent)
    const existing = await db.users.findById(request.id);
    if (existing) {
        return existing;
    }
    
    // If not exists, create
    // Use unique constraint to handle race condition
    try {
        const user = await db.users.create(request);
        return user;
    } catch (error) {
        if (isUniqueConstraintError(error)) {
            // Race condition: another request created it
            return await db.users.findById(request.id);
        }
        throw error;
    }
}
 
// Pattern 3: Financial transactions with idempotency
interface PaymentRequest {
    idempotencyKey: string;
    amount: number;
    customerId: string;
}
 
async function processPayment(request: PaymentRequest) {
    // Check for existing transaction with this key
    const existing = await db.transactions.findByIdempotencyKey(request.idempotencyKey);
    
    if (existing) {
        // Return the existing result (idempotent replay)
        return {
            id: existing.id,
            amount: existing.amount,
            status: existing.status,
            replayed: true,
        };
    }
    
    // Create transaction record BEFORE charging
    const transaction = await db.transactions.create({
        idempotencyKey: request.idempotencyKey,
        amount: request.amount,
        customerId: request.customerId,
        status: 'pending',
    });
    
    try {
        // Charge the payment processor
        const charge = await paymentProcessor.charge({
            amount: request.amount,
            customerId: request.customerId,
            metadata: { transactionId: transaction.id },
        });
        
        // Update transaction status
        await db.transactions.update(transaction.id, {
            status: 'completed',
            chargeId: charge.id,
        });
        
        return {
            id: transaction.id,
            amount: request.amount,
            status: 'completed',
            chargeId: charge.id,
        };
    } catch (error) {
        // Mark transaction as failed
        await db.transactions.update(transaction.id, {
            status: 'failed',
            error: String(error),
        });
        throw error;
    }
}
 
// Pattern 4: Send-once operations (emails, notifications)
interface NotificationRequest {
    idempotencyKey: string;
    userId: string;
    message: string;
}
 
async function sendNotification(request: NotificationRequest) {
    // Check if already sent
    const existing = await db.sentNotifications.findByIdempotencyKey(
        request.idempotencyKey
    );
    
    if (existing) {
        return { sent: true, deduplicated: true };
    }
    
    // Record intent to send BEFORE sending
    // (Prevents gap where send succeeds but recording fails)
    await db.sentNotifications.create({
        idempotencyKey: request.idempotencyKey,
        userId: request.userId,
        status: 'sending',
    });
    
    try {
        await notificationService.send(request.userId, request.message);
        
        await db.sentNotifications.update(request.idempotencyKey, {
            status: 'sent',
        });
        
        return { sent: true, deduplicated: false };
    } catch (error) {
        await db.sentNotifications.update(request.idempotencyKey, {
            status: 'failed',
        });
        throw error;
    }
}
 
// Helper declarations
declare function generateAutoIncrementId(): Promise<number>;
declare function isUniqueConstraintError(error: unknown): boolean;
declare const db: {
    users: any;
    transactions: any;
    sentNotifications: any;
};
declare const paymentProcessor: any;
declare const notificationService: any;

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182

// Redis-based idempotency storage with atomic operations
import Redis from 'ioredis';
 
interface IdempotencyResult {
    status: 'new' | 'processing' | 'completed' | 'failed';
    response?: {
        statusCode: number;
        body: string;
    };
}
 
class RedisIdempotencyStore {
    private redis: Redis;
    private defaultTTLSeconds = 86400; // 24 hours
    
    constructor(redisUrl: string) {
        this.redis = new Redis(redisUrl);
    }
    
    /**
     * Atomically acquire processing lock or retrieve existing result.
     * Uses Redis SETNX for atomic check-and-set.
     */
    async getOrLock(key: string): Promise<{
        isLocked: boolean;
        existingResult?: IdempotencyResult;
    }> {
        const existingRaw = await this.redis.get(`idempotency:${key}`);
        
        if (existingRaw) {
            const existing: IdempotencyResult = JSON.parse(existingRaw);
            return { isLocked: false, existingResult: existing };
        }
        
        // Attempt to acquire lock (SETNX = Set if Not eXists)
        const lockKey = `idempotency:${key}`;
        const lockValue = JSON.stringify({ status: 'processing' });
        
        // SET with NX (only if not exists) and EX (expire)
        const result = await this.redis.set(
            lockKey,
            lockValue,
            'EX', this.defaultTTLSeconds,
            'NX'
        );
        
        if (result === 'OK') {
            // We acquired the lock
            return { isLocked: true };
        } else {
            // Someone else acquired between our GET and SET
            // Fetch their result
            const nowExisting = await this.redis.get(lockKey);
            if (nowExisting) {
                return { isLocked: false, existingResult: JSON.parse(nowExisting) };
            }
            // Race condition edge case - retry
            return this.getOrLock(key);
        }
    }
    
    /**
     * Store completed result
     */
    async setCompleted(
        key: string,
        response: { statusCode: number; body: string }
    ): Promise<void> {
        const result: IdempotencyResult = {
            status: 'completed',
            response,
        };
        
        await this.redis.setex(
            `idempotency:${key}`,
            this.defaultTTLSeconds,
            JSON.stringify(result)
        );
    }
    
    /**
     * Mark as failed (allows retry)
     */
    async setFailed(key: string): Promise<void> {
        const result: IdempotencyResult = { status: 'failed' };
        
        await this.redis.setex(
            `idempotency:${key}`,
            this.defaultTTLSeconds,
            JSON.stringify(result)
        );
    }
    
    /**
     * Remove lock (cleanup on failure before response stored)
     */
    async removeLock(key: string): Promise<void> {
        await this.redis.del(`idempotency:${key}`);
    }
}
 
// PostgreSQL-based for stronger durability (financial use cases)
// Using a dedicated table with proper indexing
 
/*
CREATE TABLE idempotency_records (
    idempotency_key VARCHAR(255) PRIMARY KEY,
    status VARCHAR(20) NOT NULL DEFAULT 'processing',
    response_status_code INTEGER,
    response_body JSONB,
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    expires_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + INTERVAL '24 hours'
);
 
CREATE INDEX idx_idempotency_expires ON idempotency_records(expires_at);
 
-- Periodic cleanup job
DELETE FROM idempotency_records WHERE expires_at < NOW();
*/
 
class PostgresIdempotencyStore {
    constructor(private pool: any) {}
    
    async getOrLock(key: string): Promise<{
        isLocked: boolean;
        existingResult?: IdempotencyResult;
    }> {
        // Use INSERT ... ON CONFLICT for atomic check-and-set
        const result = await this.pool.query(`
            INSERT INTO idempotency_records (idempotency_key, status)
            VALUES ($1, 'processing')
            ON CONFLICT (idempotency_key) 
            DO UPDATE SET idempotency_key = EXCLUDED.idempotency_key
            RETURNING 
                idempotency_key,
                status,
                response_status_code,
                response_body,
                (xmax = 0) as is_new
        `, [key]);
        
        const row = result.rows[0];
        
        if (row.is_new) {
            return { isLocked: true };
        }
        
        return {
            isLocked: false,
            existingResult: {
                status: row.status,
                response: row.response_status_code ? {
                    statusCode: row.response_status_code,
                    body: JSON.stringify(row.response_body),
                } : undefined,
            },
        };
    }
    
    async setCompleted(
        key: string,
        response: { statusCode: number; body: string }
    ): Promise<void> {
        await this.pool.query(`
            UPDATE idempotency_records
            SET status = 'completed',
                response_status_code = $2,
                response_body = $3,
                updated_at = NOW()
            WHERE idempotency_key = $1
        `, [key, response.statusCode, response.body]);
    }
    
    async setFailed(key: string): Promise<void> {
        await this.pool.query(`
            UPDATE idempotency_records
            SET status = 'failed', updated_at = NOW()
            WHERE idempotency_key = $1
        `, [key]);
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

// Idempotency key derivation for distributed operations
import { createHash } from 'crypto';
 
/**
 * Derive a child idempotency key from a parent key.
 * Ensures deterministic, unique keys for downstream operations.
 */
function deriveIdempotencyKey(
    parentKey: string,
    operationType: string,
    context?: Record<string, string>
): string {
    const components = [
        parentKey,
        operationType,
        ...Object.entries(context || {}).map(([k, v]) => `${k}=${v}`),
    ];
    
    return createHash('sha256')
        .update(components.join(':'))
        .digest('hex')
        .slice(0, 36); // UUID-length for compatibility
}
 
// Example: Order processing spanning multiple services
interface OrderRequest {
    idempotencyKey: string;
    customerId: string;
    items: Array<{ productId: string; quantity: number }>;
    paymentMethodId: string;
}
 
class OrderService {
    async processOrder(request: OrderRequest) {
        // Parent idempotency key from client
        const parentKey = request.idempotencyKey;
        
        // Step 1: Create order record (with parent key)
        const order = await this.createOrder(request);
        
        // Step 2: Reserve inventory (derived key)
        const inventoryKey = deriveIdempotencyKey(parentKey, 'inventory-reserve', {
            orderId: order.id,
        });
        await this.inventoryService.reserve({
            idempotencyKey: inventoryKey,
            items: request.items,
        });
        
        // Step 3: Process payment (derived key)
        const paymentKey = deriveIdempotencyKey(parentKey, 'payment-charge', {
            orderId: order.id,
        });
        const payment = await this.paymentService.charge({
            idempotencyKey: paymentKey,
            amount: order.total,
            customerId: request.customerId,
            paymentMethodId: request.paymentMethodId,
        });
        
        // Step 4: Send confirmation email (derived key)
        const emailKey = deriveIdempotencyKey(parentKey, 'email-confirmation', {
            orderId: order.id,
        });
        await this.notificationService.sendOrderConfirmation({
            idempotencyKey: emailKey,
            orderId: order.id,
            email: order.customerEmail,
        });
        
        return {
            orderId: order.id,
            paymentId: payment.id,
            status: 'confirmed',
        };
    }
    
    private async createOrder(request: OrderRequest) {
        // Order creation is idempotent via the parent key
        // ...
        return { id: 'order-123', total: 100, customerEmail: 'user@example.com' };
    }
    
    private inventoryService = {
        reserve: async (req: { idempotencyKey: string; items: any[] }) => {},
    };
    private paymentService = {
        charge: async (req: { idempotencyKey: string; amount: number; customerId: string; paymentMethodId: string }) =>
            ({ id: 'payment-123' }),
    };
    private notificationService = {
        sendOrderConfirmation: async (req: { idempotencyKey: string; orderId: string; email: string }) => {},
    };
}
 
// Key derivation ensures:
// 1. Same parent key → same derived keys → idempotent across all services
// 2. Different operations get different keys (operationType discriminator)
// 3. If order 1 and order 2 have same parent key, they derive same downstream keys
// 4. If order 1 retries, all downstream calls use same keys as first attempt
 
// Example trace:
// Parent key: "abc-123"
// → Inventory key: sha256("abc-123:inventory-reserve:orderId=order-123") → "7f3d..."
// → Payment key: sha256("abc-123:payment-charge:orderId=order-123") → "2a5e..."
// → Email key: sha256("abc-123:email-confirmation:orderId=order-123") → "9c1b..."

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230

// Comprehensive idempotency test patterns
import { describe, it, expect, beforeEach } from 'vitest';
 
describe('Payment API Idempotency', () => {
    let api: PaymentAPI;
    
    beforeEach(() => {
        api = new PaymentAPI();
    });
    
    // Test 1: Basic duplicate detection
    it('should return same response for duplicate requests', async () => {
        const idempotencyKey = 'test-key-1';
        const request = { amount: 100, customerId: 'cust-1' };
        
        const response1 = await api.createPayment({ 
            ...request, 
            idempotencyKey 
        });
        
        const response2 = await api.createPayment({ 
            ...request, 
            idempotencyKey 
        });
        
        // Same response
        expect(response2.paymentId).toBe(response1.paymentId);
        expect(response2.amount).toBe(response1.amount);
        
        // Only one actual payment
        expect(api.getPaymentCount()).toBe(1);
    });
    
    // Test 2: Different keys create different resources
    it('should create separate resources for different keys', async () => {
        const request = { amount: 100, customerId: 'cust-1' };
        
        const response1 = await api.createPayment({ 
            ...request, 
            idempotencyKey: 'key-a' 
        });
        
        const response2 = await api.createPayment({ 
            ...request, 
            idempotencyKey: 'key-b' 
        });
        
        // Different payments
        expect(response2.paymentId).not.toBe(response1.paymentId);
        expect(api.getPaymentCount()).toBe(2);
    });
    
    // Test 3: Concurrent duplicate requests
    it('should handle concurrent duplicates safely', async () => {
        const idempotencyKey = 'concurrent-test';
        const request = { amount: 100, customerId: 'cust-1' };
        
        // Fire 10 concurrent requests with same key
        const promises = Array(10).fill(null).map(() =>
            api.createPayment({ ...request, idempotencyKey })
        );
        
        const responses = await Promise.all(promises);
        
        // All should return same payment ID
        const paymentIds = responses.map(r => r.paymentId);
        expect(new Set(paymentIds).size).toBe(1);
        
        // Only one payment created
        expect(api.getPaymentCount()).toBe(1);
    });
    
    // Test 4: Request body mismatch detection
    it('should reject duplicate key with different body', async () => {
        const idempotencyKey = 'mismatch-test';
        
        await api.createPayment({ 
            amount: 100, 
            customerId: 'cust-1', 
            idempotencyKey 
        });
        
        // Same key, different amount
        await expect(
            api.createPayment({ 
                amount: 200, 
                customerId: 'cust-1', 
                idempotencyKey 
            })
        ).rejects.toThrow('Idempotency key already used with different request');
    });
    
    // Test 5: Failed request allows retry
    it('should allow retry after failed request', async () => {
        const idempotencyKey = 'retry-after-failure';
        const request = { amount: 100, customerId: 'cust-1' };
        
        // First request fails
        api.simulateFailure(true);
        await expect(
            api.createPayment({ ...request, idempotencyKey })
        ).rejects.toThrow();
        
        // Retry should be allowed and succeed
        api.simulateFailure(false);
        const response = await api.createPayment({ ...request, idempotencyKey });
        
        expect(response.paymentId).toBeDefined();
        expect(api.getPaymentCount()).toBe(1);
    });
    
    // Test 6: Processing state handles slow operations
    it('should return 409 when request is still processing', async () => {
        const idempotencyKey = 'slow-operation';
        const request = { amount: 100, customerId: 'cust-1' };
        
        // Start a slow operation
        api.setProcessingDelay(5000);
        const slowPromise = api.createPayment({ ...request, idempotencyKey });
        
        // Immediate retry should get 409
        await new Promise(resolve => setTimeout(resolve, 100));
        
        await expect(
            api.createPayment({ ...request, idempotencyKey })
        ).rejects.toThrow('Request in progress');
        
        // Original completes
        const response = await slowPromise;
        expect(response.paymentId).toBeDefined();
    });
    
    // Test 7: Response consistency
    it('should return identical response structure on replay', async () => {
        const idempotencyKey = 'response-test';
        const request = { amount: 100, customerId: 'cust-1' };
        
        const response1 = await api.createPayment({ ...request, idempotencyKey });
        const response2 = await api.createPayment({ ...request, idempotencyKey });
        
        // Deep equality including all fields
        expect(response2).toEqual({
            ...response1,
            // May include a 'replayed' flag
        });
    });
});
 
// Mock API for testing
class PaymentAPI {
    private payments: Map<string, any> = new Map();
    private idempotencyRecords: Map<string, any> = new Map();
    private shouldFail = false;
    private delayMs = 0;
    
    async createPayment(request: {
        amount: number;
        customerId: string;
        idempotencyKey: string;
    }) {
        // Check idempotency
        const existing = this.idempotencyRecords.get(request.idempotencyKey);
        if (existing) {
            if (existing.status === 'processing') {
                throw new Error('Request in progress');
            }
            if (existing.status === 'completed') {
                // Verify request body matches
                if (existing.requestHash !== this.hashRequest(request)) {
                    throw new Error('Idempotency key already used with different request');
                }
                return existing.response;
            }
            // Failed - allow retry
        }
        
        // Record processing state
        this.idempotencyRecords.set(request.idempotencyKey, {
            status: 'processing',
            requestHash: this.hashRequest(request),
        });
        
        // Simulate processing delay
        if (this.delayMs > 0) {
            await new Promise(resolve => setTimeout(resolve, this.delayMs));
        }
        
        // Simulate failure
        if (this.shouldFail) {
            this.idempotencyRecords.set(request.idempotencyKey, { status: 'failed' });
            throw new Error('Payment processing failed');
        }
        
        // Create payment
        const paymentId = `pay_${Date.now()}`;
        this.payments.set(paymentId, request);
        
        const response = {
            paymentId,
            amount: request.amount,
            customerId: request.customerId,
            status: 'succeeded',
        };
        
        // Record success
        this.idempotencyRecords.set(request.idempotencyKey, {
            status: 'completed',
            requestHash: this.hashRequest(request),
            response,
        });
        
        return response;
    }
    
    getPaymentCount(): number {
        return this.payments.size;
    }
    
    simulateFailure(shouldFail: boolean): void {
        this.shouldFail = shouldFail;
    }
    
    setProcessingDelay(ms: number): void {
        this.delayMs = ms;
    }
    
    private hashRequest(request: any): string {
        return JSON.stringify({ amount: request.amount, customerId: request.customerId });
    }
}