Application-Level Handling

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

// ❌ BRITTLE MODEL: Mutable state that can conflict
interface BrittleOrder {
    id: string;
    items: OrderItem[];
    status: string;           // Direct mutation
    totalAmount: number;      // Derived but stored
    lastModified: Date;       // Race conditions
}
 
// Each update mutates the order directly
// Conflicts when multiple services update simultaneously
// No history, hard to debug issues
 
// ✅ RESILIENT MODEL: Event-sourced with clear state machine
interface OrderEvent {
    orderId: string;
    eventId: string;          // Unique, idempotent
    eventType: 'CREATED' | 'ITEM_ADDED' | 'ITEM_REMOVED' | 
               'SUBMITTED' | 'PAYMENT_CONFIRMED' | 'SHIPPED' | 'CANCELLED';
    timestamp: number;
    payload: any;
}
 
// State machine defines valid transitions
const ORDER_STATE_MACHINE = {
    DRAFT: ['SUBMITTED', 'CANCELLED'],
    SUBMITTED: ['PAYMENT_CONFIRMED', 'CANCELLED'],
    PAYMENT_CONFIRMED: ['SHIPPED', 'REFUNDING'],
    SHIPPED: ['DELIVERED', 'RETURN_REQUESTED'],
    CANCELLED: [],  // Terminal state
    DELIVERED: ['RETURN_REQUESTED'],
    // ... etc
};
 
class Order {
    private events: OrderEvent[] = [];
    
    // Derive current state from events
    get state(): OrderState {
        return this.events.reduce(
            (state, event) => this.applyEvent(state, event),
            initialOrderState
        );
    }
    
    // Commands validate against current state
    submit(): OrderEvent[] {
        const current = this.state;
        if (!ORDER_STATE_MACHINE[current.status].includes('SUBMITTED')) {
            throw new Error(`Cannot submit order in state ${current.status}`);
        }
        
        return [{
            orderId: this.id,
            eventId: generateUniqueId(),
            eventType: 'SUBMITTED',
            timestamp: Date.now(),
            payload: { submittedAt: Date.now() }
        }];
    }
    
    // Events are append-only, no conflicts
    addEvent(event: OrderEvent): void {
        // Check idempotency
        if (this.events.find(e => e.eventId === event.eventId)) {
            return;  // Already applied, skip
        }
        this.events.push(event);
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

interface InventoryReservation {
    reservationId: string;
    productId: string;
    quantity: number;
    expiresAt: number;
    status: 'PENDING' | 'CONFIRMED' | 'EXPIRED' | 'RELEASED';
}
 
class InventoryService {
    private readonly RESERVATION_TTL = 10 * 60 * 1000; // 10 minutes
    
    // Step 1: Create a reservation (quick, can be eventually consistent)
    async reserve(productId: string, quantity: number): Promise<ReservationResult> {
        const reservationId = generateUniqueId();
        const expiresAt = Date.now() + this.RESERVATION_TTL;
        
        // Optimistically create reservation
        const reservation: InventoryReservation = {
            reservationId,
            productId,
            quantity,
            expiresAt,
            status: 'PENDING'
        };
        
        await this.db.write(`reservation:${reservationId}`, reservation);
        
        // Trigger async validation
        this.validateReservationAsync(reservationId, productId, quantity);
        
        return { 
            reservationId, 
            expiresAt,
            status: 'PENDING' // Will be confirmed or rejected async
        };
    }
    
    // Step 2: Validate reservation against actual inventory
    private async validateReservationAsync(
        reservationId: string,
        productId: string,
        quantity: number
    ): Promise<void> {
        try {
            // Use stronger consistency for the actual inventory check
            const inventory = await this.db.read(
                `inventory:${productId}`,
                { consistency: 'QUORUM' }
            );
            
            // Calculate available = total - confirmed reservations - pending reservations
            const confirmedReserved = await this.getConfirmedReservations(productId);
            const pendingReserved = await this.getPendingReservations(productId);
            const available = inventory.quantity - confirmedReserved - pendingReserved;
            
            if (available >= quantity) {
                // Confirm the reservation
                await this.confirmReservation(reservationId);
            } else {
                // Not enough inventory, reject
                await this.rejectReservation(reservationId);
            }
        } catch (error) {
            // On error, leave as PENDING - will expire if not confirmed
            console.error(`Reservation validation failed: ${error}`);
        }
    }
    
    // Step 3: Confirm reservation when order is complete
    async confirmOrder(reservationId: string): Promise<ConfirmResult> {
        const reservation = await this.db.read(`reservation:${reservationId}`);
        
        if (reservation.status !== 'PENDING' && reservation.status !== 'CONFIRMED') {
            return { success: false, reason: 'Reservation expired or invalid' };
        }
        
        // Decrement actual inventory with conditional write
        const result = await this.decrementInventory(
            reservation.productId,
            reservation.quantity
        );
        
        if (result.success) {
            await this.updateReservation(reservationId, { status: 'CONFIRMED' });
            return { success: true };
        } else {
            // Inventory changed - reservation no longer valid
            await this.updateReservation(reservationId, { status: 'RELEASED' });
            return { success: false, reason: 'Inventory no longer available' };
        }
    }
    
    // Background job: Expire old reservations
    async cleanupExpiredReservations(): Promise<void> {
        const now = Date.now();
        const expired = await this.db.query(
            'reservations',
            { status: 'PENDING', expiresAt: { $lt: now } }
        );
        
        for (const reservation of expired) {
            await this.updateReservation(reservation.reservationId, { status: 'EXPIRED' });
        }
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

interface TransferSaga {
    sagaId: string;
    fromAccount: string;
    toAccount: string;
    amount: number;
    steps: SagaStep[];
    status: 'RUNNING' | 'COMPLETED' | 'COMPENSATING' | 'COMPENSATED' | 'FAILED';
}
 
interface SagaStep {
    name: string;
    status: 'PENDING' | 'COMPLETED' | 'FAILED' | 'COMPENSATED';
    action: () => Promise<void>;
    compensation: () => Promise<void>;
}
 
class TransferSagaOrchestrator {
    async executeTransfer(
        fromAccount: string, 
        toAccount: string, 
        amount: number
    ): Promise<TransferResult> {
        const sagaId = generateUniqueId();
        
        // Define the saga steps with compensation actions
        const saga: TransferSaga = {
            sagaId,
            fromAccount,
            toAccount,
            amount,
            status: 'RUNNING',
            steps: [
                {
                    name: 'VALIDATE_SOURCE',
                    status: 'PENDING',
                    action: async () => {
                        const balance = await this.getBalance(fromAccount);
                        if (balance < amount) {
                            throw new Error('Insufficient funds');
                        }
                    },
                    compensation: async () => {
                        // Nothing to compensate - validation is read-only
                    }
                },
                {
                    name: 'DEBIT_SOURCE',
                    status: 'PENDING',
                    action: async () => {
                        await this.adjustBalance(fromAccount, -amount, sagaId);
                    },
                    compensation: async () => {
                        // Reverse the debit
                        await this.adjustBalance(fromAccount, amount, `${sagaId}-compensation`);
                    }
                },
                {
                    name: 'CREDIT_DESTINATION',
                    status: 'PENDING',
                    action: async () => {
                        await this.adjustBalance(toAccount, amount, sagaId);
                    },
                    compensation: async () => {
                        // Reverse the credit
                        await this.adjustBalance(toAccount, -amount, `${sagaId}-compensation`);
                    }
                },
                {
                    name: 'RECORD_TRANSFER',
                    status: 'PENDING',
                    action: async () => {
                        await this.recordTransfer({
                            sagaId,
                            fromAccount,
                            toAccount,
                            amount,
                            timestamp: Date.now()
                        });
                    },
                    compensation: async () => {
                        await this.recordTransfer({
                            sagaId: `${sagaId}-reversal`,
                            fromAccount: toAccount,  // Reversed
                            toAccount: fromAccount,
                            amount,
                            reason: 'SAGA_COMPENSATION',
                            timestamp: Date.now()
                        });
                    }
                }
            ]
        };
        
        // Execute saga
        await this.saveSaga(saga);
        
        try {
            // Execute steps in order
            for (let i = 0; i < saga.steps.length; i++) {
                const step = saga.steps[i];
                try {
                    await step.action();
                    step.status = 'COMPLETED';
                    await this.saveSaga(saga);
                } catch (error) {
                    step.status = 'FAILED';
                    saga.status = 'COMPENSATING';
                    await this.saveSaga(saga);
                    
                    // Compensate completed steps in reverse order
                    await this.compensate(saga, i);
                    throw error;
                }
            }
            
            saga.status = 'COMPLETED';
            await this.saveSaga(saga);
            return { success: true, sagaId };
            
        } catch (error) {
            return { success: false, sagaId, error: error.message };
        }
    }
    
    private async compensate(saga: TransferSaga, failedStep: number): Promise<void> {
        // Compensate in reverse order
        for (let i = failedStep - 1; i >= 0; i--) {
            const step = saga.steps[i];
            if (step.status === 'COMPLETED') {
                await step.compensation();
                step.status = 'COMPENSATED';
                await this.saveSaga(saga);
            }
        }
        saga.status = 'COMPENSATED';
        await this.saveSaga(saga);
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

// Simplified CRDT for collaborative text editing
// Based on RGA (Replicated Growable Array)
 
interface CRDTCharacter {
    id: string;           // Globally unique ID
    value: string;        // The character
    deleted: boolean;     // Tombstone for deleted chars
    afterId: string | null; // Inserted after this character
}
 
class CRDTText {
    private characters: Map<string, CRDTCharacter> = new Map();
    private nodeId: string;  // This replica's unique ID
    private counter: number = 0;
    
    constructor(nodeId: string) {
        this.nodeId = nodeId;
    }
    
    // Generate unique, sortable ID
    private generateId(): string {
        return `${Date.now()}-${this.nodeId}-${++this.counter}`;
    }
    
    // Insert character after position
    insert(afterId: string | null, char: string): Operation {
        const newChar: CRDTCharacter = {
            id: this.generateId(),
            value: char,
            deleted: false,
            afterId
        };
        
        this.characters.set(newChar.id, newChar);
        
        // Return operation to broadcast to other replicas
        return { type: 'INSERT', character: newChar };
    }
    
    // Delete character (tombstone, don't actually remove)
    delete(charId: string): Operation {
        const char = this.characters.get(charId);
        if (char) {
            char.deleted = true;
        }
        return { type: 'DELETE', characterId: charId };
    }
    
    // Apply operation from remote replica
    applyRemote(op: Operation): void {
        if (op.type === 'INSERT') {
            // Idempotent: check if already exists
            if (!this.characters.has(op.character.id)) {
                this.characters.set(op.character.id, op.character);
            }
        } else if (op.type === 'DELETE') {
            const char = this.characters.get(op.characterId);
            if (char) {
                char.deleted = true;
            }
        }
    }
    
    // Merge entire state from another replica
    merge(otherState: Map<string, CRDTCharacter>): void {
        for (const [id, char] of otherState) {
            const existing = this.characters.get(id);
            if (!existing) {
                // New character, add it
                this.characters.set(id, char);
            } else if (char.deleted && !existing.deleted) {
                // Character was deleted on other replica
                existing.deleted = true;
            }
            // If both exist and neither is deleted, they're identical
        }
    }
    
    // Render the text by traversing the linked structure
    toString(): string {
        // Build adjacency list
        const afterMap = new Map<string | null, CRDTCharacter[]>();
        for (const char of this.characters.values()) {
            if (!afterMap.has(char.afterId)) {
                afterMap.set(char.afterId, []);
            }
            afterMap.get(char.afterId)!.push(char);
        }
        
        // Sort siblings by ID for deterministic ordering
        for (const siblings of afterMap.values()) {
            siblings.sort((a, b) => a.id.localeCompare(b.id));
        }
        
        // DFS to build string
        const result: string[] = [];
        const dfs = (afterId: string | null) => {
            const children = afterMap.get(afterId) || [];
            for (const child of children) {
                if (!child.deleted) {
                    result.push(child.value);
                }
                dfs(child.id);
            }
        };
        dfs(null);
        
        return result.join('');
    }
}
 
// Usage across replicas:
// User A types "Hi"
// User B types "Hey" 
// Both merge to "HHiey" or "HHeyi" - deterministic, no lost characters

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

import { useState, useCallback } from 'react';
 
interface Task {
    id: string;
    title: string;
    completed: boolean;
    status: 'synced' | 'pending' | 'error';
}
 
function TaskList() {
    const [tasks, setTasks] = useState<Task[]>([]);
    
    const toggleComplete = useCallback(async (taskId: string) => {
        const task = tasks.find(t => t.id === taskId);
        if (!task) return;
        
        // 1. Optimistic update: immediately update UI
        setTasks(prev => prev.map(t => 
            t.id === taskId 
                ? { ...t, completed: !t.completed, status: 'pending' }
                : t
        ));
        
        try {
            // 2. Send to server
            await api.updateTask(taskId, { completed: !task.completed });
            
            // 3. Mark as synced
            setTasks(prev => prev.map(t =>
                t.id === taskId ? { ...t, status: 'synced' } : t
            ));
        } catch (error) {
            // 4. Rollback on failure
            setTasks(prev => prev.map(t =>
                t.id === taskId 
                    ? { ...t, completed: task.completed, status: 'error' }
                    : t
            ));
            
            // Show error notification
            showNotification({
                type: 'error',
                message: 'Failed to update task. Tap to retry.',
                action: () => toggleComplete(taskId)
            });
        }
    }, [tasks]);
    
    return (
        <ul className="task-list">
            {tasks.map(task => (
                <li 
                    key={task.id}
                    className={`task ${task.status}`}
                    onClick={() => toggleComplete(task.id)}
                >
                    <input 
                        type="checkbox" 
                        checked={task.completed}
                        readOnly
                    />
                    <span>{task.title}</span>
                    
                    {/* Status indicators */}
                    {task.status === 'pending' && <SyncingSpinner />}
                    {task.status === 'error' && <ErrorIcon />}
                </li>
            ))}
        </ul>
    );
}
 
// CSS for visual feedback
/*
.task.pending { opacity: 0.7; }
.task.error { border-left: 3px solid red; }
.task.synced { /* normal styling */ }
*/

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

// Compensation for oversold inventory
class InventoryCompensation {
    async handleOversell(oversoldOrders: Order[]): Promise<void> {
        // Sort by priority (e.g., loyalty tier, order time)
        oversoldOrders.sort((a, b) => {
            // Keep orders from high-value customers
            if (a.customerTier !== b.customerTier) {
                return tierPriority(b.customerTier) - tierPriority(a.customerTier);
            }
            // Otherwise, first come first served
            return a.orderTime - b.orderTime;
        });
        
        // Cancel from the end (lowest priority)
        for (const order of oversoldOrders.reverse()) {
            await this.compensateOrder(order);
        }
    }
    
    private async compensateOrder(order: Order): Promise<void> {
        // 1. Cancel the order
        await this.orderService.cancel(order.id, 'INVENTORY_UNAVAILABLE');
        
        // 2. Refund payment
        await this.paymentService.refund(order.paymentId, order.totalAmount);
        
        // 3. Notify customer with apology
        await this.notificationService.send({
            userId: order.userId,
            template: 'ORDER_CANCELLED_STOCK',
            data: {
                orderNumber: order.orderNumber,
                productName: order.productName,
                discountCode: await this.generateApologyDiscount(order.userId)
            }
        });
        
        // 4. Log for analysis
        await this.analytics.logOversellCompensation({
            orderId: order.id,
            productId: order.productId,
            amount: order.totalAmount,
            timestamp: Date.now()
        });
    }
}
 
// Compensation for distributed saga failure
class BookingCompensation {
    async compensateFailedBooking(sagaId: string): Promise<void> {
        const saga = await this.getSaga(sagaId);
        
        // Get completed steps that need reversal
        const completedSteps = saga.steps.filter(s => s.status === 'COMPLETED');
        
        // Compensate in reverse order
        for (let i = completedSteps.length - 1; i >= 0; i--) {
            const step = completedSteps[i];
            
            switch (step.name) {
                case 'CHARGE_CREDIT_CARD':
                    await this.paymentService.refund(step.transactionId);
                    break;
                    
                case 'BOOK_FLIGHT':
                    await this.flightService.cancelReservation(step.reservationId);
                    break;
                    
                case 'BOOK_HOTEL':
                    await this.hotelService.cancelReservation(step.reservationId);
                    break;
                    
                case 'BOOK_CAR':
                    await this.carService.cancelReservation(step.reservationId);
                    break;
            }
            
            step.status = 'COMPENSATED';
            await this.saveSaga(saga);
        }
        
        saga.status = 'COMPENSATED';
        await this.saveSaga(saga);
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

class ConsistencyMonitor {
    private metrics: MetricsClient;
    
    // Track replication lag
    async measureReplicationLag(): Promise<void> {
        const testKey = `__consistency_probe_${Date.now()}`;
        const testValue = { timestamp: Date.now(), probe: true };
        
        // Write to primary
        await this.db.write(testKey, testValue, { consistency: 'ONE' });
        
        // Read from each replica
        for (const replica of this.replicas) {
            const start = Date.now();
            let found = false;
            
            while (!found && Date.now() - start < 30000) {
                const value = await replica.read(testKey);
                if (value && value.timestamp === testValue.timestamp) {
                    found = true;
                    const lag = Date.now() - testValue.timestamp;
                    this.metrics.histogram('replication_lag_ms', lag, {
                        replica: replica.id
                    });
                } else {
                    await delay(10);
                }
            }
            
            if (!found) {
                this.metrics.increment('replication_timeout', { replica: replica.id });
            }
        }
        
        // Cleanup probe
        await this.db.delete(testKey);
    }
    
    // Track conflict rate
    onWriteConflict(key: string, resolution: ConflictResolution): void {
        this.metrics.increment('write_conflicts', {
            resolution: resolution.strategy,
            keyPrefix: key.split(':')[0]
        });
    }
    
    // Track compensation events
    onCompensation(type: string, original: string): void {
        this.metrics.increment('compensations', {
            type,
            originalOperation: original
        });
    }
    
    // Monitor saga health
    async checkStuckSagas(): Promise<void> {
        const stuckThreshold = Date.now() - 60 * 60 * 1000; // 1 hour
        
        const stuckSagas = await this.db.query('sagas', {
            status: 'RUNNING',
            startedAt: { $lt: stuckThreshold }
        });
        
        this.metrics.gauge('stuck_sagas', stuckSagas.length);
        
        if (stuckSagas.length > 10) {
            this.alerting.trigger({
                severity: 'HIGH',
                message: `${stuckSagas.length} sagas stuck for over 1 hour`,
                sagas: stuckSagas.map(s => s.sagaId)
            });
        }
    }
}