Latency Is Zero (It's Not)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

The Tail at Scale Problem
========================
 
Given:
- Single request p99 latency: 500ms (1% of requests)
- User action fans out to: 100 parallel backend requests
 
Probability that ALL 100 requests complete within p99:
  = (0.99)^100 = 36.6%
 
Probability that at least ONE request exceeds p99:
  = 1 - (0.99)^100 = 63.4%
 
Result: 
For 63% of user requests, at least one backend call will be slow,
making the ENTIRE user-visible request slow.
 
With 1000 parallel requests:
  Probability of at least one slow request = 1 - (0.99)^1000 = 99.996%
 
This is why at Google/Facebook/Amazon, p99 latency optimization
is often more important than average latency optimization.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

/**
 * Parallel vs Sequential Remote Calls
 * Demonstrates the dramatic latency difference
 */
 
interface DashboardData {
    user: User;
    notifications: Notification[];
    recentOrders: Order[];
    recommendations: Product[];
}
 
// ❌ Sequential: Total latency = sum of all call latencies
async function loadDashboardSequential(): Promise<DashboardData> {
    const user = await userService.getCurrentUser();           // 50ms
    const notifications = await notificationService.getAll();  // 50ms
    const recentOrders = await orderService.getRecent();       // 50ms
    const recommendations = await productService.getRecommended(); // 50ms
    
    return { user, notifications, recentOrders, recommendations };
    // Total: ~200ms
}
 
// ✅ Parallel: Total latency = max of all call latencies
async function loadDashboardParallel(): Promise<DashboardData> {
    const [user, notifications, recentOrders, recommendations] = 
        await Promise.all([
            userService.getCurrentUser(),           // 50ms ─┐
            notificationService.getAll(),           // 50ms ─┼─ Run simultaneously
            orderService.getRecent(),               // 50ms ─┤
            productService.getRecommended(),        // 50ms ─┘
        ]);
    
    return { user, notifications, recentOrders, recommendations };
    // Total: ~50ms (4x faster!)
}
 
// ✅✅ Parallel with Timeouts and Fallbacks: Production-grade
async function loadDashboardResilient(): Promise<DashboardData> {
    // Critical data - must succeed
    const userPromise = userService.getCurrentUser();
    
    // Non-critical data - use fallbacks on timeout
    const [user, notifications, recentOrders, recommendations] = 
        await Promise.all([
            userPromise,
            withTimeoutFallback(notificationService.getAll(), 100, []),
            withTimeoutFallback(orderService.getRecent(), 100, []),
            withTimeoutFallback(productService.getRecommended(), 100, []),
        ]);
    
    return { user, notifications, recentOrders, recommendations };
}
 
async function withTimeoutFallback<T>(
    promise: Promise<T>,
    timeoutMs: number,
    fallback: T
): Promise<T> {
    try {
        return await Promise.race([
            promise,
            new Promise<T>((_, reject) => 
                setTimeout(() => reject(new Error('Timeout')), timeoutMs)
            ),
        ]);
    } catch {
        return fallback;
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

/**
 * Example: Comprehensive latency tracking
 * Captures the full request lifecycle
 */
 
interface RequestTiming {
    requestId: string;
    
    // Client-side timing (if available)
    clientStart?: number;
    
    // Server-side timing
    receivedAt: number;
    queuedDuration: number;     // Time waiting for processing
    processingStart: number;
    
    // Downstream calls
    downstreamCalls: {
        service: string;
        startedAt: number;
        completedAt: number;
        success: boolean;
    }[];
    
    // Response
    processingEnd: number;
    sentAt: number;
}
 
class LatencyTracker {
    private histograms: Map<string, number[]> = new Map();
    
    recordTiming(timing: RequestTiming): void {
        // Calculate component latencies
        const queueLatency = timing.processingStart - timing.receivedAt;
        const processingLatency = timing.processingEnd - timing.processingStart;
        const totalServerLatency = timing.sentAt - timing.receivedAt;
        
        // Downstream breakdown
        for (const call of timing.downstreamCalls) {
            const duration = call.completedAt - call.startedAt;
            this.addToHistogram(`downstream.${call.service}`, duration);
        }
        
        // Record all components
        this.addToHistogram('queue', queueLatency);
        this.addToHistogram('processing', processingLatency);
        this.addToHistogram('total', totalServerLatency);
    }
    
    getPercentiles(metric: string): { p50: number; p95: number; p99: number } {
        const values = this.histograms.get(metric) || [];
        if (values.length === 0) return { p50: 0, p95: 0, p99: 0 };
        
        values.sort((a, b) => a - b);
        
        return {
            p50: values[Math.floor(values.length * 0.50)],
            p95: values[Math.floor(values.length * 0.95)],
            p99: values[Math.floor(values.length * 0.99)],
        };
    }
    
    private addToHistogram(name: string, value: number): void {
        if (!this.histograms.has(name)) {
            this.histograms.set(name, []);
        }
        this.histograms.get(name)!.push(value);
        
        // Keep bounded size (sample oldest)
        const arr = this.histograms.get(name)!;
        if (arr.length > 100000) {
            arr.splice(0, arr.length - 100000);
        }
    }
}