Network Bottlenecks: The Invisible Constraint in Distributed Systems

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

#!/bin/bash
# =====================================================
# DIAGNOSING BANDWIDTH SATURATION
# =====================================================
 
# Real-time network interface throughput
iftop                          # Interactive bandwidth by connection
nethogs                        # Bandwidth per process
nload                          # Graph of network utilization
 
# Interface statistics
ip -s link show eth0           # Packets/bytes transmitted/received
cat /proc/net/dev              # Similar, raw numbers
 
# Sustained bandwidth test (between two hosts)
iperf3 -s                      # On server: start iperf server
iperf3 -c <server_ip>          # On client: test bandwidth
 
# Example output:
# [ ID] Interval           Transfer     Bitrate
# [  5]   0.00-10.00  sec  1.09 GBytes   937 Mbits/sec
# 
# Interpretation:
# - Close to theoretical max = saturated
# - Well below = no bandwidth issue, look elsewhere
 
 
# =====================================================
# IDENTIFYING BANDWIDTH-CONSUMING PROCESSES
# =====================================================
 
# Which process is using bandwidth?
nethogs eth0
# Shows PID, user, program, and KB/s sent/received
 
# For container environments
cat /sys/fs/cgroup/memory/docker/<container_id>/memory.usage_in_bytes
# Container network stats in /proc/<pid>/net/dev
 
 
# =====================================================
# CLOUD-SPECIFIC BANDWIDTH LIMITS
# =====================================================
 
# AWS: EBS bandwidth is often the hidden limit
# EC2 instances have EBS bandwidth caps (e.g., 4,750 Mbps for m5.xlarge)
# Symptoms: Disk I/O appears slow, but it's network to EBS
 
# Check EBS bandwidth consumption in CloudWatch:
# - VolumeReadBytes, VolumeWriteBytes
# - If near instance EBS bandwidth cap, you're throttled
 
# GCP: Network egress has per-VM limits
# Check instance type documentation for limits
 
# Azure: Similar per-VM bandwidth throttling
# Accelerated networking can help

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

# =====================================================
# LATENCY AMPLIFICATION IN MICROSERVICES
# =====================================================
 
# A typical e-commerce checkout request:
#
# Client → API Gateway → Order Service → [
#     Inventory Service,
#     Payment Service → External Payment Gateway,
#     Shipping Service,
#     Notification Service
# ]
#
# If each hop is 5ms within datacenter:
# Sequential: 5 + 5 + 5 + 5 + 5 + 5 = 30ms network latency
# Plus 100ms to external payment gateway
# Total: 130ms+ just in network latency
 
import asyncio
import time
 
# =====================================================
# ANTI-PATTERN: SEQUENTIAL CALLS
# =====================================================
 
async def checkout_sequential(order):
    """Each call waits for the previous - latency accumulates."""
    
    start = time.time()
    
    # Sequential calls - each waits for previous
    inventory = await check_inventory(order.items)     # 5ms
    payment = await process_payment(order.payment)     # 100ms (external)
    shipping = await calculate_shipping(order.address) # 5ms
    await reserve_inventory(order.items)               # 5ms
    await send_confirmation(order.user)                # 5ms
    
    total_latency = time.time() - start
    # Result: ~120ms of sequential network latency
    return total_latency
 
 
# =====================================================
# PATTERN: PARALLEL CALLS WHERE POSSIBLE
# =====================================================
 
async def checkout_parallel(order):
    """
    Identify independent operations and parallelize.
    Still respect dependencies.
    """
    
    start = time.time()
    
    # Phase 1: Independent operations in parallel
    inventory_check, shipping_calc = await asyncio.gather(
        check_inventory(order.items),        # 5ms
        calculate_shipping(order.address),   # 5ms
    )  # Total: 5ms (parallel)
    
    # Phase 2: Dependent on inventory check
    if not inventory_check.available:
        raise OutOfStockError()
    
    payment = await process_payment(order.payment)  # 100ms (external, must be sequential)
    
    # Phase 3: After payment, can parallelize again
    await asyncio.gather(
        reserve_inventory(order.items),      # 5ms
        send_confirmation(order.user),       # 5ms
    )  # Total: 5ms (parallel)
    
    total_latency = time.time() - start
    # Result: ~110ms (5 + 100 + 5 = 110ms)
    # Saved 10ms compared to sequential
    return total_latency
 
 
# =====================================================
# PATTERN: REDUCING ROUND-TRIPS
# =====================================================
 
# ANTI-PATTERN: Multiple calls to same service
async def get_product_details_bad(product_ids):
    results = []
    for pid in product_ids:
        # N round-trips for N products!
        product = await product_service.get(pid)
        results.append(product)
    return results
 
# PATTERN: Batch API
async def get_product_details_good(product_ids):
    # Single round-trip for all products
    return await product_service.get_batch(product_ids)
 
 
# =====================================================
# MEASURING CALL CHAIN LATENCY
# =====================================================
 
# Distributed tracing (Jaeger, Zipkin, Datadog APM) shows:
# 1. Total request latency
# 2. Per-service latency breakdown
# 3. Sequential vs parallel call patterns
# 4. Where latency is introduced
#
# Critical for identifying which calls to optimize or parallelize

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

TCP Three-Way Handshake:
 
Client                                Server
  |                                     |
  |-------- SYN (seq=x) --------------->|  RTT #1 start
  |                                     |
  |<------ SYN-ACK (seq=y, ack=x+1) ----|  RTT #1 end / RTT #2 start
  |                                     |
  |-------- ACK (ack=y+1) ------------->|  RTT #2 end
  |                                     |
  |  Connection established, can send   |
  |                                     |
 
Time cost: 1.5 RTT (round-trip times)
For 50ms RTT: 75ms just to open connection before any data
 
 
TLS Handshake (on top of TCP):
 
Client                                Server
  |                                     |
  |-------- TCP SYN ------------------->|
  |<------ TCP SYN-ACK -----------------|
  |-------- TCP ACK ------------------->|  (1.5 RTT for TCP)
  |                                     |
  |-------- ClientHello --------------->|
  |<------ ServerHello, Certificate ----|
  |-------- KeyExchange, Finished ----->|
  |<------ Finished --------------------|  (2 RTT for TLS 1.2)
  |                                     |
  |  Secure connection established      |
 
Time cost: 3.5 RTT total (TCP + TLS 1.2)
TLS 1.3: 2.5 RTT (1 fewer round-trip)
0-RTT resumption: 1.5 RTT for subsequent connections
 
For 50ms RTT:
- TCP only: 75ms
- TLS 1.2: 175ms
- TLS 1.3: 125ms
- TLS 1.3 0-RTT: 75ms (resumed)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

# =====================================================
# HTTP CONNECTION POOLING (CLIENT SIDE)
# =====================================================
 
import aiohttp
import httpx
 
# ANTI-PATTERN: New connection per request
async def fetch_bad(url):
    async with aiohttp.ClientSession() as session:
        # Creates new session (connection pool) per call!
        async with session.get(url) as response:
            return await response.text()
 
# Each call: TCP + TLS handshake = 150ms+ overhead
 
 
# PATTERN: Shared connection pool
class HttpClient:
    """Singleton pattern for shared connection pool."""
    
    _session = None
    
    @classmethod
    async def get_session(cls):
        if cls._session is None or cls._session.closed:
            # Pool configuration
            connector = aiohttp.TCPConnector(
                limit=100,           # Max total connections
                limit_per_host=10,   # Max per destination host
                keepalive_timeout=30, # Keep idle connections for 30s
                enable_cleanup_closed=True,
            )
            cls._session = aiohttp.ClientSession(connector=connector)
        return cls._session
    
    @classmethod
    async def fetch(cls, url):
        session = await cls.get_session()
        async with session.get(url) as response:
            return await response.text()
 
# First call: 150ms (new connection)
# Subsequent calls: 5ms (reused connection from pool)
 
 
# =====================================================
# GRPC CONNECTION MANAGEMENT
# =====================================================
 
import grpc
 
# ANTI-PATTERN: New channel per call
def call_service_bad(request):
    channel = grpc.insecure_channel('service:50051')
    stub = MyServiceStub(channel)
    return stub.Method(request)
    # Channel closed after call - connection wasted
 
# PATTERN: Shared channel with health checking
class GrpcClient:
    def __init__(self, target):
        self.channel = grpc.insecure_channel(
            target,
            options=[
                ('grpc.keepalive_time_ms', 30000),
                ('grpc.keepalive_timeout_ms', 10000),
                ('grpc.keepalive_permit_without_calls', True),
                ('grpc.http2.max_pings_without_data', 0),
            ]
        )
        self.stub = MyServiceStub(self.channel)
    
    def call(self, request):
        return self.stub.Method(request)
 
# Connection established once, reused indefinitely
 
 
# =====================================================
# MONITORING CONNECTION HEALTH
# =====================================================
 
# Key metrics to track:
# 1. Connection pool utilization: active / max connections
# 2. Wait time for connection from pool
# 3. Connection error rate (failed to establish)
# 4. New connections per second (should be low with good pooling)
# 5. Connection lifetime (long = healthy reuse)
 
# Detect connection leaks:
# - Pool exhausted but few active connections
# - Connections not returned to pool after use
# - Always use context managers / try-finally for cleanup

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

#!/bin/bash
# =====================================================
# STEP 1: BASIC CONNECTIVITY AND LATENCY
# =====================================================
 
# Ping: basic latency measurement
ping -c 10 target-host
# Check: min/avg/max/stddev
# High stddev = jitter = unstable connection
 
# MTR: traceroute with continuous stats
mtr target-host
# Shows per-hop latency and packet loss
# Identifies which network segment is slow
 
# TCP-specific latency (more realistic than ICMP)
hping3 -S -p 443 target-host
# Or use tcping:
tcping target-host 443
 
 
# =====================================================
# STEP 2: BANDWIDTH TESTING
# =====================================================
 
# iperf3: point-to-point bandwidth test
# On server:
iperf3 -s
 
# On client:
iperf3 -c server-ip -t 30 -P 4
# -t 30: run for 30 seconds
# -P 4: use 4 parallel streams
 
# Interpretation:
# Measured vs expected = saturation indicator
 
 
# =====================================================
# STEP 3: DNS RESOLUTION
# =====================================================
 
# DNS lookup time (often overlooked latency source)
dig target-host | grep "Query time"
# Or:
time nslookup target-host
 
# If high: 
# - Check DNS server proximity
# - Implement DNS caching (dnsmasq, resolved)
# - Consider DNS prefetching in applications
 
 
# =====================================================
# STEP 4: TLS HANDSHAKE ANALYSIS
# =====================================================
 
# Measure TLS handshake time
curl -w "DNS: %{time_namelookup}s
Connect: %{time_connect}s
TLS: %{time_appconnect}s
First byte: %{time_starttransfer}s
Total: %{time_total}s
" -o /dev/null -s https://target-host
 
# Example output:
# DNS: 0.024s
# Connect: 0.055s      (TCP handshake done)
# TLS: 0.142s          (TLS handshake done)
# First byte: 0.198s   (server processing + first response)
# Total: 0.234s
 
# Breakdown:
# DNS time: 24ms
# TCP handshake: 55-24 = 31ms
# TLS handshake: 142-55 = 87ms
# Server processing: 198-142 = 56ms
# Data transfer: 234-198 = 36ms
 
 
# =====================================================
# STEP 5: PACKET CAPTURE ANALYSIS
# =====================================================
 
# Capture traffic for detailed analysis
tcpdump -i eth0 -w capture.pcap host target-host
 
# Analyze in Wireshark:
# - Look for retransmissions (packet loss)
# - Check TCP window size (could be limiting throughput)
# - Identify slow segments in the connection
 
# Quick stats from tcpdump:
tcpdump -r capture.pcap -q | wc -l  # packet count
tcpdump -r capture.pcap 'tcp[tcpflags] & tcp-syn != 0' | wc -l  # connections
 
 
# =====================================================
# STEP 6: APPLICATION-LEVEL TRACING
# =====================================================
 
# Distributed tracing shows where time is spent
# Tools: Jaeger, Zipkin, Datadog APM, Honeycomb
 
# Look for:
# - Time between spans (network latency)
# - External service call duration
# - Database query time vs connection acquisition time

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

# =====================================================
# BACKEND-FOR-FRONTEND (BFF) PATTERN
# =====================================================
# 
# Problem: Mobile app needs data from 5 services
# Without BFF: 5 round-trips to remote services (500ms+ latency)
# With BFF: 1 round-trip to BFF, BFF fans out internally
 
# =====================================================
# CLIENT PERSPECTIVE
# =====================================================
 
# WITHOUT BFF (from mobile app)
async def load_product_page_old(product_id):
    # Each call crosses internet to backend
    product = await api.get(f"/products/{product_id}")    # 100ms
    reviews = await api.get(f"/reviews/{product_id}")     # 100ms  
    inventory = await api.get(f"/inventory/{product_id}") # 100ms
    pricing = await api.get(f"/pricing/{product_id}")     # 100ms
    related = await api.get(f"/related/{product_id}")     # 100ms
    # Total: ~500ms of network latency from mobile
 
# WITH BFF (from mobile app)
async def load_product_page_new(product_id):
    # One call to BFF, which aggregates internally
    response = await api.get(f"/bff/product-page/{product_id}")
    # Total: ~100ms from mobile (one round-trip)
    return response  # Contains all data pre-assembled
 
 
# =====================================================
# BFF SERVER IMPLEMENTATION
# =====================================================
 
from fastapi import FastAPI
import asyncio
 
app = FastAPI()
 
@app.get("/bff/product-page/{product_id}")
async def get_product_page(product_id: str):
    """
    Aggregate data from multiple services for product page.
    Services are in same datacenter - ~2ms latency each.
    """
    
    # Parallel calls within datacenter
    product, reviews, inventory, pricing, related = await asyncio.gather(
        product_service.get(product_id),    # 2ms
        review_service.get(product_id),     # 2ms
        inventory_service.get(product_id),  # 2ms
        pricing_service.get(product_id),    # 2ms
        related_service.get(product_id),    # 2ms
    )
    # Total internal latency: ~2ms (parallel)
    
    # Return pre-assembled response
    return {
        "product": product,
        "reviews": reviews,
        "inventory": inventory,
        "pricing": pricing,
        "related_products": related,
    }
 
# Client latency breakdown:
# - Mobile -> BFF: 100ms
# - BFF -> Services (parallel): 2ms
# - Total: ~102ms vs ~500ms = 5x improvement