I/O Hardware Performance

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

#!/bin/bash
# Systematic I/O Bottleneck Analysis
# Collects data from all layers to identify constraints
 
echo "=== I/O Bottleneck Analysis ==="
echo "Timestamp: $(date)"
echo
 
# ============================================
# 1. STORAGE DEVICE ANALYSIS
# ============================================
 
echo "--- Storage Devices ---"
iostat -xm 1 5 | tail -20
 
# Key indicators:
# %util approaching 100%: Device is saturated
# avgqu-sz growing: Requests queuing
# await increasing: Latency rising with load
 
# Per-device detail
for dev in /sys/block/nvme* /sys/block/sd*; do
    devname=$(basename $dev)
    if [ -d "$dev" ]; then
        echo "Device: $devname"
        echo "  Queue depth: $(cat $dev/queue/nr_requests 2>/dev/null)"
        echo "  Scheduler: $(cat $dev/queue/scheduler 2>/dev/null)"
        # Check for errors
        if [ -f "$dev/device/errors" ]; then
            echo "  Errors: $(cat $dev/device/errors)"
        fi
    fi
done
 
echo
 
# ============================================
# 2. PCIe BANDWIDTH ANALYSIS
# ============================================
 
echo "--- PCIe Configuration ---"
# Check link speed and width
lspci -vvv 2>/dev/null | grep -E "LnkCap|LnkSta" | head -20
 
# Look for:
# - Link speed downgraded from capability
# - Width reduced (x4 when capable of x16)
 
# Per-NVMe device PCIe stats
echo "NVMe PCIe Info:"
for ctrl in /sys/class/nvme/nvme*; do
    if [ -d "$ctrl" ]; then
        echo "  $(basename $ctrl):"
        cat $ctrl/device/link_speed 2>/dev/null || echo "    Unknown"
    fi
done
 
echo
 
# ============================================
# 3. MEMORY BANDWIDTH ANALYSIS
# ============================================
 
echo "--- Memory Bandwidth ---"
# Using perf if available (requires root and PMU support)
if command -v perf &> /dev/null; then
    timeout 5 perf stat -e 'LLC-loads,LLC-load-misses,LLC-stores,LLC-store-misses' -a sleep 5 2>&1 | head -20
fi
 
# NUMA topology and memory distribution
numastat 2>/dev/null | head -20
 
# Memory controller utilization (Intel-specific)
if [ -d /sys/devices/uncore_imc_0 ]; then
    echo "Memory Controller Performance Counters available"
fi
 
echo
 
# ============================================  
# 4. NETWORK BOTTLENECK ANALYSIS
# ============================================
 
echo "--- Network Interfaces ---"
for iface in $(ls /sys/class/net/ | grep -v lo); do
    echo "Interface: $iface"
    speed=$(cat /sys/class/net/$iface/speed 2>/dev/null)
    echo "  Link Speed: ${speed}Mbps"
    
    # Get current throughput
    rx_bytes=$(cat /sys/class/net/$iface/statistics/rx_bytes)
    tx_bytes=$(cat /sys/class/net/$iface/statistics/tx_bytes)
    sleep 1
    rx_bytes2=$(cat /sys/class/net/$iface/statistics/rx_bytes)
    tx_bytes2=$(cat /sys/class/net/$iface/statistics/tx_bytes)
    
    rx_rate=$((($rx_bytes2 - $rx_bytes) * 8 / 1000000))
    tx_rate=$((($tx_bytes2 - $tx_bytes) * 8 / 1000000))
    
    echo "  RX Rate: ${rx_rate}Mbps  TX Rate: ${tx_rate}Mbps"
    
    if [ -n "$speed" ] && [ "$speed" -gt 0 ]; then
        echo "  RX Utilization: $(($rx_rate * 100 / $speed))%"
        echo "  TX Utilization: $(($tx_rate * 100 / $speed))%"
    fi
    
    # Check for errors
    echo "  Errors: $(cat /sys/class/net/$iface/statistics/tx_errors) TX, $(cat /sys/class/net/$iface/statistics/rx_errors) RX"
    echo "  Dropped: $(cat /sys/class/net/$iface/statistics/rx_dropped) RX"
done
 
echo
 
# ============================================
# 5. CPU I/O WAIT ANALYSIS
# ============================================
 
echo "--- CPU I/O Wait ---"
vmstat 1 5 | tail -5
# High 'wa' (I/O wait) indicates CPU is waiting for I/O
# If I/O devices aren't saturated but CPU is waiting, look for:
# - Synchronous I/O patterns
# - Single-threaded I/O submission
# - Lock contention
 
echo
 
# ============================================
# 6. QUEUE SATURATION ANALYSIS
# ============================================
 
echo "--- Queue Analysis ---"
# Block layer queue stats
for dev in /sys/block/*/queue; do
    devname=$(echo $dev | cut -d/ -f4)
    echo "$devname:"
    echo "  Max sectors per request: $(cat $dev/max_sectors_kb 2>/dev/null) KB"
    echo "  Queue depth: $(cat $dev/nr_requests 2>/dev/null)"
done
 
echo
echo "=== Analysis Complete ===
echo "Look for:
echo "  - Devices at 100% utilization
echo "  - Growing queue depths (avgqu-sz in iostat)
echo "  - High I/O wait with low device utilization (software bottleneck)
echo "  - PCIe link speed/width degradation
echo "  - Memory bandwidth saturation (high LLC misses)
echo "  - Network interfaces at line rate"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

#!/bin/bash
# Storage Bottleneck Diagnostic
 
echo "=== Storage Bottleneck Analysis ==="
 
# 1. Check interface negotiation vs capability
echo "--- PCIe Link Status ---"
for dev in /sys/class/nvme/nvme*/device; do
    if [ -d "$dev" ]; then
        devname=$(basename $(dirname $dev))
        echo "$devname:"
        # Current link
        cur_speed=$(cat $dev/current_link_speed 2>/dev/null)
        cur_width=$(cat $dev/current_link_width 2>/dev/null)
        # Maximum capability
        max_speed=$(cat $dev/max_link_speed 2>/dev/null)
        max_width=$(cat $dev/max_link_width 2>/dev/null)
        
        echo "  Current: $cur_speed x $cur_width"
        echo "  Maximum: $max_speed x $max_width"
        
        if [ "$cur_speed" != "$max_speed" ] || [ "$cur_width" != "$max_width" ]; then
            echo "  WARNING: Link running below capability!"
        fi
    fi
done
 
# 2. Check SMART for controller saturation hints
echo ""
echo "--- NVMe SMART Data ---"
for dev in /dev/nvme*n1; do
    if [ -b "$dev" ]; then
        echo "Device: $dev"
        nvme smart-log $dev 2>/dev/null | grep -E "temperature|throttle|warning"
    fi
done
 
# 3. Compare theoretical vs measured throughput
echo ""
echo "--- Sequential Read Test (10s) ---"
for dev in /dev/nvme*n1; do
    if [ -b "$dev" ]; then
        echo "Testing $dev..."
        # This requires fio installed
        fio --name=seqread --filename=$dev --rw=read --bs=128k \
            --direct=1 --ioengine=io_uring --iodepth=32 \
            --runtime=10 --time_based --group_reporting 2>&1 | grep -E "READ:|bw="
    fi
done
 
# 4. Check for thermal throttling
echo ""
echo "--- Thermal Check ---"
sensors 2>/dev/null | grep -iE "nvme|ssd|drive"
 
echo ""
echo "=== Interpretation ==="
echo "1. If link speed < max: PCIe slot/cable issue"
echo "2. If temperature high: Thermal throttling active"  
echo "3. If throughput << interface limit: Media/controller bottleneck"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

#!/usr/bin/env python3
"""
Memory and Interconnect Bottleneck Analysis
 
Analyzes memory bandwidth utilization and NUMA efficiency
to identify interconnect bottlenecks.
"""
 
import subprocess
import re
import os
 
def get_numa_topology():
    """Parse NUMA topology from numactl."""
    try:
        result = subprocess.run(['numactl', '--hardware'], 
                              capture_output=True, text=True)
        return result.stdout
    except FileNotFoundError:
        return "numactl not installed"
 
def get_numa_stats():
    """Get NUMA memory statistics."""
    try:
        result = subprocess.run(['numastat'], capture_output=True, text=True)
        return result.stdout
    except FileNotFoundError:
        return "numastat not installed"
 
def analyze_device_numa():
    """Check NVMe device NUMA locality."""
    devices = []
    nvme_path = '/sys/class/nvme'
    
    if os.path.exists(nvme_path):
        for nvme in os.listdir(nvme_path):
            numa_node_file = f'{nvme_path}/{nvme}/device/numa_node'
            if os.path.exists(numa_node_file):
                with open(numa_node_file) as f:
                    numa_node = f.read().strip()
                devices.append({
                    'device': nvme,
                    'numa_node': numa_node
                })
    
    return devices
 
def estimate_memory_bandwidth():
    """
    Estimate current memory bandwidth usage using perf.
    Requires Linux perf with memory controller PMU support.
    """
    # This is architecture-specific; shown for Intel
    perf_cmd = [
        'perf', 'stat', '-e',
        'uncore_imc_0/cas_count_read/,'
        'uncore_imc_0/cas_count_write/',
        '-a', 'sleep', '1'
    ]
    
    try:
        result = subprocess.run(perf_cmd, capture_output=True, text=True)
        return result.stderr  # perf outputs stats to stderr
    except FileNotFoundError:
        return "perf not available or insufficient permissions"
 
def check_pcie_topology():
    """Analyze PCIe topology for potential bottlenecks."""
    try:
        result = subprocess.run(['lspci', '-tv'], capture_output=True, text=True)
        return result.stdout
    except FileNotFoundError:
        return "lspci not installed"
 
def main():
    print("=" * 60)
    print("Memory and Interconnect Bottleneck Analysis")
    print("=" * 60)
    
    print("\n--- NUMA Topology ---")
    print(get_numa_topology())
    
    print("\n--- NUMA Statistics ---")
    print(get_numa_stats())
    
    print("\n--- NVMe Device NUMA Locality ---")
    devices = analyze_device_numa()
    for dev in devices:
        print(f"  {dev['device']}: NUMA node {dev['numa_node']}")
        if dev['numa_node'] == '-1':
            print("    WARNING: Device not associated with NUMA node!")
    
    print("\n--- PCIe Topology ---")
    print(check_pcie_topology())
    
    print("\n--- Memory Bandwidth Sample ---")
    print(estimate_memory_bandwidth())
    
    print("\n--- Recommendations ---")
    print("1. Bind I/O-intensive processes to same NUMA node as storage devices")
    print("2. Check for PCIe switches causing bandwidth sharing")
    print("3. Monitor memory bandwidth during high I/O to detect saturation")
    print("4. Verify all PCIe devices negotiated maximum link speed/width")
 
if __name__ == "__main__":
    main()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

#!/bin/bash
# Network Hardware Bottleneck Analysis
 
IFACE=${1:-eth0}
 
echo "=== Network Bottleneck Analysis: $IFACE ==="
echo
 
# 1. Check negotiated link speed
echo "--- Link Status ---"
ethtool $IFACE 2>/dev/null | grep -E "Speed|Duplex|Link detected"
 
# 2. Check PCIe bandwidth for high-speed NICs
echo ""
echo "--- NIC PCIe Status ---"
# Find NIC's PCIe device
nic_pci=$(ethtool -i $IFACE 2>/dev/null | grep bus-info | awk '{print $2}')
if [ -n "$nic_pci" ]; then
    echo "PCI Address: $nic_pci"
    lspci -vvv -s $nic_pci 2>/dev/null | grep -E "LnkCap|LnkSta" | head -4
fi
 
# 3. Check ring buffer configuration
echo ""
echo "--- Ring Buffer Settings ---"
ethtool -g $IFACE 2>/dev/null
 
# 4. Check RSS/RPS queue configuration
echo ""
echo "--- RSS Queue Configuration ---"
ethtool -l $IFACE 2>/dev/null
 
# 5. Check for packet drops/errors
echo ""
echo "--- Interface Statistics ---"
ethtool -S $IFACE 2>/dev/null | grep -iE "drop|error|miss|overflow" | head -20
 
# 6. Current throughput measurement
echo ""
echo "--- Current Throughput (5 second sample) ---"
rx1=$(cat /sys/class/net/$IFACE/statistics/rx_bytes)
tx1=$(cat /sys/class/net/$IFACE/statistics/tx_bytes)
sleep 5
rx2=$(cat /sys/class/net/$IFACE/statistics/rx_bytes)
tx2=$(cat /sys/class/net/$IFACE/statistics/tx_bytes)
 
rx_mbps=$(( ($rx2 - $rx1) * 8 / 5 / 1000000 ))
tx_mbps=$(( ($tx2 - $tx1) * 8 / 5 / 1000000 ))
 
echo "RX: $rx_mbps Mbps"
echo "TX: $tx_mbps Mbps"
 
# Get negotiated speed for utilization calculation
speed=$(ethtool $IFACE 2>/dev/null | grep Speed | awk '{print $2}' | tr -d 'Mb/s')
if [ -n "$speed" ] && [ "$speed" -gt 0 ]; then
    echo ""
    echo "RX Utilization: $(( $rx_mbps * 100 / $speed ))%"
    echo "TX Utilization: $(( $tx_mbps * 100 / $speed ))%"
fi
 
# 7. Interrupt distribution
echo ""
echo "--- Interrupt Distribution ---"
cat /proc/interrupts | grep -i $IFACE | head -10
 
echo ""
echo "=== Summary ==="
echo "Check for:"
echo "  - Link speed < expected (auto-negotiation issues)"
echo "  - PCIe width/speed < NIC capability"
echo "  - Drops or errors in statistics"
echo "  - Unbalanced interrupt distribution"
echo "  - High utilization on single queue while others idle"