I/O Hardware Performance

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192

"""
Bandwidth Utilization Analysis Framework
 
Provides comprehensive utilities for measuring, analyzing, and 
reporting bandwidth utilization across I/O subsystems.
"""
 
import time
from dataclasses import dataclass
from typing import List, Optional
import subprocess
import re
 
@dataclass
class BandwidthSample:
    """Single bandwidth measurement sample."""
    timestamp: float
    bytes_read: int
    bytes_written: int
    device_busy_pct: float  # 0-100
    
@dataclass  
class UtilizationReport:
    """Comprehensive utilization analysis."""
    device: str
    max_bandwidth_mbps: float
    
    # Throughput metrics
    avg_read_mbps: float
    avg_write_mbps: float
    peak_read_mbps: float
    peak_write_mbps: float
    
    # Utilization metrics
    avg_utilization_pct: float
    peak_utilization_pct: float
    sustained_utilization_pct: float  # During active periods only
    
    # Efficiency analysis
    read_write_ratio: float
    bandwidth_efficiency: float  # Actual vs theoretical
 
 
class BandwidthUtilizationAnalyzer:
    """
    Analyzes bandwidth utilization for block devices.
    
    Uses /proc/diskstats on Linux for accurate measurement.
    """
    
    def __init__(self, device: str, max_bandwidth_mbps: float):
        """
        Initialize analyzer.
        
        Args:
            device: Device name (e.g., 'nvme0n1', 'sda')
            max_bandwidth_mbps: Theoretical max bandwidth in MB/s
        """
        self.device = device
        self.max_bandwidth_mbps = max_bandwidth_mbps
        self.samples: List[BandwidthSample] = []
        
    def collect_sample(self) -> BandwidthSample:
        """Collect current bandwidth utilization sample."""
        # Read from /proc/diskstats
        with open('/proc/diskstats', 'r') as f:
            for line in f:
                fields = line.split()
                if len(fields) >= 14 and fields[2] == self.device:
                    # Fields: major minor name rd_ios rd_mrg rd_sect rd_ticks
                    #         wr_ios wr_mrg wr_sect wr_ticks ios_inflight
                    #         io_ticks weighted_io_ticks
                    bytes_read = int(fields[5]) * 512    # Sectors to bytes
                    bytes_written = int(fields[9]) * 512
                    io_ticks = int(fields[12])           # ms active
                    
                    sample = BandwidthSample(
                        timestamp=time.time(),
                        bytes_read=bytes_read,
                        bytes_written=bytes_written,
                        device_busy_pct=0.0  # Calculated from successive samples
                    )
                    self.samples.append(sample)
                    return sample
                    
        raise ValueError(f"Device {self.device} not found in /proc/diskstats")
    
    def collect_samples(self, duration_seconds: float, interval_seconds: float = 1.0):
        """Collect samples over a duration."""
        end_time = time.time() + duration_seconds
        while time.time() < end_time:
            self.collect_sample()
            time.sleep(interval_seconds)
    
    def analyze(self) -> UtilizationReport:
        """Analyze collected samples to produce utilization report."""
        if len(self.samples) < 2:
            raise ValueError("Need at least 2 samples for analysis")
        
        read_rates = []
        write_rates = []
        utilizations = []
        
        for i in range(1, len(self.samples)):
            prev, curr = self.samples[i-1], self.samples[i]
            dt = curr.timestamp - prev.timestamp
            
            if dt <= 0:
                continue
                
            # Calculate rates in MB/s
            read_rate = (curr.bytes_read - prev.bytes_read) / dt / (1024 * 1024)
            write_rate = (curr.bytes_written - prev.bytes_written) / dt / (1024 * 1024)
            total_rate = read_rate + write_rate
            
            utilization = (total_rate / self.max_bandwidth_mbps) * 100
            
            read_rates.append(read_rate)
            write_rates.append(write_rate)
            utilizations.append(min(utilization, 100))  # Cap at 100%
        
        # Calculate sustained utilization (only during active periods)
        active_utilizations = [u for u in utilizations if u > 1.0]  # >1% = active
        sustained_util = sum(active_utilizations) / len(active_utilizations) if active_utilizations else 0
        
        # Calculate read/write ratio
        total_read = sum(read_rates)
        total_write = sum(write_rates)
        rw_ratio = total_read / total_write if total_write > 0 else float('inf')
        
        return UtilizationReport(
            device=self.device,
            max_bandwidth_mbps=self.max_bandwidth_mbps,
            avg_read_mbps=sum(read_rates) / len(read_rates),
            avg_write_mbps=sum(write_rates) / len(write_rates),
            peak_read_mbps=max(read_rates),
            peak_write_mbps=max(write_rates),
            avg_utilization_pct=sum(utilizations) / len(utilizations),
            peak_utilization_pct=max(utilizations),
            sustained_utilization_pct=sustained_util,
            read_write_ratio=rw_ratio,
            bandwidth_efficiency=(sum(utilizations) / len(utilizations)) / 100
        )
 
    def print_report(self, report: UtilizationReport):
        """Print formatted utilization report."""
        print(f"\n{'='*60}")
        print(f"Bandwidth Utilization Report: {report.device}")
        print(f"{'='*60}")
        print(f"Max Bandwidth: {report.max_bandwidth_mbps:.1f} MB/s")
        print()
        print("Throughput:")
        print(f"  Average Read:  {report.avg_read_mbps:8.2f} MB/s")
        print(f"  Average Write: {report.avg_write_mbps:8.2f} MB/s")
        print(f"  Peak Read:     {report.peak_read_mbps:8.2f} MB/s")
        print(f"  Peak Write:    {report.peak_write_mbps:8.2f} MB/s")
        print()
        print("Utilization:")
        print(f"  Average:   {report.avg_utilization_pct:6.2f}%")
        print(f"  Peak:      {report.peak_utilization_pct:6.2f}%")
        print(f"  Sustained: {report.sustained_utilization_pct:6.2f}%")
        print()
        print("Efficiency:")
        print(f"  Read/Write Ratio: {report.read_write_ratio:.2f}")
        print(f"  Bandwidth Efficiency: {report.bandwidth_efficiency:.1%}")
        
        # Provide recommendations
        print()
        print("Analysis:")
        if report.avg_utilization_pct < 20:
            print("  ⚠ Low average utilization - check for IOPS bottleneck or idle time")
        elif report.avg_utilization_pct > 80:
            print("  ⚠ High utilization - approaching saturation")
        else:
            print("  ✓ Healthy utilization range")
            
        if report.peak_utilization_pct > 95 and report.avg_utilization_pct < 50:
            print("  ⚠ Bursty workload - consider spreading load or adding caching")
 
 
# Example usage
if __name__ == "__main__":
    analyzer = BandwidthUtilizationAnalyzer(
        device="nvme0n1",
        max_bandwidth_mbps=7000  # PCIe 4.0 x4 NVMe theoretical
    )
    
    print("Collecting samples for 60 seconds...")
    analyzer.collect_samples(duration_seconds=60, interval_seconds=1.0)
    
    report = analyzer.analyze()
    analyzer.print_report(report)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

#!/bin/bash
# Bandwidth Utilization Monitoring Commands
 
# ============================================
# STORAGE BANDWIDTH UTILIZATION
# ============================================
 
# iostat with utilization (%util column)
# Shows both throughput and device utilization
iostat -xm 1
 
# Example output interpretation:
# Device   r/s   w/s   rMB/s   wMB/s  %util
# nvme0n1  1200  800   600     400    45%
#
# This indicates:
# - Total throughput: 600 + 400 = 1000 MB/s
# - Device is busy 45% of the time
# - If max bandwidth is 7000 MB/s: 1000/7000 = 14.3% bandwidth utilization
# - But device is only busy 45% of time
# - During busy time: 14.3% / 45% = ~32% sustained utilization
 
# ============================================
# NETWORK BANDWIDTH UTILIZATION
# ============================================
 
# sar for network interface utilization
sar -n DEV 1
 
# More detailed with iftop
sudo iftop -i eth0 -t -s 10
 
# Calculate utilization for 10 GbE link (1250 MB/s max)
# If observing 800 MB/s: 800/1250 = 64% utilization
 
# ============================================
# PCIE BANDWIDTH UTILIZATION
# ============================================
 
# Use perf with PCIe events (requires appropriate PMU support)
sudo perf stat -e 'pci/r/w bytes' -a sleep 10
 
# Or use Intel PCM (Performance Counter Monitor)
# Shows per-socket PCIe bandwidth
sudo pcm-pcie
 
# ============================================
# MEMORY BANDWIDTH UTILIZATION
# ============================================
 
# Intel PCM for memory bandwidth
sudo pcm-memory 1
 
# Or use perf with memory controller events
sudo perf stat -e 'uncore_imc/cas_count_read/,uncore_imc/cas_count_write/' -a sleep 10
 
# ============================================
# AUTOMATED UTILIZATION TRACKING
# ============================================
 
# Log to file for historical analysis
(
while true; do
    echo "=== $(date) ===" >> /var/log/bandwidth_util.log
    iostat -xm 1 1 | tail -n +7 >> /var/log/bandwidth_util.log
    sleep 60
done
) &
 
# Prometheus node_exporter provides:
# - node_disk_read_bytes_total
# - node_disk_written_bytes_total
# - node_network_receive_bytes_total
# - node_network_transmit_bytes_total
# Calculate utilization as rate(metric) / max_bandwidth

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

#!/bin/bash
# Bandwidth Management with cgroups v2
# Isolate workloads and prevent noisy neighbor effects
 
# Enable cgroup v2 controllers
echo "+io +cpu +memory" > /sys/fs/cgroup/cgroup.subtree_control
 
# ============================================
# Create isolated cgroups for different workloads
# ============================================
 
# High priority workload (production database)
mkdir -p /sys/fs/cgroup/prod-db
echo "256:0 rbps=3000000000 wbps=2000000000" > /sys/fs/cgroup/prod-db/io.max
# Device 256:0 (check with lsblk), read 3GB/s, write 2GB/s
 
# Low priority workload (backup job)  
mkdir -p /sys/fs/cgroup/backup
echo "256:0 rbps=500000000 wbps=500000000" > /sys/fs/cgroup/backup/io.max
# Limited to 500MB/s read and write
 
# Best-effort workload (dev/test)
mkdir -p /sys/fs/cgroup/dev
echo "256:0 rbps=max wbps=max" > /sys/fs/cgroup/dev/io.max
echo "256:0 100" > /sys/fs/cgroup/dev/io.weight  # Lower weight for fair sharing
 
# ============================================
# Set I/O priority weights (relative scheduling)
# ============================================
 
# Higher weight = higher priority in contention
# Range: 1-10000, default 100
 
echo "256:0 500" > /sys/fs/cgroup/prod-db/io.weight   # 5x priority
echo "256:0 50" > /sys/fs/cgroup/backup/io.weight     # 0.5x priority  
echo "256:0 100" > /sys/fs/cgroup/dev/io.weight       # Normal
 
# ============================================
# Launch processes in cgroups
# ============================================
 
# Run database in prod cgroup
echo $DATABASE_PID > /sys/fs/cgroup/prod-db/cgroup.procs
 
# Run backup in limited cgroup
cgexec -g io:backup /usr/bin/backup-script.sh
 
# ============================================
# Monitor per-cgroup I/O
# ============================================
 
# View current I/O statistics per cgroup
cat /sys/fs/cgroup/*/io.stat
 
# Example output:
# 256:0 rbytes=1234567890 wbytes=987654321 rios=12345 wios=9876 dbytes=0 dios=0
 
# ============================================
# NUMA-aware bandwidth isolation
# ============================================
 
# Bind to specific NUMA node for consistent performance
echo "0" > /sys/fs/cgroup/prod-db/cpuset.mems
echo "0-7" > /sys/fs/cgroup/prod-db/cpuset.cpus
 
# This ensures database traffic uses NUMA-local memory
# and PCIe paths, reducing cross-socket bandwidth contention

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153

/**
 * High-Bandwidth I/O Pattern Using io_uring
 * 
 * Demonstrates techniques for maximizing bandwidth utilization:
 * - Large aligned I/O requests (1MB)
 * - Deep queue depth (64 outstanding)
 * - Kernel-side polling to reduce syscall overhead
 * - Registered buffers to avoid memory mapping overhead
 */
 
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <string.h>
#include <liburing.h>
 
#define QUEUE_DEPTH 64
#define BLOCK_SIZE (1024 * 1024)  // 1MB blocks for high bandwidth
#define NUM_BLOCKS 1024
 
struct io_data {
    int fd;
    off_t offset;
    struct iovec iov;
};
 
/**
 * Initialize io_uring with optimal settings for bandwidth
 */
int setup_io_uring(struct io_uring *ring) {
    struct io_uring_params params = {0};
    
    // Enable kernel-side polling (reduces syscalls)
    params.flags = IORING_SETUP_SQPOLL;
    params.sq_thread_idle = 1000;  // ms before kernel thread sleeps
    
    int ret = io_uring_queue_init_params(QUEUE_DEPTH, ring, &params);
    if (ret < 0) {
        fprintf(stderr, "io_uring init failed: %d\n", ret);
        return -1;
    }
    
    return 0;
}
 
/**
 * Submit read requests to fill queue
 */
int submit_reads(struct io_uring *ring, int fd, 
                 void **buffers, off_t *offsets, int count) {
    for (int i = 0; i < count; i++) {
        struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
        if (!sqe) {
            // Queue full, submit and wait for space
            io_uring_submit(ring);
            sqe = io_uring_get_sqe(ring);
        }
        
        // Prepare read with pre-allocated aligned buffer
        io_uring_prep_read(sqe, fd, buffers[i], BLOCK_SIZE, offsets[i]);
        
        // Store index for completion tracking
        io_uring_sqe_set_data(sqe, (void*)(long)i);
    }
    
    return io_uring_submit(ring);
}
 
/**
 * Process completions and resubmit new reads
 */
int process_completions(struct io_uring *ring) {
    struct io_uring_cqe *cqe;
    unsigned head;
    int completed = 0;
    
    io_uring_for_each_cqe(ring, head, cqe) {
        if (cqe->res < 0) {
            fprintf(stderr, "I/O error: %d\n", cqe->res);
        } else if (cqe->res != BLOCK_SIZE) {
            fprintf(stderr, "Short read: %d\n", cqe->res);
        }
        completed++;
    }
    
    if (completed > 0) {
        io_uring_cq_advance(ring, completed);
    }
    
    return completed;
}
 
/**
 * Main high-bandwidth read loop
 */
int high_bandwidth_read(const char *path, size_t total_bytes) {
    struct io_uring ring;
    int fd;
    
    // Open with O_DIRECT for direct device access
    fd = open(path, O_RDONLY | O_DIRECT);
    if (fd < 0) {
        perror("open");
        return -1;
    }
    
    if (setup_io_uring(&ring) < 0) {
        close(fd);
        return -1;
    }
    
    // Allocate aligned buffers
    void *buffers[QUEUE_DEPTH];
    off_t offsets[QUEUE_DEPTH];
    
    for (int i = 0; i < QUEUE_DEPTH; i++) {
        posix_memalign(&buffers[i], BLOCK_SIZE, BLOCK_SIZE);
        offsets[i] = (off_t)i * BLOCK_SIZE;
    }
    
    // Initial submission to fill queue
    submit_reads(&ring, fd, buffers, offsets, QUEUE_DEPTH);
    
    size_t bytes_read = 0;
    off_t next_offset = QUEUE_DEPTH * BLOCK_SIZE;
    
    while (bytes_read < total_bytes) {
        // Wait for at least one completion
        struct io_uring_cqe *cqe;
        io_uring_wait_cqe(&ring, &cqe);
        
        // Process all available completions
        int completed = process_completions(&ring);
        bytes_read += completed * BLOCK_SIZE;
        
        // Resubmit to maintain queue depth
        for (int i = 0; i < completed && next_offset < total_bytes; i++) {
            struct io_uring_sqe *sqe = io_uring_get_sqe(&ring);
            io_uring_prep_read(sqe, fd, buffers[i], BLOCK_SIZE, next_offset);
            next_offset += BLOCK_SIZE;
        }
        io_uring_submit(&ring);
    }
    
    // Cleanup
    io_uring_queue_exit(&ring);
    for (int i = 0; i < QUEUE_DEPTH; i++) {
        free(buffers[i]);
    }
    close(fd);
    
    return 0;
}