NUMA Architecture

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

#!/bin/bash
# Comprehensive NUMA profiling script
 
echo "=== System NUMA Configuration ==="
numactl --hardware
 
echo -e "\n=== System-Wide NUMA Statistics ==="
numastat
 
echo -e "\n=== NUMA Memory Distribution ==="
numastat -m
 
echo -e "\n=== Per-Process NUMA Breakdown ==="
# Replace 'myapp' with your application name
for pid in $(pgrep myapp); do
    echo "--- PID $pid ---"
    numastat -p $pid
done
 
echo -e "\n=== Live NUMA Monitoring (5 seconds) ==="
# Monitor numa_hit/miss rate changes
echo "Timestamp | numa_hit | numa_miss | numa_foreign | local_node | other_node"
for i in {1..5}; do
    stats=$(grep -E "numa_(hit|miss|foreign)|local_node|other_node" /proc/vmstat)
    hit=$(echo "$stats" | grep numa_hit | awk '{print $2}')
    miss=$(echo "$stats" | grep numa_miss | awk '{print $2}')
    foreign=$(echo "$stats" | grep numa_foreign | awk '{print $2}')
    local=$(echo "$stats" | grep local_node | awk '{print $2}')
    other=$(echo "$stats" | grep other_node | awk '{print $2}')
    echo "$(date +%H:%M:%S) | $hit | $miss | $foreign | $local | $other"
    sleep 1
done
 
echo -e "\n=== Memory Placement for Specific Process ==="
# Show NUMA mapping of process memory regions
pid=$(pgrep -f myapp | head -1)
if [ -n "$pid" ]; then
    echo "Memory regions for PID $pid:"
    head -20 /proc/$pid/numa_maps
fi
 
echo -e "\n=== Hardware Performance Counters (if available) ==="
# Check if perf can measure NUMA events
perf list 2>/dev/null | grep -i numa || echo "No NUMA perf events found"
 
# Example: measure NUMA events during application run
# perf stat -e node-loads,node-stores,node-load-misses,node-store-misses ./myapp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

# Basic NUMA event recording
perf stat -e node-loads,node-stores,node-load-misses,node-store-misses \
    ./my_application
 
# Sample output:
#  Performance counter stats for './my_application':
#       456,789,012      node-loads
#       123,456,789      node-stores
#        45,678,901      node-load-misses   # 10% remote loads (BAD)
#           345,678      node-store-misses  # 0.3% remote stores (OK)
 
# Calculate NUMA efficiency:
# Local Load Ratio = 1 - (node-load-misses / node-loads)
# Target: > 95%
 
# Record for detailed analysis
perf record -e node-loads,node-load-misses -g ./my_application
perf report
 
# AMD systems: Data Fabric events
perf stat -e amd_df/event=0x07,umask=0x02/ ./my_application
 
# Intel systems: UPI traffic
perf stat -e uncore_upi/event=0x02,umask=0x0f/ ./my_application
 
# Memory bandwidth by NUMA node (Intel)
perf stat -e uncore_imc_0/cas_count_read/,uncore_imc_0/cas_count_write/ \
          -e uncore_imc_1/cas_count_read/,uncore_imc_1/cas_count_write/ \
          ./my_application

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

#include <numa.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <pthread.h>
 
#define ARRAY_SIZE (512 * 1024 * 1024)  // 512 MB per thread
#define ITERATIONS 100000000
#define WARMUP_ITERATIONS 10000000
#define NUM_RUNS 5
 
typedef struct {
    int thread_id;
    int numa_node;
    void *buffer;
    double latency_ns;
    double bandwidth_gbps;
} ThreadResult;
 
// Pointer-chasing latency benchmark
double benchmark_latency(void *buffer, size_t size, int iterations) {
    void **chain = (void **)buffer;
    size_t num_elements = size / sizeof(void *);
    
    // Create random pointer chain (defeats prefetching)
    for (size_t i = 0; i < num_elements - 1; i++) {
        chain[i] = &chain[(i * 179 + 17) % num_elements];
    }
    chain[num_elements - 1] = &chain[0];
    
    // Touch all pages
    for (size_t i = 0; i < num_elements; i += 512) {
        volatile void *touch = chain[i];
        (void)touch;
    }
    
    // Warmup
    void **p = chain;
    for (int i = 0; i < WARMUP_ITERATIONS; i++) {
        p = (void **)*p;
    }
    
    // Timed run
    struct timespec start, end;
    clock_gettime(CLOCK_MONOTONIC, &start);
    
    p = chain;
    for (int i = 0; i < iterations; i++) {
        p = (void **)*p;
    }
    
    clock_gettime(CLOCK_MONOTONIC, &end);
    
    // Prevent optimization
    volatile void *sink = p;
    (void)sink;
    
    double elapsed_ns = (end.tv_sec - start.tv_sec) * 1e9 +
                        (end.tv_nsec - start.tv_nsec);
    return elapsed_ns / iterations;
}
 
// Sequential bandwidth benchmark
double benchmark_bandwidth(void *buffer, size_t size) {
    // Warmup
    memset(buffer, 1, size);
    
    struct timespec start, end;
    clock_gettime(CLOCK_MONOTONIC, &start);
    
    // Read entire buffer (bandwidth test)
    volatile long sum = 0;
    long *data = (long *)buffer;
    size_t count = size / sizeof(long);
    for (size_t i = 0; i < count; i += 8) {
        sum += data[i] + data[i+1] + data[i+2] + data[i+3] +
               data[i+4] + data[i+5] + data[i+6] + data[i+7];
    }
    
    clock_gettime(CLOCK_MONOTONIC, &end);
    
    double elapsed_sec = (end.tv_sec - start.tv_sec) +
                         (end.tv_nsec - start.tv_nsec) / 1e9;
    return (size / 1e9) / elapsed_sec;  // GB/s
}
 
void *benchmark_thread(void *arg) {
    ThreadResult *result = (ThreadResult *)arg;
    
    // Pin to NUMA node
    numa_run_on_node(result->numa_node);
    
    // Allocate locally
    result->buffer = numa_alloc_local(ARRAY_SIZE);
    if (!result->buffer) {
        fprintf(stderr, "Thread %d: allocation failed\n", result->thread_id);
        return NULL;
    }
    
    // Touch to materialize
    memset(result->buffer, 0, ARRAY_SIZE);
    
    // Run benchmarks
    result->latency_ns = benchmark_latency(result->buffer, ARRAY_SIZE, ITERATIONS);
    result->bandwidth_gbps = benchmark_bandwidth(result->buffer, ARRAY_SIZE);
    
    numa_free(result->buffer, ARRAY_SIZE);
    return NULL;
}
 
int main() {
    if (numa_available() < 0) {
        fprintf(stderr, "NUMA not available\n");
        return 1;
    }
    
    int num_nodes = numa_max_node() + 1;
    printf("NUMA Benchmark: %d nodes\n\n", num_nodes);
    
    // Run benchmark on each node
    printf("%-8s %15s %15s\n", "Node", "Latency (ns)", "Bandwidth (GB/s)");
    printf("%-8s %15s %15s\n", "----", "-----------", "---------------");
    
    for (int node = 0; node < num_nodes; node++) {
        double total_latency = 0;
        double total_bandwidth = 0;
        
        for (int run = 0; run < NUM_RUNS; run++) {
            ThreadResult result = {
                .thread_id = 0,
                .numa_node = node,
            };
            
            pthread_t thread;
            pthread_create(&thread, NULL, benchmark_thread, &result);
            pthread_join(thread, NULL);
            
            total_latency += result.latency_ns;
            total_bandwidth += result.bandwidth_gbps;
        }
        
        printf("Node %-3d %15.1f %15.1f\n",
               node,
               total_latency / NUM_RUNS,
               total_bandwidth / NUM_RUNS);
    }
    
    return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

#include <numa.h>
#include <pthread.h>
 
typedef struct {
    int node;
    void *local_data;
    size_t data_size;
    void (*work_fn)(void *, size_t);
} Worker;
 
void *colocated_worker(void *arg) {
    Worker *w = (Worker *)arg;
    
    // Step 1: Bind thread to designated NUMA node
    numa_run_on_node(w->node);
    
    // Step 2: Allocate data locally (thread is now on correct node)
    w->local_data = numa_alloc_local(w->data_size);
    
    // Step 3: Initialize data (first-touch on local node)
    memset(w->local_data, 0, w->data_size);
    
    // Step 4: Process - all accesses are local
    w->work_fn(w->local_data, w->data_size);
    
    numa_free(w->local_data, w->data_size);
    return NULL;
}
 
void spawn_colocated_workers(int num_workers, size_t data_per_worker,
                             void (*work_fn)(void *, size_t)) {
    int num_nodes = numa_max_node() + 1;
    
    pthread_t threads[num_workers];
    Worker workers[num_workers];
    
    for (int i = 0; i < num_workers; i++) {
        workers[i].node = i % num_nodes;  // Round-robin across nodes
        workers[i].data_size = data_per_worker;
        workers[i].work_fn = work_fn;
        
        pthread_create(&threads[i], NULL, colocated_worker, &workers[i]);
    }
    
    for (int i = 0; i < num_workers; i++) {
        pthread_join(threads[i], NULL);
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

# Conceptual data partitioning for NUMA
# (Actual memory binding requires C/ctypes or specialized libraries)
 
def partition_for_numa(data, num_nodes):
    """
    Partition data for NUMA-aware processing.
    
    Each partition should be:
    1. Allocated on its designated node
    2. Processed by threads running on that node
    3. Independent (no cross-partition access during processing)
    """
    partition_size = len(data) // num_nodes
    partitions = []
    
    for node in range(num_nodes):
        start = node * partition_size
        end = start + partition_size if node < num_nodes - 1 else len(data)
        
        # In real code, allocate this partition on 'node' using numa_alloc_onnode
        partition = data[start:end]
        partitions.append((node, partition))
    
    return partitions
 
def process_partitioned(partitions, process_fn):
    """
    Process each partition on its designated NUMA node.
    
    In real implementation:
    1. Pin thread to node
    2. Access only local partition
    3. Aggregate results at end (minimize cross-node sync)
    """
    results = []
    for node, partition in partitions:
        # spawn_thread_on_node(node, process_fn, partition)
        result = process_fn(partition)
        results.append(result)
    
    return aggregate(results)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

# PostgreSQL NUMA tuning
 
# Option 1: Interleave shared_buffers (large, shared by all backends)
numactl --interleave=all postgres -c shared_buffers=64GB
 
# Option 2: Run separate PostgreSQL instances per NUMA node
# Node 0 instance
numactl --cpunodebind=0 --membind=0 \
    postgres -D /var/lib/postgresql/node0 -p 5432
 
# Node 1 instance  
numactl --cpunodebind=1 --membind=1 \
    postgres -D /var/lib/postgresql/node1 -p 5433
 
# Then use connection pooler (PgBouncer) to route queries
 
# MySQL NUMA tuning
# In my.cnf:
# innodb_numa_interleave = 1  # Interleave buffer pool
 
# Or via command line
numactl --interleave=all mysqld \
    --innodb-buffer-pool-size=64G \
    --innodb-buffer-pool-instances=8
 
# MongoDB NUMA tuning
# MongoDB strongly recommends disabling zone_reclaim_mode
echo 0 > /proc/sys/vm/zone_reclaim_mode
 
# Then run with interleave
numactl --interleave=all mongod --dbpath /data/db

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

# Redis NUMA tuning
# Redis is single-threaded, so strict node binding is ideal
 
# Bind to single node (best if data fits in one node's memory)
numactl --cpunodebind=0 --membind=0 redis-server /etc/redis.conf
 
# For larger datasets, run multiple instances
for node in 0 1 2 3; do
    port=$((6379 + node))
    numactl --cpunodebind=$node --membind=$node \
        redis-server --port $port --daemonize yes
done
 
# Memcached NUMA tuning
# Memcached is multi-threaded, use interleave for shared hash table
 
numactl --interleave=all memcached -m 65536 -t $(nproc)
 
# Or per-node instances (better for very large deployments)
for node in 0 1 2 3; do
    port=$((11211 + node))
    cores_on_node=$(numactl -H | grep "node $node cpus:" | cut -d: -f2)
    num_cores=$(echo $cores_on_node | wc -w)
    
    numactl --cpunodebind=$node --membind=$node \
        memcached -p $port -t $num_cores -m 16384 -d
done

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

# JVM NUMA tuning
 
# Enable JVM NUMA support (works with G1GC)
java -XX:+UseNUMA -XX:+UseG1GC -Xmx64g -jar application.jar
 
# For ZGC (NUMA-aware by default in Java 15+)
java -XX:+UseZGC -Xmx64g -jar application.jar
 
# Combine with numactl for additional control
numactl --interleave=all \
    java -XX:+UseNUMA -XX:+UseG1GC -Xmx64g -jar application.jar
 
# Elasticsearch NUMA settings
# In jvm.options:
# -XX:+UseG1GC
# -XX:+UseNUMA
 
# Also consider:
# - Running one ES node per NUMA node (smaller heap, more instances)
# - Setting node.processors to CPUs on one NUMA node
 
# Kafka NUMA tuning
# Bind broker to single NUMA node for best latency
numactl --cpunodebind=0 --membind=0 \
    kafka-server-start.sh config/server.properties
 
# For throughput, interleave (log data is shared across partitions)
numactl --interleave=all \
    kafka-server-start.sh config/server.properties

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

// WRONG: Allocate in main(), workers access remote memory
void *shared_data;
 
int main() {
    shared_data = malloc(HUGE_SIZE);
    memset(shared_data, 0, HUGE_SIZE);  // All pages on main()'s node!
    
    for (int i = 0; i < NUM_WORKERS; i++) {
        spawn_worker(i);  // Workers run on various nodes
    }
    // Workers mostly access REMOTE memory!
}
 
// RIGHT: Workers allocate their own data
void worker(int id) {
    numa_run_on_node(id % numa_max_node());
    
    void *my_data = numa_alloc_local(WORKER_SIZE);
    memset(my_data, 0, WORKER_SIZE);  // First touch on local node
    
    process(my_data);  // All LOCAL accesses
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

// WRONG: Counters share cache lines
struct BadCounters {
    long count_thread_0;  // Offset 0
    long count_thread_1;  // Offset 8  - SAME CACHE LINE!
    long count_thread_2;  // Offset 16 - SAME CACHE LINE!
    long count_thread_3;  // Offset 24 - SAME CACHE LINE!
};
// Every increment invalidates other threads' caches!
 
// RIGHT: Pad to separate cache lines
#define CACHE_LINE 64
 
struct GoodCounters {
    long count_thread_0;
    char _pad0[CACHE_LINE - sizeof(long)];
    
    long count_thread_1;
    char _pad1[CACHE_LINE - sizeof(long)];
    
    long count_thread_2;
    char _pad2[CACHE_LINE - sizeof(long)];
    
    long count_thread_3;
    char _pad3[CACHE_LINE - sizeof(long)];
};
// Each counter on its own cache line - no false sharing

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

#!/bin/bash
# Production NUMA monitoring script
# Run via cron every minute, export to monitoring system
 
OUTPUT_FILE="/var/log/numa_metrics.json"
 
# Collect NUMA statistics
numa_stats=$(numastat -c 2>/dev/null | tail -n +3)
 
# Parse key metrics
numa_hit=$(grep numa_hit /proc/vmstat | awk '{print $2}')
numa_miss=$(grep numa_miss /proc/vmstat | awk '{print $2}')
numa_foreign=$(grep numa_foreign /proc/vmstat | awk '{print $2}')
pgmigrate=$(grep pgmigrate_success /proc/vmstat | awk '{print $2}')
 
# Calculate hit ratio
if [ $((numa_hit + numa_miss)) -gt 0 ]; then
    hit_ratio=$(echo "scale=4; $numa_hit / ($numa_hit + $numa_miss) * 100" | bc)
else
    hit_ratio="100"
fi
 
# Per-node memory (example for 4 nodes)
for node in 0 1 2 3; do
    meminfo="/sys/devices/system/node/node$node/meminfo"
    if [ -f "$meminfo" ]; then
        total=$(grep MemTotal "$meminfo" | awk '{print $4}')
        free=$(grep MemFree "$meminfo" | awk '{print $4}')
        used=$((total - free))
        usage=$((used * 100 / total))
        eval "node${node}_usage=$usage"
        eval "node${node}_used_mb=$((used / 1024))"
    fi
done
 
# Output as JSON for ingestion by Prometheus/Datadog/etc.
cat > "$OUTPUT_FILE" << EOF
{
  "timestamp": "$(date -Iseconds)",
  "numa_hit": $numa_hit,
  "numa_miss": $numa_miss,
  "numa_foreign": $numa_foreign,
  "hit_ratio_pct": $hit_ratio,
  "pgmigrate_success": $pgmigrate,
  "node0_usage_pct": ${node0_usage:- 0},
                            "node1_usage_pct": ${ node1_usage: -0 },
                        "node2_usage_pct": ${ node2_usage: -0 },
                        "node3_usage_pct": ${ node3_usage: -0 }
}
EOF
 
# Alert if hit ratio drops
if(($(echo "$hit_ratio < 90" | bc - l))); then
    echo "ALERT: NUMA hit ratio dropped to ${hit_ratio}%"
    # Integrate with alerting system
    fi