Thrashing Solutions: Restoring System Stability

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

# Memory Capacity Planning Calculator
# Guides sizing decisions for physical memory
 
from dataclasses import dataclass
from typing import List
 
@dataclass
class ProcessProfile:
    name: str
    instances: int
    wss_per_instance_mb: int  # Working set size
    peak_multiplier: float     # Peak/average ratio
    growth_rate_yearly: float  # Expected growth
 
@dataclass
class SystemOverhead:
    os_base_mb: int = 512      # Base OS requirement
    os_per_gb_mb: int = 10     # OS overhead scales with RAM
    kernel_buffers_mb: int = 256
    system_services_mb: int = 512
 
@dataclass  
class CacheRequirements:
    file_cache_percent: float = 15.0  # % of RAM for file cache
    application_cache_mb: int = 0
    db_buffer_pool_mb: int = 0
 
def calculate_memory_requirement(
    processes: List[ProcessProfile],
    overhead: SystemOverhead,
    cache: CacheRequirements,
    headroom_percent: float = 20.0,
    planning_horizon_years: int = 2
) -> dict:
    """
    Calculate total memory requirement with growth projection.
    """
    
    # Calculate base process requirements
    base_process_memory = 0
    peak_process_memory = 0
    
    for proc in processes:
        base = proc.instances * proc.wss_per_instance_mb
        peak = base * proc.peak_multiplier
        
        # Project growth
        growth_factor = (1 + proc.growth_rate_yearly) ** planning_horizon_years
        projected = peak * growth_factor
        
        base_process_memory += base
        peak_process_memory += projected
    
    # Cache requirements
    # Note: File cache is dynamic, but we allocate space for it
    cache_memory = cache.application_cache_mb + cache.db_buffer_pool_mb
    
    # Subtotal before OS overhead (need this to calculate OS scaling)
    subtotal_mb = peak_process_memory + cache_memory
    
    # OS overhead (scales partially with total RAM)
    # Assume we're sizing for the subtotal
    estimated_total_gb = subtotal_mb / 1024 * 1.3  # Rough estimate
    os_memory = (
        overhead.os_base_mb + 
        overhead.os_per_gb_mb * estimated_total_gb +
        overhead.kernel_buffers_mb +
        overhead.system_services_mb
    )
    
    # Total before headroom
    total_before_headroom = subtotal_mb + os_memory
    
    # File cache allocation (on top of requirements)
    file_cache_alloc = total_before_headroom * (cache.file_cache_percent / 100)
    
    # Headroom for stability
    working_total = total_before_headroom + file_cache_alloc
    headroom_mb = working_total * (headroom_percent / 100)
    
    total_required = working_total + headroom_mb
    
    # Convert to practical RAM sizes (round up to standard sizes)
    practical_sizes = [8, 16, 32, 64, 128, 256, 512, 1024, 2048]  # GB
    total_gb = total_required / 1024
    
    recommended_gb = next(
        (size for size in practical_sizes if size >= total_gb),
        practical_sizes[-1]
    )
    
    return {
        'base_process_memory_mb': base_process_memory,
        'peak_process_memory_mb': peak_process_memory,
        'os_overhead_mb': os_memory,
        'cache_memory_mb': cache_memory + file_cache_alloc,
        'headroom_mb': headroom_mb,
        'total_required_mb': total_required,
        'total_required_gb': total_gb,
        'recommended_gb': recommended_gb,
        'planning_horizon_years': planning_horizon_years
    }
 
# Example usage
if __name__ == '__main__':
    processes = [
        ProcessProfile("Web Server", 4, 512, 1.5, 0.2),
        ProcessProfile("Database", 1, 8192, 1.3, 0.3),
        ProcessProfile("Cache Server", 1, 4096, 1.2, 0.25),
        ProcessProfile("Background Workers", 8, 256, 2.0, 0.15),
    ]
    
    cache = CacheRequirements(
        file_cache_percent=10.0,
        db_buffer_pool_mb=4096  # Additional DB cache
    )
    
    result = calculate_memory_requirement(
        processes, 
        SystemOverhead(),
        cache,
        headroom_percent=25.0,
        planning_horizon_years=2
    )
    
    print(f"Total Required: {result['total_required_gb']:.1f} GB")
    print(f"Recommended RAM: {result['recommended_gb']} GB")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

#!/bin/bash
# Hot-Add Memory Detection and Integration (Linux)
# Demonstrates OS-side handling of memory hot-plug
 
# Monitor for memory hot-plug events
echo "Monitoring for memory hot-plug events..."
 
# Initial memory state
INITIAL_MEM=$(grep MemTotal /proc/meminfo | awk '{print $2}')
echo "Initial memory: $((INITIAL_MEM / 1024)) MB"
 
# Watch for ACPI memory events
# In practice, udev rules handle this automatically
 
# After physical installation, new memory appears in sysfs
# List current memory blocks
echo -e "
Current memory blocks:"
ls /sys/devices/system/memory/ | grep memory
 
# Check a specific memory block's state
# (memory blocks can be online or offline)
for mem in /sys/devices/system/memory/memory*/state; do
    block=$(dirname $mem | xargs basename)
    state=$(cat $mem)
    echo "$block: $state"
done
 
# Hot-added memory may initially be 'offline'
# To bring it online:
bring_online_new_memory() {
    for mem in /sys/devices/system/memory/memory*/state; do
        state=$(cat $mem)
        if [ "$state" == "offline" ]; then
            block=$(dirname $mem | xargs basename)
            echo "Bringing $block online..."
            echo online > $mem
            if [ $? -eq 0 ]; then
                echo "  Success: $block is now online"
            else
                echo "  Failed to online $block"
            fi
        fi
    done
}
 
# Verify memory increase
verify_memory_increase() {
    NEW_MEM=$(grep MemTotal /proc/meminfo | awk '{print $2}')
    INCREASE=$((NEW_MEM - INITIAL_MEM))
    echo -e "
Memory verification:"
    echo "Previous: $((INITIAL_MEM / 1024)) MB"
    echo "Current:  $((NEW_MEM / 1024)) MB"
    echo "Added:    $((INCREASE / 1024)) MB"
}
 
# Usage example
# bring_online_new_memory
# verify_memory_increase

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

# Cloud Memory Auto-Scaling Logic
# Demonstrates decision-making for memory-based scaling
 
import boto3
from dataclasses import dataclass
from typing import List, Optional
 
@dataclass
class InstanceType:
    name: str
    vcpus: int
    memory_gb: int
    hourly_cost: float
 
# Memory-optimized instance progression
INSTANCE_PROGRESSION = [
    InstanceType("t3.medium", 2, 4, 0.0416),
    InstanceType("t3.large", 2, 8, 0.0832),
    InstanceType("t3.xlarge", 4, 16, 0.1664),
    InstanceType("r6g.large", 2, 16, 0.1008),
    InstanceType("r6g.xlarge", 4, 32, 0.2016),
    InstanceType("r6g.2xlarge", 8, 64, 0.4032),
    InstanceType("r6g.4xlarge", 16, 128, 0.8064),
]
 
def get_current_memory_metrics(instance_id: str) -> dict:
    """Get memory metrics from CloudWatch."""
    cloudwatch = boto3.client('cloudwatch')
    
    # Custom metrics must be published by agent
    response = cloudwatch.get_metric_statistics(
        Namespace='CWAgent',
        MetricName='mem_used_percent',
        Dimensions=[{'Name': 'InstanceId', 'Value': instance_id}],
        StartTime=datetime.utcnow() - timedelta(minutes=15),
        EndTime=datetime.utcnow(),
        Period=300,
        Statistics=['Average', 'Maximum']
    )
    
    return {
        'avg_percent': response['Datapoints'][-1]['Average'],
        'max_percent': response['Datapoints'][-1]['Maximum'],
    }
 
def recommend_instance_change(
    current_instance: InstanceType,
    memory_metrics: dict,
    scale_up_threshold: float = 85.0,
    scale_down_threshold: float = 40.0
) -> Optional[InstanceType]:
    """Recommend instance type change based on memory usage."""
    
    current_index = next(
        i for i, inst in enumerate(INSTANCE_PROGRESSION)
        if inst.name == current_instance.name
    )
    
    if memory_metrics['avg_percent'] > scale_up_threshold:
        # Need more memory
        if current_index < len(INSTANCE_PROGRESSION) - 1:
            return INSTANCE_PROGRESSION[current_index + 1]
        else:
            print("Already at maximum instance size!")
            return None
    
    elif memory_metrics['avg_percent'] < scale_down_threshold:
        # Can reduce memory
        if current_index > 0:
            candidate = INSTANCE_PROGRESSION[current_index - 1]
            # Ensure we won't immediately need to scale up again
            projected_usage = (
                memory_metrics['avg_percent'] * 
                current_instance.memory_gb / 
                candidate.memory_gb
            )
            if projected_usage < scale_up_threshold * 0.8:
                return candidate
    
    return None  # Current size is appropriate
 
def calculate_scaling_cost_impact(
    current: InstanceType,
    proposed: InstanceType,
    hours_per_month: int = 720
) -> dict:
    """Calculate cost impact of instance change."""
    
    current_monthly = current.hourly_cost * hours_per_month
    proposed_monthly = proposed.hourly_cost * hours_per_month
    difference = proposed_monthly - current_monthly
    
    return {
        'current_monthly': current_monthly,
        'proposed_monthly': proposed_monthly,
        'difference': difference,
        'percent_change': (difference / current_monthly) * 100
    }