Chaos Engineering Tools

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

┌─────────────────────────────────────────────────────────────────────┐
│                    CHAOS MONKEY ARCHITECTURE                         │
├─────────────────────────────────────────────────────────────────────┤
│                                                                      │
│  ┌──────────────┐     ┌─────────────────┐     ┌─────────────────┐  │
│  │   Scheduler  │────▶│   Chaos Monkey  │────▶│  AWS/Spinnaker  │  │
│  │   (Cron)     │     │   Core Engine   │     │     API         │  │
│  └──────────────┘     └────────┬────────┘     └─────────────────┘  │
│                                │                                     │
│                                ▼                                     │
│                    ┌─────────────────────┐                          │
│                    │   Discovery Layer   │                          │
│                    │  (Eureka/Consul)    │                          │
│                    └──────────┬──────────┘                          │
│                               │                                      │
│         ┌─────────────────────┼─────────────────────┐               │
│         ▼                     ▼                     ▼               │
│  ┌─────────────┐      ┌─────────────┐      ┌─────────────┐        │
│  │  App Group  │      │  App Group  │      │  App Group  │        │
│  │  (ASG-A)    │      │  (ASG-B)    │      │  (ASG-C)    │        │
│  ├─────────────┤      ├─────────────┤      ├─────────────┤        │
│  │ Instance 1  │      │ Instance 1  │      │ Instance 1  │        │
│  │ Instance 2  │      │ Instance 2  │      │ Instance 2  │        │
│  │ Instance 3  │◀─ TERMINATION ─────────────┘                      │
│  │ Instance 4  │  (Random Selection)                               │
│  └─────────────┘                                                    │
│                                                                      │
│  ┌─────────────────────────────────────────────────────────────┐   │
│  │                     SAFETY CONTROLS                          │   │
│  │  • Time Windows (Business hours only)                        │   │
│  │  • Opt-out Groups (Critical services)                        │   │
│  │  • Kill Limits (Max terminations per run)                    │   │
│  │  • Deployment Awareness (Skip during deploys)                │   │
│  └─────────────────────────────────────────────────────────────┘   │
└─────────────────────────────────────────────────────────────────────┘

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

// Simplified Chaos Monkey Selection Algorithm
 
function selectVictim():
    // Step 1: Get all eligible applications
    applications = discoveryService.getAllApplications()
    
    // Step 2: Filter out opted-out applications
    eligible_apps = applications.filter(app => 
        !app.hasChaosMonkeyOptOut() &&
        !app.isCurrentlyDeploying() &&
        app.instanceCount() > app.minimumInstanceCount()
    )
    
    // Step 3: Randomly select an application
    selected_app = random.choice(eligible_apps)
    
    // Step 4: Get instances for selected application
    instances = selected_app.getHealthyInstances()
    
    // Step 5: Filter by availability zone balance
    // Avoid terminating if it would create AZ imbalance
    balanced_instances = instances.filter(instance =>
        !wouldCauseAZImbalance(selected_app, instance)
    )
    
    // Step 6: Apply additional safety filters
    safe_instances = balanced_instances.filter(instance =>
        !instance.hasActiveConnections(threshold: HIGH) &&
        !instance.wasRecentlyLaunched(minutes: 10) &&
        !instance.isLastHealthyInstance()
    )
    
    // Step 7: Final random selection
    if safe_instances.isEmpty():
        return null  // No safe target this run
    
    return random.choice(safe_instances)
 
function executeTermination(victim):
    // Log intent before action
    auditLog.record({
        action: "TERMINATION_INITIATED",
        instance: victim.id,
        application: victim.app,
        timestamp: now(),
        reason: "Chaos Monkey scheduled run"
    })
    
    // Execute termination via cloud API
    cloudProvider.terminateInstance(victim.id)
    
    // Log completion
    auditLog.record({
        action: "TERMINATION_COMPLETED",
        instance: victim.id,
        timestamp: now()
    })
    
    // Notify monitoring systems
    monitoring.recordChaosEvent(victim)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

// Example: Chaos Monkey deployment awareness
 
interface DeploymentStatus {
  application: string;
  stage: 'ROLLING' | 'CANARY' | 'COMPLETE' | 'FAILED';
  startTime: Date;
  instances: {
    new: string[];
    old: string[];
  };
}
 
class ChaosMonkeyDeploymentGuard {
  private deploymentTracker: SpinnakerClient;
  
  async shouldAllowChaos(application: string): Promise<boolean> {
    const deployment = await this.deploymentTracker.getCurrentDeployment(application);
    
    if (!deployment) {
      return true; // No active deployment, chaos allowed
    }
    
    // Never chaos during active rollout
    if (deployment.stage === 'ROLLING') {
      console.log(`Skipping chaos for ${application}: deployment in progress`);
      return false;
    }
    
    // Never chaos during canary analysis
    if (deployment.stage === 'CANARY') {
      console.log(`Skipping chaos for ${application}: canary in progress`);
      return false;
    }
    
    // Allow chaos after deployment completes, but only after stabilization period
    if (deployment.stage === 'COMPLETE') {
      const stabilizationPeriod = 30 * 60 * 1000; // 30 minutes
      const timeSinceComplete = Date.now() - deployment.startTime.getTime();
      
      if (timeSinceComplete < stabilizationPeriod) {
        console.log(`Skipping chaos for ${application}: stabilization period`);
        return false;
      }
    }
    
    return true;
  }
  
  // When Chaos Monkey considers terminating a specific instance
  async shouldTerminateInstance(instanceId: string): Promise<boolean> {
    const deployment = await this.deploymentTracker.findDeploymentWithInstance(instanceId);
    
    if (!deployment) {
      return true;
    }
    
    // Only terminate old instances during rollout, not new ones
    // This lets us test if the old version survives reduced capacity
    if (deployment.instances.new.includes(instanceId)) {
      console.log(`Protecting new instance ${instanceId} during deployment`);
      return false;
    }
    
    return true;
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

# Example Chaos Monkey Configuration
# Based on Spinnaker/Chaos Monkey integration
 
chaosMonkey:
  # Global enable/disable
  enabled: true
  
  # Schedule configuration
  schedule:
    # When chaos can run
    timezone: "America/Los_Angeles"
    startHour: 9
    endHour: 15
    daysOfWeek:
      - MONDAY
      - TUESDAY
      - WEDNESDAY
      - THURSDAY
      - FRIDAY
    
    # How often to run
    frequency: HOURLY  # Options: HOURLY, DAILY, WEEKLY
    
  # Safety controls
  safety:
    # Minimum instances before chaos can terminate
    minInstancesPerASG: 2
    
    # Maximum percentage of instances to terminate per group
    maxTerminationPercentage: 50
    
    # Cooldown after termination before next is allowed
    cooldownMinutes: 60
    
    # Opt-out mechanism
    optOutTagKey: "chaosmonkey.optout"
    optOutTagValue: "true"
    
    # Always protect these application patterns
    protectedApplicationPatterns:
      - ".*-critical$"
      - "^auth-.*"
      - "^payment-.*"
    
  # Cloud provider configuration
  aws:
    region: "us-west-2"
    # Use IAM role or explicit credentials
    useInstanceProfile: true
    
  # Notification configuration
  notifications:
    slack:
      enabled: true
      channel: "#chaos-events"
      webhookUrl: "${SLACK_WEBHOOK_URL}"
      
    email:
      enabled: true
      recipients:
        - "sre-team@company.com"
      onlyOnError: true
      
  # Audit logging
  audit:
    destination: "cloudwatch"
    logGroup: "/chaos-monkey/audit"
    retentionDays: 90

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

// Example: Chaos Monkey metrics collection and dashboard integration
 
interface ChaosEvent {
  eventId: string;
  timestamp: Date;
  application: string;
  instanceId: string;
  region: string;
  availabilityZone: string;
}
 
interface ChaosMetrics {
  // Time measurements
  timeToDetection: number;      // Seconds until monitoring detected instance loss
  timeToReplacement: number;    // Seconds until new instance launched
  timeToHealthy: number;        // Seconds until new instance serving traffic
  totalRecoveryTime: number;    // End-to-end recovery time
  
  // Impact measurements
  errorRateBefore: number;      // Baseline error rate (15min before)
  errorRateDuring: number;      // Error rate during chaos window
  errorRateAfter: number;       // Error rate post-recovery
  
  latencyP50Before: number;
  latencyP50During: number;
  latencyP95Before: number;
  latencyP95During: number;
  
  // Capacity measurements
  instanceCountBefore: number;
  instanceCountMinimum: number; // Lowest point during chaos
  instanceCountAfter: number;
  
  // Traffic measurements
  requestRateBefore: number;
  requestRateDuring: number;
  failedRequestsDuring: number;
}
 
class ChaosMetricsCollector {
  private prometheus: PrometheusClient;
  
  async collectMetrics(event: ChaosEvent): Promise<ChaosMetrics> {
    const windowStart = new Date(event.timestamp.getTime() - 15 * 60 * 1000);
    const windowEnd = new Date(event.timestamp.getTime() + 30 * 60 * 1000);
    
    // Query metrics around chaos event
    const [
      errorRates,
      latencies,
      instanceCounts,
      requestRates
    ] = await Promise.all([
      this.prometheus.query(`
        rate(http_requests_total{app="${event.application}", status=~"5.."}[1m])
        / rate(http_requests_total{app="${event.application}"}[1m])
      `, windowStart, windowEnd),
      
      this.prometheus.query(`
        histogram_quantile(0.95, 
          rate(http_request_duration_seconds_bucket{app="${event.application}"}[1m])
        )
      `, windowStart, windowEnd),
      
      this.prometheus.query(`
        count(up{app="${event.application}"} == 1)
      `, windowStart, windowEnd),
      
      this.prometheus.query(`
        rate(http_requests_total{app="${event.application}"}[1m])
      `, windowStart, windowEnd)
    ]);
    
    // Calculate recovery timing
    const recoveryTimestamp = await this.findRecoveryPoint(event, instanceCounts);
    
    return {
      timeToDetection: this.calculateDetectionTime(event, instanceCounts),
      timeToReplacement: this.calculateReplacementTime(event, instanceCounts),
      timeToHealthy: this.calculateHealthyTime(event, instanceCounts),
      totalRecoveryTime: (recoveryTimestamp.getTime() - event.timestamp.getTime()) / 1000,
      
      errorRateBefore: this.average(errorRates, windowStart, event.timestamp),
      errorRateDuring: this.max(errorRates, event.timestamp, recoveryTimestamp),
      errorRateAfter: this.average(errorRates, recoveryTimestamp, windowEnd),
      
      latencyP50Before: this.percentile(latencies, 50, windowStart, event.timestamp),
      latencyP50During: this.percentile(latencies, 50, event.timestamp, recoveryTimestamp),
      latencyP95Before: this.percentile(latencies, 95, windowStart, event.timestamp),
      latencyP95During: this.percentile(latencies, 95, event.timestamp, recoveryTimestamp),
      
      instanceCountBefore: instanceCounts.at(event.timestamp),
      instanceCountMinimum: this.min(instanceCounts, event.timestamp, recoveryTimestamp),
      instanceCountAfter: instanceCounts.at(windowEnd),
      
      requestRateBefore: this.average(requestRates, windowStart, event.timestamp),
      requestRateDuring: this.average(requestRates, event.timestamp, recoveryTimestamp),
      failedRequestsDuring: this.calculateFailedRequests(errorRates, requestRates,
                                                         event.timestamp, recoveryTimestamp)
    };
  }
  
  // Calculate chaos health score: 0-100 (higher is better)
  calculateHealthScore(metrics: ChaosMetrics): number {
    let score = 100;
    
    // Penalize slow recovery (target: < 300 seconds)
    if (metrics.totalRecoveryTime > 300) {
      score -= Math.min(30, (metrics.totalRecoveryTime - 300) / 10);
    }
    
    // Penalize error rate increase
    const errorRateIncrease = metrics.errorRateDuring - metrics.errorRateBefore;
    if (errorRateIncrease > 0.001) {  // > 0.1% increase
      score -= Math.min(40, errorRateIncrease * 4000);
    }
    
    // Penalize latency spike
    const latencyIncrease = metrics.latencyP95During / metrics.latencyP95Before;
    if (latencyIncrease > 1.2) {  // > 20% increase
      score -= Math.min(20, (latencyIncrease - 1) * 20);
    }
    
    // Penalize failed requests
    if (metrics.failedRequestsDuring > 0) {
      score -= Math.min(10, metrics.failedRequestsDuring / 10);
    }
    
    return Math.max(0, Math.round(score));
  }
}