1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193

# Chaos Automation Platform - Kubernetes Deployment
apiVersion: v1
kind: Namespace
metadata:
  name: chaos-platform
---
# Experiment Registry (stores experiment definitions)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: experiment-registry
  namespace: chaos-platform
spec:
  replicas: 2
  selector:
    matchLabels:
      app: experiment-registry
  template:
    metadata:
      labels:
        app: experiment-registry
    spec:
      containers:
        - name: registry
          image: chaos-platform/experiment-registry:v1.2
          ports:
            - containerPort: 8080
          env:
            - name: DATABASE_URL
              valueFrom:
                secretKeyRef:
                  name: chaos-db
                  key: url
          volumeMounts:
            - name: experiments
              mountPath: /experiments
      volumes:
        - name: experiments
          configMap:
            name: experiment-definitions
---
# Scheduler (determines when experiments run)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: chaos-scheduler
  namespace: chaos-platform
spec:
  replicas: 1  # Only one scheduler to avoid duplicates
  selector:
    matchLabels:
      app: chaos-scheduler
  template:
    spec:
      containers:
        - name: scheduler
          image: chaos-platform/scheduler:v1.2
          env:
            - name: REGISTRY_URL
              value: "http://experiment-registry:8080"
            - name: ORCHESTRATOR_URL
              value: "http://chaos-orchestrator:8080"
            - name: MAINTENANCE_CALENDAR_URL
              valueFrom:
                configMapKeyRef:
                  name: chaos-config
                  key: maintenance-calendar-url
---
# Orchestrator (executes experiments)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: chaos-orchestrator
  namespace: chaos-platform
spec:
  replicas: 3
  selector:
    matchLabels:
      app: chaos-orchestrator
  template:
    spec:
      serviceAccountName: chaos-admin  # Needs permissions to inject chaos
      containers:
        - name: orchestrator
          image: chaos-platform/orchestrator:v1.2
          env:
            - name: INJECTION_ENGINE
              value: "litmus"  # or "gremlin", "chaos-mesh", etc.
            - name: METRICS_URL
              value: "http://prometheus:9090"
            - name: ABORT_CONTROLLER_URL
              value: "http://abort-controller:8080"
---
# Abort Controller (safety mechanism)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: abort-controller
  namespace: chaos-platform
spec:
  replicas: 3  # High availability critical
  selector:
    matchLabels:
      app: abort-controller
  template:
    spec:
      containers:
        - name: abort-controller
          image: chaos-platform/abort-controller:v1.2
          env:
            - name: METRICS_URL
              value: "http://prometheus:9090"
            - name: SLACK_WEBHOOK
              valueFrom:
                secretKeyRef:
                  name: chaos-alerts
                  key: slack-webhook
          # Liveness probe - abort controller must itself be healthy
          livenessProbe:
            httpGet:
              path: /health
              port: 8080
            initialDelaySeconds: 5
            periodSeconds: 5
            failureThreshold: 2  # Fast failure detection
---
# Experiment CRD (Custom Resource Definition)
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: chaosexperiments.chaos.platform.io
spec:
  group: chaos.platform.io
  versions:
    - name: v1
      served: true
      storage: true
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              properties:
                name:
                  type: string
                hypothesis:
                  type: string
                injection:
                  type: object
                  properties:
                    type:
                      type: string
                      enum: [latency, error, kill, resource]
                    target:
                      type: string
                    severity:
                      type: string
                    duration:
                      type: string
                blastRadius:
                  type: object
                  properties:
                    trafficPercentage:
                      type: integer
                      minimum: 0
                      maximum: 100
                    regions:
                      type: array
                      items:
                        type: string
                schedule:
                  type: string  # cron expression
                abortConditions:
                  type: array
                  items:
                    type: object
                    properties:
                      metric:
                        type: string
                      threshold:
                        type: number
                      operator:
                        type: string
                      duration:
                        type: string
  scope: Namespaced
  names:
    plural: chaosexperiments
    singular: chaosexperiment
    kind: ChaosExperiment
    shortNames:
      - chexp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

# GitHub Actions workflow with chaos integration
name: Deploy with Chaos Validation
 
on:
  push:
    branches: [main]
 
jobs:
  build-and-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Build
        run: npm run build
      - name: Unit Tests
        run: npm test
      - name: Integration Tests
        run: npm run test:integration
 
  deploy-staging:
    needs: build-and-test
    runs-on: ubuntu-latest
    steps:
      - name: Deploy to Staging
        run: ./deploy.sh staging
      - name: Wait for Stabilization
        run: sleep 120  # Wait 2 minutes for deployment to stabilize
 
  chaos-staging:
    needs: deploy-staging
    runs-on: ubuntu-latest
    steps:
      - name: Install Chaos CLI
        run: |
          curl -LO https://chaos-platform.io/cli/install.sh
          bash install.sh
          
      - name: Run Instance Termination Experiment
        id: chaos-instance
        run: |
          chaos-cli run \
            --experiment instance-termination \
            --environment staging \
            --wait-for-completion \
            --timeout 10m
        continue-on-error: true
        
      - name: Run Latency Injection Experiment
        id: chaos-latency
        run: |
          chaos-cli run \
            --experiment api-latency-500ms \
            --environment staging \
            --wait-for-completion \
            --timeout 10m
        continue-on-error: true
        
      - name: Run Database Failover Experiment
        id: chaos-db
        run: |
          chaos-cli run \
            --experiment db-replica-failure \
            --environment staging \
            --wait-for-completion \
            --timeout 10m
        continue-on-error: true
      
      - name: Evaluate Chaos Results
        run: |
          # Fail the pipeline if any experiment failed
          if [ "${{ steps.chaos-instance.outcome }}" == "failure" ] || \
             [ "${{ steps.chaos-latency.outcome }}" == "failure" ] || \
             [ "${{ steps.chaos-db.outcome }}" == "failure" ]; then
            echo "❌ Chaos experiments failed - blocking production deployment"
            exit 1
          fi
          echo "✅ All chaos experiments passed"
 
  deploy-canary:
    needs: chaos-staging
    runs-on: ubuntu-latest
    steps:
      - name: Deploy Canary (5% traffic)
        run: ./deploy.sh production --canary 5
      - name: Wait for Canary Stabilization
        run: sleep 300  # 5 minutes
 
  chaos-canary:
    needs: deploy-canary
    runs-on: ubuntu-latest
    steps:
      - name: Chaos on Canary
        run: |
          chaos-cli run \
            --experiment canary-resilience-suite \
            --environment production \
            --target canary \
            --blast-radius 100 \  # 100% of canary (which is 5% of prod)
            --wait-for-completion
            
      - name: Validate Canary Health
        run: |
          # Check canary metrics are within SLO
          ./validate-canary.sh
 
  deploy-production:
    needs: chaos-canary
    runs-on: ubuntu-latest
    steps:
      - name: Full Production Deployment
        run: ./deploy.sh production --full
      - name: Monitor Deployment
        run: ./monitor-deploy.sh
 
  chaos-post-deploy:
    needs: deploy-production
    runs-on: ubuntu-latest
    steps:
      - name: Notify Team
        run: |
          slack-notify "Deployment complete. Running post-deploy chaos validation..."
          
      - name: Post-Deploy Chaos Suite
        run: |
          chaos-cli run \
            --experiment post-deploy-validation \
            --environment production \
            --blast-radius 1 \  # 1% of production
            --wait-for-completion
            
      - name: Report Results
        if: always()
        run: |
          chaos-cli report --format markdown > chaos-report.md
          slack-notify -f chaos-report.md

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189

from dataclasses import dataclass
from typing import List, Optional, Set
from enum import Enum
import random
from datetime import datetime, timedelta
import pytz
 
class ScheduleFrequency(Enum):
    HOURLY = "hourly"
    DAILY = "daily"
    WEEKLY = "weekly"
    MONTHLY = "monthly"
 
class TrafficCorrelation(Enum):
    HIGH_TRAFFIC = "high_traffic"
    LOW_TRAFFIC = "low_traffic"
    ANY = "any"
 
@dataclass
class TimeWindow:
    """Defines a valid time window for experiment execution."""
    start_hour: int  # 0-23
    end_hour: int
    days_of_week: List[int]  # 0=Monday, 6=Sunday
    timezone: str = "UTC"
    
    def is_current_time_valid(self) -> bool:
        """Check if current time falls within this window."""
        tz = pytz.timezone(self.timezone)
        now = datetime.now(tz)
        
        if now.weekday() not in self.days_of_week:
            return False
        if not (self.start_hour <= now.hour < self.end_hour):
            return False
        return True
 
@dataclass
class ExperimentSchedule:
    """Complete scheduling configuration for an experiment."""
    experiment_id: str
    frequency: ScheduleFrequency
    valid_windows: List[TimeWindow]
    traffic_correlation: TrafficCorrelation
    mutex_groups: Set[str]  # Can't run with other experiments in same group
    priority: int  # Higher priority experiments run first
    min_interval_hours: int  # Minimum time between runs
    jitter_minutes: int = 30  # Randomize execution time by +/- this amount
    
@dataclass
class MaintenanceWindow:
    """Periods when no experiments should run."""
    start: datetime
    end: datetime
    reason: str
 
class ChaosScheduler:
    """
    Intelligent scheduling for chaos experiments.
    Handles conflicts, maintenance windows, and traffic patterns.
    """
    
    def __init__(self):
        self.schedules: List[ExperimentSchedule] = []
        self.maintenance_windows: List[MaintenanceWindow] = []
        self.last_runs: dict = {}  # experiment_id -> last run timestamp
        self.active_mutex_groups: Set[str] = set()
    
    def add_schedule(self, schedule: ExperimentSchedule):
        self.schedules.append(schedule)
    
    def add_maintenance_window(self, window: MaintenanceWindow):
        self.maintenance_windows.append(window)
    
    def get_due_experiments(self) -> List[str]:
        """
        Return list of experiment IDs that should run now.
        Respects priorities, mutex groups, and scheduling constraints.
        """
        now = datetime.utcnow()
        
        # Check if we're in a maintenance window
        if self._in_maintenance_window(now):
            return []
        
        candidates = []
        
        for schedule in self.schedules:
            if self._should_run(schedule, now):
                candidates.append(schedule)
        
        # Sort by priority
        candidates.sort(key=lambda s: s.priority, reverse=True)
        
        # Select experiments respecting mutex groups
        selected = []
        blocked_groups: Set[str] = set(self.active_mutex_groups)
        
        for schedule in candidates:
            if schedule.mutex_groups & blocked_groups:
                continue  # Would conflict with running or selected experiment
            
            selected.append(schedule.experiment_id)
            blocked_groups.update(schedule.mutex_groups)
        
        return selected
    
    def _should_run(self, schedule: ExperimentSchedule, now: datetime) -> bool:
        """Check all conditions for whether an experiment should run."""
        
        # Check if within valid time window
        if not any(w.is_current_time_valid() for w in schedule.valid_windows):
            return False
        
        # Check minimum interval since last run
        last_run = self.last_runs.get(schedule.experiment_id)
        if last_run:
            hours_since = (now - last_run).total_seconds() / 3600
            if hours_since < schedule.min_interval_hours:
                return False
        
        # Check if due based on frequency
        if not self._is_due(schedule, now):
            return False
        
        # Check traffic correlation
        if not self._matches_traffic_pattern(schedule.traffic_correlation):
            return False
        
        return True
    
    def _is_due(self, schedule: ExperimentSchedule, now: datetime) -> bool:
        """Check if experiment is due based on frequency."""
        last_run = self.last_runs.get(schedule.experiment_id)
        
        if last_run is None:
            return True  # Never run before
        
        hours_since = (now - last_run).total_seconds() / 3600
        
        required_hours = {
            ScheduleFrequency.HOURLY: 1,
            ScheduleFrequency.DAILY: 24,
            ScheduleFrequency.WEEKLY: 168,
            ScheduleFrequency.MONTHLY: 720,
        }
        
        # Add jitter
        jitter = random.randint(-schedule.jitter_minutes, schedule.jitter_minutes) / 60
        
        return hours_since >= (required_hours[schedule.frequency] + jitter)
    
    def _in_maintenance_window(self, now: datetime) -> bool:
        for window in self.maintenance_windows:
            if window.start <= now <= window.end:
                return True
        return False
    
    def _matches_traffic_pattern(self, correlation: TrafficCorrelation) -> bool:
        """Check if current traffic matches desired pattern."""
        if correlation == TrafficCorrelation.ANY:
            return True
        
        # In practice, query your metrics system
        current_traffic = self._get_current_traffic_level()
        
        if correlation == TrafficCorrelation.HIGH_TRAFFIC:
            return current_traffic > 0.7  # Above 70% of peak
        elif correlation == TrafficCorrelation.LOW_TRAFFIC:
            return current_traffic < 0.3  # Below 30% of peak
        
        return True
    
    def _get_current_traffic_level(self) -> float:
        """Query current traffic relative to typical peak."""
        # Stub - in practice, query Prometheus/etc.
        return 0.5
    
    def mark_complete(self, experiment_id: str):
        """Record that an experiment has completed."""
        self.last_runs[experiment_id] = datetime.utcnow()
    
    def acquire_mutex(self, mutex_groups: Set[str]):
        """Mark mutex groups as active."""
        self.active_mutex_groups.update(mutex_groups)
    
    def release_mutex(self, mutex_groups: Set[str]):
        """Release mutex groups."""
        self.active_mutex_groups -= mutex_groups

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239

interface ExperimentMetrics {
  before: MetricSample[];
  during: MetricSample[];
  after: MetricSample[];
}
 
interface MetricSample {
  name: string;
  timestamp: Date;
  value: number;
}
 
interface AnalysisResult {
  experimentId: string;
  hypothesisConfirmed: boolean;
  confidence: number;  // 0-1
  findings: Finding[];
  recommendations: string[];
  sloViolations: SLOViolation[];
  unexpectedImpacts: UnexpectedImpact[];
}
 
interface Finding {
  severity: 'info' | 'warning' | 'critical';
  metric: string;
  description: string;
  evidence: {
    baseline: number;
    observed: number;
    deviation: number;
  };
}
 
class ExperimentAnalyzer {
  private readonly sloDefinitions: Map<string, SLODefinition>;
  private readonly historicalResults: ExperimentResult[];
  
  constructor(slos: SLODefinition[], history: ExperimentResult[]) {
    this.sloDefinitions = new Map(slos.map(s => [s.name, s]));
    this.historicalResults = history;
  }
  
  /**
   * Perform comprehensive analysis of experiment results.
   */
  analyze(
    experimentId: string,
    hypothesis: Hypothesis,
    metrics: ExperimentMetrics
  ): AnalysisResult {
    
    const findings: Finding[] = [];
    const recommendations: string[] = [];
    
    // 1. Calculate baseline statistics
    const baseline = this.calculateBaseline(metrics.before);
    
    // 2. Compare during-experiment metrics to baseline
    const comparison = this.compareToBaseline(metrics.during, baseline);
    
    // 3. Check each hypothesis criterion
    const hypothesisResult = this.evaluateHypothesis(hypothesis, comparison);
    
    // 4. Check SLO compliance
    const sloViolations = this.checkSLOCompliance(metrics.during);
    
    // 5. Detect unexpected impacts
    const unexpectedImpacts = this.detectUnexpectedImpacts(
      metrics.during,
      hypothesis.expectedImpactedMetrics
    );
    
    // 6. Compare to historical results
    const trend = this.compareTrend(experimentId, hypothesisResult);
    
    // 7. Generate findings
    for (const [metric, stats] of Object.entries(comparison)) {
      if (stats.deviationPercent > 10) {
        findings.push({
          severity: stats.deviationPercent > 50 ? 'critical' : 'warning',
          metric,
          description: `${metric} deviated ${stats.deviationPercent.toFixed(1)}% from baseline`,
          evidence: {
            baseline: stats.baselineMean,
            observed: stats.observedMean,
            deviation: stats.deviationPercent
          }
        });
      }
    }
    
    // 8. Generate recommendations
    if (sloViolations.length > 0) {
      recommendations.push(
        'SLO violations detected. Review resilience mechanisms for affected services.'
      );
    }
    
    if (unexpectedImpacts.length > 0) {
      recommendations.push(
        'Unexpected services were impacted. Update dependency maps and consider blast radius.'
      );
    }
    
    if (trend === 'degrading') {
      recommendations.push(
        'Resilience for this scenario is degrading over time. Investigate recent changes.'
      );
    }
    
    return {
      experimentId,
      hypothesisConfirmed: hypothesisResult.confirmed,
      confidence: hypothesisResult.confidence,
      findings,
      recommendations,
      sloViolations,
      unexpectedImpacts
    };
  }
  
  private calculateBaseline(samples: MetricSample[]): Map<string, BaselineStats> {
    const grouped = this.groupByMetric(samples);
    const result = new Map<string, BaselineStats>();
    
    for (const [metric, values] of grouped) {
      const sorted = values.sort((a, b) => a - b);
      result.set(metric, {
        mean: this.mean(values),
        stdDev: this.stdDev(values),
        p50: this.percentile(sorted, 50),
        p95: this.percentile(sorted, 95),
        p99: this.percentile(sorted, 99),
        min: sorted[0],
        max: sorted[sorted.length - 1]
      });
    }
    
    return result;
  }
  
  private compareToBaseline(
    samples: MetricSample[],
    baseline: Map<string, BaselineStats>
  ): Record<string, ComparisonResult> {
    const grouped = this.groupByMetric(samples);
    const result: Record<string, ComparisonResult> = {};
    
    for (const [metric, values] of grouped) {
      const base = baseline.get(metric);
      if (!base) continue;
      
      const observedMean = this.mean(values);
      const deviation = Math.abs(observedMean - base.mean);
      const deviationPercent = (deviation / base.mean) * 100;
      
      // Statistical significance using z-score
      const zScore = base.stdDev > 0 ? deviation / base.stdDev : 0;
      const isSignificant = zScore > 2;  // 95% confidence
      
      result[metric] = {
        baselineMean: base.mean,
        observedMean,
        deviationPercent,
        zScore,
        isSignificant
      };
    }
    
    return result;
  }
  
  private evaluateHypothesis(
    hypothesis: Hypothesis,
    comparison: Record<string, ComparisonResult>
  ): { confirmed: boolean; confidence: number } {
    let criteriamet = 0;
    let totalCriteria = hypothesis.criteria.length;
    
    for (const criterion of hypothesis.criteria) {
      const metricResult = comparison[criterion.metric];
      if (!metricResult) continue;
      
      // Check if criterion is satisfied
      switch (criterion.condition) {
        case 'stays_below':
          if (metricResult.observedMean < criterion.threshold) {
            criteriamet++;
          }
          break;
        case 'stays_above':
          if (metricResult.observedMean > criterion.threshold) {
            criteriamet++;
          }
          break;
        case 'deviation_under':
          if (metricResult.deviationPercent < criterion.threshold) {
            criteriamet++;
          }
          break;
      }
    }
    
    const confidence = totalCriteria > 0 ? criteriamet / totalCriteria : 0;
    
    return {
      confirmed: confidence >= 0.8,  // 80% of criteria must pass
      confidence
    };
  }
  
  // Helper methods
  private groupByMetric(samples: MetricSample[]): Map<string, number[]> {
    const grouped = new Map<string, number[]>();
    for (const sample of samples) {
      const values = grouped.get(sample.name) || [];
      values.push(sample.value);
      grouped.set(sample.name, values);
    }
    return grouped;
  }
  
  private mean(values: number[]): number {
    return values.reduce((a, b) => a + b, 0) / values.length;
  }
  
  private stdDev(values: number[]): number {
    const avg = this.mean(values);
    const squareDiffs = values.map(v => Math.pow(v - avg, 2));
    return Math.sqrt(this.mean(squareDiffs));
  }
  
  private percentile(sorted: number[], p: number): number {
    const idx = Math.ceil((p / 100) * sorted.length) - 1;
    return sorted[Math.max(0, idx)];
  }
}
 
export { ExperimentAnalyzer, AnalysisResult };