Chaos Engineering Tools

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

┌─────────────────────────────────────────────────────────────────────┐
│                    KUBERNETES FAILURE SURFACE                        │
├─────────────────────────────────────────────────────────────────────┤
│                                                                      │
│  Layer 1: APPLICATION                                               │
│  ┌─────────────────────────────────────────────────────────────┐   │
│  │  Container crashes, OOM kills, application errors           │   │
│  │  Health check failures, readiness probe issues              │   │
│  └─────────────────────────────────────────────────────────────┘   │
│                                                                      │
│  Layer 2: POD                                                        │
│  ┌─────────────────────────────────────────────────────────────┐   │
│  │  Pod eviction, pod preemption, init container failures      │   │
│  │  Sidecar issues, volume mount failures                      │   │
│  └─────────────────────────────────────────────────────────────┘   │
│                                                                      │
│  Layer 3: WORKLOAD CONTROLLERS                                      │
│  ┌─────────────────────────────────────────────────────────────┐   │
│  │  Deployment rollout failures, ReplicaSet issues             │   │
│  │  StatefulSet ordering violations, DaemonSet gaps            │   │
│  └─────────────────────────────────────────────────────────────┘   │
│                                                                      │
│  Layer 4: SERVICES & NETWORKING                                     │
│  ┌─────────────────────────────────────────────────────────────┐   │
│  │  Service routing failures, Ingress misconfigurations        │   │
│  │  NetworkPolicy blocks, DNS resolution failures              │   │
│  └─────────────────────────────────────────────────────────────┘   │
│                                                                      │
│  Layer 5: CLUSTER INFRASTRUCTURE                                    │
│  ┌─────────────────────────────────────────────────────────────┐   │
│  │  Node failures, etcd issues, API server unavailability      │   │
│  │  Scheduler failures, controller-manager issues              │   │
│  └─────────────────────────────────────────────────────────────┘   │
│                                                                      │
│  Layer 6: CLOUD PROVIDER                                            │
│  ┌─────────────────────────────────────────────────────────────┐   │
│  │  Load balancer issues, PersistentVolume provisioning        │   │
│  │  Cloud controller failures, CSI driver problems             │   │
│  └─────────────────────────────────────────────────────────────┘   │
└─────────────────────────────────────────────────────────────────────┘

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

┌─────────────────────────────────────────────────────────────────────┐
│                     LITMUSCHAOS ARCHITECTURE                         │
├─────────────────────────────────────────────────────────────────────┤
│                                                                      │
│  ┌──────────────────────────────────────────────────────────────┐  │
│  │                    LITMUS PORTAL (UI)                         │  │
│  │  ┌────────────┐  ┌────────────┐  ┌─────────────────────────┐ │  │
│  │  │ Experiment │  │  Workflow  │  │   Analytics &           │ │  │
│  │  │  Designer  │  │  Builder   │  │   Observability         │ │  │
│  │  └────────────┘  └────────────┘  └─────────────────────────┘ │  │
│  └─────────────────────────┬────────────────────────────────────┘  │
│                            │                                        │
│  ┌─────────────────────────▼────────────────────────────────────┐  │
│  │                   CHAOS CENTER (Backend)                      │  │
│  │  ┌────────────┐  ┌────────────┐  ┌─────────────────────────┐ │  │
│  │  │  GraphQL   │  │  Mongo DB  │  │   Authentication &      │ │  │
│  │  │    API     │  │  (State)   │  │   Authorization         │ │  │
│  │  └────────────┘  └────────────┘  └─────────────────────────┘ │  │
│  └─────────────────────────┬────────────────────────────────────┘  │
│                            │                                        │
│  ════════════════════════════════════════════════════════════════  │
│                         CLUSTER BOUNDARY                            │
│  ════════════════════════════════════════════════════════════════  │
│                            │                                        │
│  ┌─────────────────────────▼────────────────────────────────────┐  │
│  │              SUBSCRIBER (Cluster Agent)                       │  │
│  │  • Connects cluster to Chaos Center                          │  │
│  │  • Receives workflow execution requests                      │  │
│  │  • Reports experiment results back                           │  │
│  └─────────────────────────┬────────────────────────────────────┘  │
│                            │                                        │
│  ┌─────────────────────────▼────────────────────────────────────┐  │
│  │              CHAOS OPERATOR (Controller)                      │  │
│  │  • Watches ChaosEngine CRDs                                  │  │
│  │  • Orchestrates chaos experiments                            │  │
│  │  • Manages experiment lifecycle                              │  │
│  └────────┬─────────────────────────────────────┬───────────────┘  │
│           │                                     │                   │
│  ┌────────▼────────┐                 ┌──────────▼──────────────┐   │
│  │  CHAOS RUNNER   │                 │  CHAOS EXPORTER         │   │
│  │  (Experiment    │                 │  (Prometheus metrics)   │   │
│  │   Executor)     │                 │                         │   │
│  └────────┬────────┘                 └─────────────────────────┘   │
│           │                                                         │
│  ┌────────▼────────────────────────────────────────────────────┐   │
│  │              TARGET APPLICATION / INFRASTRUCTURE             │   │
│  │  Pods, Deployments, StatefulSets, Nodes, etc.               │   │
│  └──────────────────────────────────────────────────────────────┘   │
└─────────────────────────────────────────────────────────────────────┘

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

# Example: Pod Delete Chaos Experiment
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosExperiment
metadata:
  name: pod-delete
  namespace: litmus
spec:
  definition:
    # Scope and permissions
    scope: Namespaced
    permissions:
      - apiGroups: [""]
        resources: ["pods"]
        verbs: ["create", "delete", "get", "list", "patch", "update"]
      - apiGroups: ["apps"]
        resources: ["deployments", "replicasets"]
        verbs: ["get", "list"]
      - apiGroups: ["litmuschaos.io"]
        resources: ["chaosengines", "chaosexperiments", "chaosresults"]
        verbs: ["create", "get", "list", "patch", "update"]
    
    # Experiment image
    image: litmuschaos/go-runner:latest
    imagePullPolicy: Always
    
    # Arguments and environment
    args:
      - -c
      - ./experiments -name pod-delete
    command:
      - /bin/bash
    
    env:
      - name: TOTAL_CHAOS_DURATION
        value: "30"
      - name: CHAOS_INTERVAL
        value: "10"
      - name: FORCE
        value: "false"
      - name: TARGET_PODS
        value: ""  # Empty means random selection
      - name: PODS_AFFECTED_PERC
        value: "50"  # Affect 50% of matching pods
      - name: RAMP_TIME
        value: "0"
      - name: SEQUENCE
        value: "parallel"
 
---
# ChaosEngine binds experiment to target application
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
  name: nginx-pod-delete-chaos
  namespace: production
spec:
  # Enable engine
  engineState: active
  
  # Target application
  appinfo:
    appns: production
    applabel: app=nginx
    appkind: deployment
  
  # Experiment selection
  experiments:
    - name: pod-delete
      spec:
        components:
          env:
            - name: TOTAL_CHAOS_DURATION
              value: "60"
            - name: CHAOS_INTERVAL
              value: "15"
            - name: FORCE
              value: "true"
            - name: PODS_AFFECTED_PERC
              value: "30"
        
        # Probes define success criteria
        probe:
          - name: healthcheck
            type: httpProbe
            mode: Continuous
            runProperties:
              probeTimeout: 5
              retry: 1
              interval: 5
            httpProbe/inputs:
              url: http://nginx-service:80/health
              method:
                get:
                  criteria: ==
                  responseCode: "200"
  
  # Cleanup after experiment
  annotationCheck: "true"
  chaosServiceAccount: pod-delete-sa

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

# Example: Custom ChaosHub Experiment for Internal Payment Service
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosExperiment
metadata:
  name: payment-service-latency
  namespace: litmus
  labels:
    chaoshub: internal
    app.kubernetes.io/category: payment
    app.kubernetes.io/domain: fintech
spec:
  definition:
    scope: Namespaced
    
    permissions:
      - apiGroups: [""]
        resources: ["pods", "services"]
        verbs: ["get", "list"]
      - apiGroups: ["apps"]
        resources: ["deployments"]
        verbs: ["get", "list"]
      - apiGroups: ["litmuschaos.io"]
        resources: ["chaosengines", "chaosresults"]
        verbs: ["create", "get", "list", "patch", "update"]
    
    image: internal-registry/chaos-experiments:payment-v1.2.0
    
    # Custom experiment logic
    args:
      - -c
      - |
        # Inject latency into payment service dependencies
        ./inject-latency \
          --target-service payment-gateway \
          --upstream-latency-ms ${UPSTREAM_LATENCY_MS} \
          --downstream-latency-ms ${DOWNSTREAM_LATENCY_MS} \
          --duration ${CHAOS_DURATION}
    command:
      - /bin/bash
    
    env:
      # Payment-specific parameters
      - name: UPSTREAM_LATENCY_MS
        value: "500"
      - name: DOWNSTREAM_LATENCY_MS
        value: "200"
      - name: CHAOS_DURATION
        value: "120"
      
      # Safety parameters
      - name: ABORT_ON_ERROR_RATE
        value: "5"  # Abort if error rate exceeds 5%
      - name: MONITOR_INTERVAL
        value: "10"
      
      # Notification settings
      - name: SLACK_WEBHOOK
        valueFrom:
          secretKeyRef:
            name: chaos-secrets
            key: slack-webhook
    
    # Labels for discovery
    labels:
      experiment: payment-service-latency
      tier: critical
      owner: payments-team
    
    # ConfigMap for additional configuration
    configMaps:
      - name: payment-chaos-config
        mountPath: /etc/chaos
 
---
# Usage documentation embedded in experiment
apiVersion: v1
kind: ConfigMap
metadata:
  name: payment-chaos-config
  namespace: litmus
data:
  README.md: |
    # Payment Service Latency Experiment
    
    ## Purpose
    Tests the payment service's behavior when upstream payment providers
    experience latency. Validates circuit breaker configuration and
    fallback to cached authorization responses.
    
    ## Prerequisites
    - Payment service deployed with circuit breaker enabled
    - Fallback cache warmed with test authorization data
    - Monitoring dashboard open during experiment
    
    ## Expected Behavior
    - Circuit breaker should open at 500ms sustained latency
    - Fallback cache should serve pre-authorized transactions
    - Error rate should not exceed 2% for cached operations
    
    ## Rollback Procedure
    1. Halt ChaosEngine: kubectl delete chaosengine payment-latency-test
    2. All latency injection stops automatically
    3. Circuit breakers will close after recovery period (30s default)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202

# Example: Comprehensive Service Resilience Workflow
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  name: service-resilience-test
  namespace: litmus
spec:
  entrypoint: resilience-test-dag
  serviceAccountName: litmus-admin
  
  # Workflow-level arguments
  arguments:
    parameters:
      - name: target-namespace
        value: production
      - name: target-deployment
        value: api-gateway
      - name: chaos-duration
        value: "120"
  
  templates:
    # DAG orchestration template
    - name: resilience-test-dag
      dag:
        tasks:
          # Step 1: Pre-chaos validation
          - name: pre-chaos-health-check
            template: health-check
            arguments:
              parameters:
                - name: phase
                  value: pre-chaos
          
          # Step 2a: Pod chaos (in parallel with Step 2b)
          - name: pod-delete-chaos
            template: run-pod-delete
            depends: pre-chaos-health-check
          
          # Step 2b: Network chaos (in parallel with Step 2a)
          - name: network-latency-chaos
            template: run-network-latency
            depends: pre-chaos-health-check
          
          # Step 3: Wait for both chaos experiments
          - name: mid-chaos-health-check
            template: health-check
            depends: "pod-delete-chaos && network-latency-chaos"
            arguments:
              parameters:
                - name: phase
                  value: mid-chaos
          
          # Step 4: Node-level chaos (escalation)
          - name: node-cpu-chaos
            template: run-node-cpu-hog
            depends: mid-chaos-health-check
            # Only run if mid-chaos check passed
            when: "{{steps.mid-chaos-health-check.outputs.result}} == passed"
          
          # Step 5: Recovery validation
          - name: post-chaos-health-check
            template: health-check
            depends: node-cpu-chaos
            arguments:
              parameters:
                - name: phase
                  value: post-chaos
          
          # Step 6: Generate report
          - name: generate-report
            template: chaos-report
            depends: post-chaos-health-check
    
    # Health check template
    - name: health-check
      inputs:
        parameters:
          - name: phase
      container:
        image: internal-registry/chaos-tools:latest
        command: ["/bin/bash", "-c"]
        args:
          - |
            echo "Running health check: {{inputs.parameters.phase}}"
            
            # Check deployment availability
            AVAILABLE=$(kubectl get deployment \
              {{workflow.parameters.target-deployment}} \
              -n {{workflow.parameters.target-namespace}} \
              -o jsonpath='{.status.availableReplicas}')
            
            DESIRED=$(kubectl get deployment \
              {{workflow.parameters.target-deployment}} \
              -n {{workflow.parameters.target-namespace}} \
              -o jsonpath='{.spec.replicas}')
            
            # Check error rate from Prometheus
            ERROR_RATE=$(curl -s "http://prometheus:9090/api/v1/query?query=rate(http_requests_total{status=~"5..",deployment="{{workflow.parameters.target-deployment}}"}[5m])" | jq -r '.data.result[0].value[1]')
            
            echo "Available: $AVAILABLE, Desired: $DESIRED, Error Rate: $ERROR_RATE"
            
            # Validation logic
            if [ "$AVAILABLE" -ge "$((DESIRED - 1))" ] && [ $(echo "$ERROR_RATE < 0.05" | bc -l) -eq 1 ]; then
              echo "passed" > /tmp/result
            else
              echo "failed" > /tmp/result
            fi
      outputs:
        parameters:
          - name: result
            valueFrom:
              path: /tmp/result
    
    # Pod delete experiment template
    - name: run-pod-delete
      container:
        image: litmuschaos/litmus-operator:latest
        args:
          - -c
          - |
            kubectl apply -f - <<EOF
            apiVersion: litmuschaos.io/v1alpha1
            kind: ChaosEngine
            metadata:
              name: pod-delete-engine
              namespace: {{workflow.parameters.target-namespace}}
            spec:
              engineState: active
              appinfo:
                appns: {{workflow.parameters.target-namespace}}
                applabel: app={{workflow.parameters.target-deployment}}
                appkind: deployment
              experiments:
                - name: pod-delete
                  spec:
                    components:
                      env:
                        - name: TOTAL_CHAOS_DURATION
                          value: "{{workflow.parameters.chaos-duration}}"
                        - name: PODS_AFFECTED_PERC
                          value: "30"
            EOF
            
            # Wait for experiment completion
            while true; do
              STATUS=$(kubectl get chaosengine pod-delete-engine \
                -n {{workflow.parameters.target-namespace}} \
                -o jsonpath='{.status.engineStatus}')
              if [ "$STATUS" == "completed" ]; then
                break
              fi
              sleep 10
            done
        command:
          - /bin/bash
    
    # Network latency experiment template
    - name: run-network-latency
      container:
        image: litmuschaos/litmus-operator:latest
        args:
          - -c
          - |
            kubectl apply -f - <<EOF
            apiVersion: litmuschaos.io/v1alpha1
            kind: ChaosEngine
            metadata:
              name: network-latency-engine
              namespace: {{workflow.parameters.target-namespace}}
            spec:
              engineState: active
              appinfo:
                appns: {{workflow.parameters.target-namespace}}
                applabel: app={{workflow.parameters.target-deployment}}
                appkind: deployment
              experiments:
                - name: pod-network-latency
                  spec:
                    components:
                      env:
                        - name: TOTAL_CHAOS_DURATION
                          value: "{{workflow.parameters.chaos-duration}}"
                        - name: NETWORK_LATENCY
                          value: "200"
                        - name: CONTAINER_RUNTIME
                          value: containerd
            EOF
            
            # Wait for completion
            while true; do
              STATUS=$(kubectl get chaosengine network-latency-engine \
                -n {{workflow.parameters.target-namespace}} \
                -o jsonpath='{.status.engineStatus}')
              if [ "$STATUS" == "completed" ]; then
                break
              fi
              sleep 10
            done
        command:
          - /bin/bash
    
    # Additional templates for node-cpu-hog and report generation...

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

# Example: ChaosEngine with Comprehensive Probes
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
  name: payment-service-chaos
  namespace: production
spec:
  engineState: active
  appinfo:
    appns: production
    applabel: app=payment-service
    appkind: deployment
  
  experiments:
    - name: pod-delete
      spec:
        probe:
          # HTTP Probe: Verify service is responding
          - name: payment-health-probe
            type: httpProbe
            mode: Continuous
            runProperties:
              probeTimeout: 5
              retry: 3
              interval: 2
              probePollingInterval: 1
              initialDelaySeconds: 3
            httpProbe/inputs:
              url: http://payment-service.production.svc:8080/health
              method:
                get:
                  criteria: ==
                  responseCode: "200"
          
          # Prometheus Probe: Verify error rate SLO
          - name: error-rate-slo-probe
            type: promProbe
            mode: Edge  # Check at start and end
            runProperties:
              probeTimeout: 10
              retry: 2
              interval: 5
            promProbe/inputs:
              endpoint: http://prometheus.monitoring.svc:9090
              query: |
                sum(rate(http_requests_total{service="payment-service",status=~"5.."}[5m])) /
                sum(rate(http_requests_total{service="payment-service"}[5m])) * 100
              comparator:
                type: float
                criteria: <=
                value: "1.0"  # Error rate <= 1%
          
          # Kubernetes Probe: Verify minimum replicas
          - name: replica-count-probe
            type: k8sProbe
            mode: Continuous
            runProperties:
              probeTimeout: 5
              retry: 2
              interval: 3
            k8sProbe/inputs:
              group: apps
              version: v1
              resource: deployments
              namespace: production
              fieldSelector: metadata.name=payment-service
              operation: present
          
          # Command Probe: Custom database connectivity check
          - name: database-connectivity-probe
            type: cmdProbe
            mode: OnChaos
            runProperties:
              probeTimeout: 30
              retry: 1
              interval: 10
            cmdProbe/inputs:
              command: |
                /bin/bash -c '
                  # Check database connection from a test pod
                  kubectl run db-test --rm -i --restart=Never \
                    --image=postgres:13 \
                    --namespace=production \
                    -- psql -h postgres -U app -c "SELECT 1" || exit 1
                  echo "Database connectivity verified"
                '
              source: inline
              comparator:
                type: string
                criteria: contains
                value: "Database connectivity verified"
 
  # Annotation for Prometheus metrics
  monitoringEnabled: true
  
  # Cleanup policy
  chaosServiceAccount: litmus-admin
  annotationCheck: "false"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

# Prometheus alerts for LitmusChaos events
groups:
  - name: litmus-chaos-alerts
    rules:
      # Alert when chaos experiments fail
      - alert: ChaosExperimentFailed
        expr: litmuschaos_experiment_verdict{verdict="Fail"} > 0
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: "Chaos experiment failed: {{ $labels.chaosengine_name }}"
          description: |
            Experiment {{ $labels.experiment_name }} in namespace 
            {{ $labels.namespace }} failed. This may indicate a 
            resilience gap in {{ $labels.app_label }}.
          runbook: https://wiki.company.com/chaos/experiment-failures
      
      # Alert when probe failures occur during chaos
      - alert: ChaosProbeFailure
        expr: litmuschaos_probe_success_percentage < 100
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: "Chaos probe failures detected"
          description: |
            Probe {{ $labels.probe_name }} had failures during chaos 
            experiment {{ $labels.chaosengine_name }}. Success rate: 
            {{ $value }}%
      
      # Alert on unusually high chaos activity
      - alert: HighChaosActivity
        expr: sum(rate(litmuschaos_experiment_total[1h])) > 10
        for: 5m
        labels:
          severity: info
        annotations:
          summary: "High chaos engineering activity detected"
          description: |
            More than 10 chaos experiments triggered in the last 
            hour. Verify this is expected activity.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

# GitOps Repository Structure for Chaos-as-Code
 
chaos-experiments/
├── base/                          # Shared experiment definitions
│   ├── experiments/
│   │   ├── pod-delete.yaml
│   │   ├── network-latency.yaml
│   │   └── node-cpu-hog.yaml
│   └── kustomization.yaml
│
├── environments/
│   ├── staging/                   # Staging-specific configuration
│   │   ├── engines/
│   │   │   ├── api-gateway-chaos.yaml
│   │   │   └── payment-service-chaos.yaml
│   │   ├── schedules/
│   │   │   └── daily-resilience-test.yaml
│   │   └── kustomization.yaml
│   │
│   └── production/                # Production-specific configuration
│       ├── engines/
│       │   ├── api-gateway-chaos.yaml
│       │   └── payment-service-chaos.yaml
│       ├── schedules/
│       │   └── weekly-gameday.yaml
│       └── kustomization.yaml
│
├── workflows/                     # Complex multi-step workflows
│   ├── service-resilience-test.yaml
│   ├── database-failover-test.yaml
│   └── full-stack-chaos.yaml
│
└── README.md                      # Documentation and runbooks

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

# Argo CD Application for Chaos Experiments
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: chaos-experiments-production
  namespace: argocd
spec:
  project: chaos-engineering
  
  source:
    repoURL: https://github.com/company/chaos-experiments.git
    targetRevision: main
    path: environments/production
    
    # Kustomize for environment-specific patches
    kustomize:
      namePrefix: prod-
      commonLabels:
        environment: production
        managed-by: argocd
  
  destination:
    server: https://kubernetes.default.svc
    namespace: litmus
  
  syncPolicy:
    automated:
      prune: true  # Remove experiments no longer in Git
      selfHeal: true  # Revert manual changes
    syncOptions:
      - CreateNamespace=true
    
    # Retry on transient failures
    retry:
      limit: 5
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m
 
---
# Argo CD Project with appropriate permissions
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
  name: chaos-engineering
  namespace: argocd
spec:
  description: Chaos Engineering experiments and workflows
  
  # Limit which clusters can receive chaos
  destinations:
    - namespace: litmus
      server: https://kubernetes.default.svc
    - namespace: staging
      server: https://kubernetes.default.svc
    - namespace: production
      server: https://kubernetes.default.svc
  
  # Limit which repos can define chaos
  sourceRepos:
    - https://github.com/company/chaos-experiments.git
  
  # Cluster resources chaos can manage
  clusterResourceWhitelist:
    - group: ""
      kind: Namespace
    - group: litmuschaos.io
      kind: "*"
  
  # Namespace resources
  namespaceResourceWhitelist:
    - group: ""
      kind: ServiceAccount
    - group: rbac.authorization.k8s.io
      kind: Role
    - group: rbac.authorization.k8s.io
      kind: RoleBinding
    - group: litmuschaos.io
      kind: "*"