Deployment Strategies

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

#!/bin/bash
# Kubernetes Application Rollback Procedures
 
set -euo pipefail
 
DEPLOYMENT="payment-service"
NAMESPACE="production"
 
echo "═══════════════════════════════════════════════════════════"
echo "🔄 Initiating rollback for $DEPLOYMENT"
echo "═══════════════════════════════════════════════════════════"
 
# Step 1: Check current status
echo "Current deployment status:"
kubectl -n "$NAMESPACE" get deployment "$DEPLOYMENT" -o wide
 
# Step 2: View rollout history
echo ""
echo "Rollout history:"
kubectl -n "$NAMESPACE" rollout history deployment/"$DEPLOYMENT"
 
# Step 3: Check what will be rolled back to
CURRENT_REVISION=$(kubectl -n "$NAMESPACE" rollout history deployment/"$DEPLOYMENT" --output jsonpath='{.metadata.generation}')
PREVIOUS_REVISION=$((CURRENT_REVISION - 1))
echo ""
echo "Will roll back from revision $CURRENT_REVISION to revision $PREVIOUS_REVISION"
 
# Step 4: Get details of previous revision
echo ""
echo "Previous revision details:"
kubectl -n "$NAMESPACE" rollout history deployment/"$DEPLOYMENT" --revision="$PREVIOUS_REVISION"
 
# Step 5: Perform the rollback
echo ""
echo "Executing rollback..."
kubectl -n "$NAMESPACE" rollout undo deployment/"$DEPLOYMENT"
 
# Step 6: Monitor rollback progress
echo ""
echo "Monitoring rollback progress..."
kubectl -n "$NAMESPACE" rollout status deployment/"$DEPLOYMENT" --timeout=300s
 
# Step 7: Verify rollback
echo ""
echo "Verifying rollback:"
kubectl -n "$NAMESPACE" get deployment "$DEPLOYMENT" -o wide
 
# Step 8: Check pod status
echo ""
echo "Pod status after rollback:"
kubectl -n "$NAMESPACE" get pods -l app="$DEPLOYMENT" -o wide
 
# Step 9: Notify team
echo ""
echo "═══════════════════════════════════════════════════════════"
echo "✅ Rollback complete!"
echo "═══════════════════════════════════════════════════════════"
echo ""
echo "Post-rollback actions:"
echo "1. Verify application health: kubectl -n $NAMESPACE logs -l app=$DEPLOYMENT --tail=50"
echo "2. Check error rates in monitoring dashboard"
echo "3. Notify incident channel"
echo "4. Create post-mortem ticket"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

-- Example: Reversible migration pattern
-- Each migration has explicit UP and DOWN operations
 
-- ========================================
-- Migration: V20240115_Add_User_Preferences
-- ========================================
 
-- UP (forward migration)
CREATE TABLE user_preferences (
    id BIGINT PRIMARY KEY AUTO_INCREMENT,
    user_id BIGINT NOT NULL,
    preference_key VARCHAR(100) NOT NULL,
    preference_value TEXT,
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
    UNIQUE KEY uk_user_preference (user_id, preference_key)
);
 
CREATE INDEX idx_preferences_user ON user_preferences(user_id);
 
-- DOWN (rollback migration)
-- Execute this if the deployment fails and rollback is needed
 
DROP INDEX idx_preferences_user ON user_preferences;
DROP TABLE user_preferences;
 
-- ========================================
-- Migration: V20240116_Add_Email_Verified_Column  
-- ========================================
 
-- UP
ALTER TABLE users ADD COLUMN email_verified BOOLEAN DEFAULT FALSE;
 
-- Update existing users based on some criteria
UPDATE users SET email_verified = TRUE WHERE confirmed_at IS NOT NULL;
 
-- DOWN
-- NOTE: This loses the email_verified data
ALTER TABLE users DROP COLUMN email_verified;
 
-- ========================================
-- IRREVERSIBLE Migration Example
-- Migration: V20240117_Remove_Legacy_Column
-- ========================================
 
-- UP
-- First, verify no application code references this column
ALTER TABLE users DROP COLUMN legacy_field;
 
-- DOWN
-- IRREVERSIBLE: Cannot restore data that was in legacy_field
-- Require backup restoration if rollback needed
-- 
-- Procedure if rollback needed:
-- 1. Restore full database from backup
-- 2. OR: Re-add column (but data is lost)
--    ALTER TABLE users ADD COLUMN legacy_field VARCHAR(255);
--
-- IMPORTANT: Before running this migration:
-- - Ensure full recent backup exists
-- - Ensure no rollback will be needed (migration has been in staging)
-- - Consider keeping column as deprecated instead of dropping

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

#!/bin/bash
# Configuration Rollback Procedures
 
# ========================================
# Kubernetes ConfigMap Rollback
# ========================================
 
# ConfigMaps don't have built-in versioning, but you can use labels
 
# Current approach: version ConfigMaps with timestamps
kubectl create configmap app-config-v20240115 \
    --from-file=config.yaml \
    -o yaml --dry-run=client | kubectl apply -f -
 
# Update deployment to use specific ConfigMap version
kubectl patch deployment payment-service \
    -p '{"spec":{"template":{"spec":{"volumes":[{"name":"config","configMap":{"name":"app-config-v20240115"}}]}}}}'
 
# To rollback: switch to previous ConfigMap version
kubectl patch deployment payment-service \
    -p '{"spec":{"template":{"spec":{"volumes":[{"name":"config","configMap":{"name":"app-config-v20240114"}}]}}}}'
 
# ========================================
# AWS Parameter Store Rollback
# ========================================
 
# Parameter Store maintains version history automatically
 
# View parameter history
aws ssm get-parameter-history \
    --name /production/payment-service/api-key \
    --output table
 
# Get previous version value
PREVIOUS_VALUE=$(aws ssm get-parameter \
    --name /production/payment-service/api-key:2 \
    --with-decryption \
    --query 'Parameter.Value' \
    --output text)
 
# Roll back by setting parameter to previous value
aws ssm put-parameter \
    --name /production/payment-service/api-key \
    --value "$PREVIOUS_VALUE" \
    --type SecureString \
    --overwrite
 
# ========================================
# Terraform Infrastructure Rollback
# ========================================
 
# Option 1: Git revert and re-apply (preferred)
cd terraform/
git log --oneline -5
# abc123 Update API gateway config (BROKEN)
# def456 Add new Lambda function
# ghi789 Previous working state
 
git revert abc123  # Creates revert commit
terraform plan     # Review changes
terraform apply    # Apply rollback
 
# Option 2: Apply previous state file (risky, not recommended)
# This can cause state drift and unexpected behavior
# Only use if Git history is not available
 
# ========================================
# Feature Flag Rollback (Instant)
# ========================================
 
# LaunchDarkly CLI example
ldcli flags update \
    --project production \
    --key new-payment-flow \
    --set-toggle-off
 
# Generic API call
curl -X PATCH https://flags.internal/api/flags/new-payment-flow \
    -H "Authorization: Bearer $FLAG_API_KEY" \
    -d '{"enabled": false}'

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

# Argo Rollouts with automatic rollback on failure
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: payment-service
spec:
  replicas: 10
  selector:
    matchLabels:
      app: payment-service
  template:
    metadata:
      labels:
        app: payment-service
    spec:
      containers:
      - name: payment-service
        image: payment-service:v2.0.0
        
  strategy:
    canary:
      steps:
      - setWeight: 10
      - analysis:
          templates:
          - templateName: success-rate-check
          args:
          - name: service
            value: payment-service-canary
      - setWeight: 50
      - analysis:
          templates:
          - templateName: success-rate-check
      - setWeight: 100
      
      # Automatic rollback configuration
      autoPromotionEnabled: true  # Promote automatically if analysis passes
      
      # Rollback on analysis failure
      analysis:
        successfulRunHistoryLimit: 3
        unsuccessfulRunHistoryLimit: 3
        
      # Abort and rollback conditions
      abortScaleDownDelaySeconds: 30
 
---
# Analysis template that triggers rollback
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: success-rate-check
spec:
  args:
  - name: service
  metrics:
  - name: success-rate
    interval: 30s
    # If this condition fails 3 times, rollback is triggered
    successCondition: result[0] >= 0.99
    failureCondition: result[0] < 0.95  # Immediate failure threshold
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          sum(rate(http_requests_total{
            service="{{args.service}}",
            status!~"5.."
          }[2m]))
          /
          sum(rate(http_requests_total{
            service="{{args.service}}"
          }[2m]))

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

#!/bin/bash
# Coordinated multi-service rollback procedure
 
set -euo pipefail
 
# Step 1: Disable the feature flag to stop new traffic to broken functionality
echo "Step 1: Disabling feature flag..."
curl -X PATCH https://flags.internal/api/flags/new-payment-flow \
    -H "Authorization: Bearer $FLAG_API_KEY" \
    -d '{"enabled": false}'
 
echo "Waiting for flag propagation..."
sleep 10
 
# Step 2: Roll back the dependent service first (consumer)
echo "Step 2: Rolling back payment-service..."
kubectl -n production rollout undo deployment/payment-service
 
kubectl -n production rollout status deployment/payment-service --timeout=300s
 
# Step 3: Roll back the upstream service (producer)
echo "Step 3: Rolling back order-service..."
kubectl -n production rollout undo deployment/order-service
 
kubectl -n production rollout status deployment/order-service --timeout=300s
 
# Step 4: Verify both services are healthy
echo "Step 4: Verifying service health..."
 
PAYMENT_HEALTH=$(curl -s -o /dev/null -w "%{http_code}" https://payment.internal/health)
ORDER_HEALTH=$(curl -s -o /dev/null -w "%{http_code}" https://order.internal/health)
 
if [[ "$PAYMENT_HEALTH" == "200" && "$ORDER_HEALTH" == "200" ]]; then
    echo "✅ Both services healthy after rollback"
else
    echo "❌ Health check failed: payment=$PAYMENT_HEALTH, order=$ORDER_HEALTH"
    exit 1
fi
 
# Step 5: Verify cross-service communication
echo "Step 5: Running integration test..."
npm run test:integration:payment-order
 
# Step 6: Document the rollback
echo "Step 6: Creating incident record..."
curl -X POST https://incidents.internal/api/incidents \
    -H "Content-Type: application/json" \
    -d '{
        "title": "Coordinated rollback: payment-service, order-service",
        "severity": "P2",
        "services": ["payment-service", "order-service"],
        "rolled_back_by": "'"$USER"'",
        "timestamp": "'"$(date -Iseconds)"'"
    }'
 
echo ""
echo "═══════════════════════════════════════════════════════════"
echo "✅ Coordinated rollback complete"
echo "═══════════════════════════════════════════════════════════"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

// Rollback-Friendly Design Patterns
 
// ========================================
// Pattern 1: Version-Aware Message Handling
// ========================================
 
interface OrderMessage {
  version: 1 | 2 | 3;
  orderId: string;
  // Version-specific fields
  [key: string]: unknown;
}
 
class OrderProcessor {
  async processOrder(message: OrderMessage): Promise<void> {
    // Handle all known versions
    switch (message.version) {
      case 1:
        return this.processV1(message as OrderV1);
      case 2:
        return this.processV2(message as OrderV2);
      case 3:
        return this.processV3(message as OrderV3);
      default:
        // Forward compatibility: log and skip unknown versions
        console.warn(`Unknown message version: ${message.version}`);
        return;
    }
  }
  
  private async processV1(order: OrderV1): Promise<void> {
    // Legacy processing
  }
  
  private async processV2(order: OrderV2): Promise<void> {
    // V2 added shipping_address as separate field
    const shippingAddress = order.shipping_address ?? order.address;
    // Process with shipping address
  }
  
  private async processV3(order: OrderV3): Promise<void> {
    // V3 added gift options
    const giftOptions = order.gift_options ?? { isGift: false };
    // Process with gift options
  }
}
 
// ========================================
// Pattern 2: Idempotent Operations
// ========================================
 
class PaymentService {
  async processPayment(request: PaymentRequest): Promise<PaymentResult> {
    // Check if this exact payment was already processed
    const existingPayment = await this.db.payments.findUnique({
      where: { idempotencyKey: request.idempotencyKey }
    });
    
    if (existingPayment) {
      // Already processed - return the previous result
      // Safe even if this is a retry after rollback
      return {
        success: true,
        paymentId: existingPayment.id,
        cached: true
      };
    }
    
    // Process new payment
    const payment = await this.chargeCard(request);
    
    // Store with idempotency key
    await this.db.payments.create({
      data: {
        ...payment,
        idempotencyKey: request.idempotencyKey
      }
    });
    
    return { success: true, paymentId: payment.id, cached: false };
  }
}
 
// ========================================
// Pattern 3: Graceful Degradation
// ========================================
 
class RecommendationService {
  async getRecommendations(userId: string): Promise<Product[]> {
    try {
      // Try new ML-based recommendations (v2 feature)
      if (await this.flags.isEnabled('ml-recommendations', userId)) {
        return await this.mlRecommendations.getForUser(userId);
      }
    } catch (error) {
      // ML service down - log and fall through to fallback
      console.error('ML recommendations failed:', error);
    }
    
    try {
      // Try collaborative filtering (v1 feature)
      return await this.collaborativeFiltering.getForUser(userId);
    } catch (error) {
      console.error('Collaborative filtering failed:', error);
    }
    
    // Ultimate fallback: popular products (always works)
    return await this.getPopularProducts();
  }
}