Blue-Green Deployments

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

# AWS Blue-Green with Application Load Balancer
# Traffic switching via listener rule modification
 
resource "aws_lb" "main" {
  name               = "app-alb"
  internal           = false
  load_balancer_type = "application"
  subnets            = var.public_subnet_ids
  
  tags = {
    Environment = "production"
  }
}
 
# Blue target group - current production
resource "aws_lb_target_group" "blue" {
  name     = "app-blue"
  port     = 8080
  protocol = "HTTP"
  vpc_id   = var.vpc_id
  
  health_check {
    enabled             = true
    healthy_threshold   = 2
    unhealthy_threshold = 3
    timeout             = 5
    interval            = 30
    path                = "/health"
    matcher             = "200"
  }
  
  tags = {
    Color = "blue"
  }
}
 
# Green target group - next release
resource "aws_lb_target_group" "green" {
  name     = "app-green"
  port     = 8080
  protocol = "HTTP"
  vpc_id   = var.vpc_id
  
  health_check {
    enabled             = true
    healthy_threshold   = 2
    unhealthy_threshold = 3
    timeout             = 5
    interval            = 30
    path                = "/health"
    matcher             = "200"
  }
  
  tags = {
    Color = "green"
  }
}
 
# Production listener - the switch point
resource "aws_lb_listener" "production" {
  load_balancer_arn = aws_lb.main.arn
  port              = "443"
  protocol          = "HTTPS"
  ssl_policy        = "ELBSecurityPolicy-TLS-1-2-2017-01"
  certificate_arn   = var.certificate_arn
  
  default_action {
    type             = "forward"
    # SWITCH: Change this to green when deploying
    target_group_arn = var.active_color == "blue" ? aws_lb_target_group.blue.arn : aws_lb_target_group.green.arn
  }
}
 
# Test listener - access inactive environment for validation
resource "aws_lb_listener" "test" {
  load_balancer_arn = aws_lb.main.arn
  port              = "8443"
  protocol          = "HTTPS"
  ssl_policy        = "ELBSecurityPolicy-TLS-1-2-2017-01"
  certificate_arn   = var.certificate_arn
  
  default_action {
    type             = "forward"
    # Points to the INACTIVE environment for testing
    target_group_arn = var.active_color == "blue" ? aws_lb_target_group.green.arn : aws_lb_target_group.blue.arn
  }
}
 
# Variable to control which environment is active
variable "active_color" {
  description = "Which color environment is serving production traffic"
  type        = string
  default     = "blue"
  
  validation {
    condition     = contains(["blue", "green"], var.active_color)
    error_message = "Active color must be 'blue' or 'green'."
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

# Blue Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-blue
  labels:
    app: myapp
    version: v1.0.0
spec:
  replicas: 5
  selector:
    matchLabels:
      app: myapp
      color: blue
  template:
    metadata:
      labels:
        app: myapp
        color: blue
        version: v1.0.0
    spec:
      containers:
      - name: app
        image: myapp:v1.0.0
        ports:
        - containerPort: 8080
        readinessProbe:
          httpGet:
            path: /health/ready
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5
 
---
# Green Deployment (new version)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-green
  labels:
    app: myapp
    version: v1.1.0
spec:
  replicas: 5
  selector:
    matchLabels:
      app: myapp
      color: green
  template:
    metadata:
      labels:
        app: myapp
        color: green
        version: v1.1.0
    spec:
      containers:
      - name: app
        image: myapp:v1.1.0
        ports:
        - containerPort: 8080
        readinessProbe:
          httpGet:
            path: /health/ready
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5
 
---
# Production Service - selector determines which deployment receives traffic
apiVersion: v1
kind: Service
metadata:
  name: myapp
spec:
  selector:
    app: myapp
    # SWITCH: Change this label to switch traffic
    color: blue  # or "green"
  ports:
  - port: 80
    targetPort: 8080
  type: ClusterIP
 
---
# Test Service - always points to inactive environment
apiVersion: v1
kind: Service
metadata:
  name: myapp-test
spec:
  selector:
    app: myapp
    # Points to INACTIVE color for pre-switch testing
    color: green  # Opposite of production
  ports:
  - port: 80
    targetPort: 8080
  type: ClusterIP

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

-- Example: Safely renaming a column in a blue-green environment
-- Total migration requires 4 deployment cycles
 
-- ============================================
-- DEPLOYMENT 1: Add new column
-- ============================================
-- Run this migration before deploying new code
ALTER TABLE users ADD COLUMN username VARCHAR(255);
 
-- Copy existing data
UPDATE users SET username = user_name WHERE username IS NULL;
 
-- At this point:
-- - Blue (v1) still reads/writes user_name (works)
-- - Green (v2) can read both columns, writes to both (works)
 
-- ============================================
-- DEPLOYMENT 2: Deploy code that reads both, writes both
-- ============================================
-- Application code change (pseudo-code):
-- 
-- function getUsername(user):
--     return user.username ?? user.user_name  // prefer new, fallback to old
--
-- function saveUser(user):
--     user.username = value
--     user.user_name = value  // write to BOTH columns
 
-- ============================================
-- DEPLOYMENT 3: Deploy code that only uses new column
-- ============================================
-- Application code change (pseudo-code):
-- 
-- function getUsername(user):
--     return user.username  // only new column
--
-- function saveUser(user):
--     user.username = value  // only new column
 
-- Verify no code references old column before proceeding
 
-- ============================================
-- DEPLOYMENT 4: Remove old column
-- ============================================
-- Only after all instances use new column exclusively
ALTER TABLE users DROP COLUMN user_name;
 
-- Safe because:
-- - No running code reads user_name
-- - No running code writes user_name
-- - Rollback would deploy code that reads username (new column)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

import Redis from 'ioredis';
import { v4 as uuidv4 } from 'uuid';
 
/**
 * Session store backed by Redis - shared between blue and green environments.
 * Both environments connect to the same Redis cluster.
 */
class SessionStore {
  private redis: Redis;
  private sessionTTL: number = 24 * 60 * 60; // 24 hours
  
  constructor(redisUrl: string) {
    // Both blue and green environments use the same Redis cluster
    this.redis = new Redis(redisUrl);
  }
  
  async createSession(userId: string, data: SessionData): Promise<string> {
    const sessionId = uuidv4();
    
    // Include schema version for forward/backward compatibility
    const sessionPayload: VersionedSession = {
      schemaVersion: 2,  // Increment when session structure changes
      userId,
      data,
      createdAt: new Date().toISOString(),
      // New fields added in v2 - old code will ignore, new code uses
      environment: process.env.ENVIRONMENT_COLOR,
      lastActiveAt: new Date().toISOString(),
    };
    
    await this.redis.setex(
      `session:${sessionId}`,
      this.sessionTTL,
      JSON.stringify(sessionPayload)
    );
    
    return sessionId;
  }
  
  async getSession(sessionId: string): Promise<SessionData | null> {
    const raw = await this.redis.get(`session:${sessionId}`);
    if (!raw) return null;
    
    const session = JSON.parse(raw) as VersionedSession;
    
    // Handle different schema versions
    // Old sessions (v1) may not have new fields
    return this.migrateSession(session);
  }
  
  private migrateSession(session: VersionedSession): SessionData {
    // Handle missing fields from older schema versions
    // This allows green (new version) to read sessions created by blue (old version)
    
    if (!session.schemaVersion || session.schemaVersion < 2) {
      // Migrate v1 session to v2 format
      session.lastActiveAt = session.createdAt;
      session.environment = 'unknown';
    }
    
    return session.data;
  }
  
  async updateSession(sessionId: string, data: Partial<SessionData>): Promise<void> {
    const existing = await this.redis.get(`session:${sessionId}`);
    if (!existing) throw new Error('Session not found');
    
    const session = JSON.parse(existing) as VersionedSession;
    session.data = { ...session.data, ...data };
    session.lastActiveAt = new Date().toISOString();
    
    await this.redis.setex(
      `session:${sessionId}`,
      this.sessionTTL,
      JSON.stringify(session)
    );
  }
}
 
interface SessionData {
  cart?: CartItem[];
  preferences?: UserPreferences;
  [key: string]: unknown;
}
 
interface VersionedSession {
  schemaVersion: number;
  userId: string;
  data: SessionData;
  createdAt: string;
  environment?: string;      // Added in v2
  lastActiveAt?: string;     // Added in v2
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

#!/bin/bash
# Pre-switch validation script for blue-green deployment
 
set -euo pipefail
 
INACTIVE_ENV="green"
TEST_ENDPOINT="https://test.example.com"  # Points to inactive environment
MAIN_ENDPOINT="https://api.example.com"   # Points to active environment
 
echo "🔍 Starting pre-switch validation for $INACTIVE_ENV environment..."
 
# 1. Health check verification
echo "Step 1: Verifying health checks..."
HEALTH=$(curl -s -o /dev/null -w "%{http_code}" "$TEST_ENDPOINT/health/ready")
if [ "$HEALTH" != "200" ]; then
    echo "❌ Health check failed with status $HEALTH"
    exit 1
fi
echo "✅ Health checks passing"
 
# 2. Smoke tests
echo "Step 2: Running smoke tests..."
npm run test:smoke -- --endpoint="$TEST_ENDPOINT"
if [ $? -ne 0 ]; then
    echo "❌ Smoke tests failed"
    exit 1
fi
echo "✅ Smoke tests passing"
 
# 3. API version verification
echo "Step 3: Verifying API version..."
DEPLOYED_VERSION=$(curl -s "$TEST_ENDPOINT/version" | jq -r '.version')
EXPECTED_VERSION="${DEPLOY_VERSION:- unknown}"
if ["$DEPLOYED_VERSION" != "$EXPECTED_VERSION"]; then
    echo "❌ Version mismatch: expected $EXPECTED_VERSION, got $DEPLOYED_VERSION"
    exit 1
fi
echo "✅ Version verified: $DEPLOYED_VERSION"
 
# 4. Performance baseline
echo "Step 4: Running performance baseline..."
LATENCY_P99 = $(curl - s "$TEST_ENDPOINT/metrics" | grep 'http_request_duration_seconds' | grep 'quantile="0.99"' | awk '{print $2}')
LATENCY_THRESHOLD="0.5"
if(($(echo "$LATENCY_P99 > $LATENCY_THRESHOLD" | bc - l))); then
    echo "❌ P99 latency $LATENCY_P99 exceeds threshold $LATENCY_THRESHOLD"
    exit 1
    fi
echo "✅ Performance baseline met: p99=${LATENCY_P99}s"
 
# 5. Database connectivity
echo "Step 5: Verifying database connectivity..."
    DB_CHECK = $(curl - s "$TEST_ENDPOINT/health/dependencies" | jq - r '.database.status')
    if ["$DB_CHECK" != "healthy"]; then
    echo "❌ Database connectivity check failed"
    exit 1
    fi
echo "✅ Database connectivity verified"
 
# 6. Feature flag verification
echo "Step 6: Verifying feature flags..."
    FLAGS = $(curl - s "$TEST_ENDPOINT/debug/feature-flags")
echo "Active feature flags: $FLAGS"
 
# 7. Compare metrics with active environment
echo "Step 7: Comparing error rates..."
    ACTIVE_ERRORS = $(curl - s "$MAIN_ENDPOINT/metrics" | grep 'http_requests_total.*status="5' | awk '{sum+=$2} END {print sum}')
    INACTIVE_ERRORS = $(curl - s "$TEST_ENDPOINT/metrics" | grep 'http_requests_total.*status="5' | awk '{sum+=$2} END {print sum}')
echo "Active environment errors: $ACTIVE_ERRORS"
echo "Inactive environment errors: $INACTIVE_ERRORS"
 
echo ""
echo "═══════════════════════════════════════════════════════════"
echo "✅ All pre-switch validations passed"
echo "═══════════════════════════════════════════════════════════"
echo ""
echo "Ready to switch traffic to $INACTIVE_ENV environment."
echo "To proceed, run: ./switch-traffic.sh $INACTIVE_ENV"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

#!/bin/bash
# Traffic switching script for blue - green deployment
 
set - euo pipefail
 
    TARGET_COLOR = "$1"
 
    if [["$TARGET_COLOR" != "blue" && "$TARGET_COLOR" != "green"]]; then
    echo "Usage: $0 <blue|green>"
    exit 1
    fi
 
# Get current state
    CURRENT_COLOR = $(aws ssm get - parameter--name / app / active - color--query 'Parameter.Value' --output text)
 
    if ["$CURRENT_COLOR" == "$TARGET_COLOR"]; then
    echo "Traffic already routing to $TARGET_COLOR"
    exit 0
    fi
 
echo "═══════════════════════════════════════════════════════════"
echo "🔄 Switching traffic from $CURRENT_COLOR to $TARGET_COLOR"
echo "═══════════════════════════════════════════════════════════"
 
# Record the switch in audit log
aws logs put - log - events     --log - group - name / app / deployments     --log - stream - name traffic - switches     --log - events timestamp = $(date +% s000), message = "Switching traffic from $CURRENT_COLOR to $TARGET_COLOR by ${USER:-unknown}"
 
# Verify target environment is healthy before switching
echo "Verifying $TARGET_COLOR environment health..."
    TARGET_LB = $(aws elbv2 describe - target - groups--names "app-$TARGET_COLOR" --query 'TargetGroups[0].TargetGroupArn' --output text)
    HEALTH = $(aws elbv2 describe - target - health--target - group - arn "$TARGET_LB" --query 'TargetHealthDescriptions[?TargetHealth.State==`healthy`] | length(@)')
 
    if ["$HEALTH" - lt 1]; then
    echo "❌ No healthy targets in $TARGET_COLOR environment. Aborting switch."
    exit 1
    fi
echo "✅ $HEALTH healthy targets in $TARGET_COLOR"
 
# Update load balancer listener
    LISTENER_ARN = $(aws elbv2 describe - listeners--load - balancer - arn "$LB_ARN" --query 'Listeners[?Port==`443`].ListenerArn' --output text)
 
echo "Updating listener to forward to $TARGET_COLOR..."
aws elbv2 modify - listener     --listener - arn "$LISTENER_ARN"     --default -actions Type = forward, TargetGroupArn = "$TARGET_LB"
 
# Update parameter store with new active color
aws ssm put - parameter     --name / app / active - color     --value "$TARGET_COLOR"     --type String     --overwrite
 
# Verify the switch
sleep 5
echo "Verifying traffic switch..."
    ACTIVE_TARGETS = $(aws elbv2 describe - listeners--listener - arns "$LISTENER_ARN" --query 'Listeners[0].DefaultActions[0].TargetGroupArn' --output text)
 
    if [["$ACTIVE_TARGETS" == * "$TARGET_COLOR" * ]]; then
    echo ""
    echo "═══════════════════════════════════════════════════════════"
    echo "✅ Traffic successfully switched to $TARGET_COLOR"
    echo "═══════════════════════════════════════════════════════════"
    echo ""
    echo "Previous environment ($CURRENT_COLOR) is still running."
    echo "To rollback, run: $0 $CURRENT_COLOR"
else
    echo "❌ Switch verification failed"
    exit 1
    fi

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

# Kubernetes HPA for idle environment
# Keeps minimal capacity until pre -switch scale - up
 
apiVersion: autoscaling / v2
    kind: HorizontalPodAutoscaler
    metadata:
    name: app - green - hpa
    spec:
    scaleTargetRef:
    apiVersion: apps / v1
    kind: Deployment
    name: app - green
  
  # Idle: maintain minimal capacity
  # Active: scale based on load
    minReplicas: 2    # Minimal for idle
  maxReplicas: 20   # Full capacity for active
  
  metrics:
        - type: Resource
    resource:
    name: cpu
    target:
    type: Utilization
    averageUtilization: 70
 
    behavior:
    # Scale up quickly for traffic switch
    scaleUp:
      stabilizationWindowSeconds: 0
    policies:
    - type: Percent
    value: 100
    periodSeconds: 15
    # Scale down slowly after becoming idle
    scaleDown:
    stabilizationWindowSeconds: 300
    policies:
    - type: Percent
    value: 10
    periodSeconds: 60