Failover Strategies

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196

// TypeScript: Database Failover Coordinator Pattern
 
interface DatabaseNode {
    id: string;
    host: string;
    port: number;
    role: 'primary' | 'replica' | 'unknown';
    isHealthy: boolean;
    replicationLag: number;  // Seconds behind primary
    lastHeartbeat: number;
}
 
interface FailoverConfig {
    healthCheckIntervalMs: number;
    failureThresholdMs: number;      // Time before declaring failure
    maxReplicationLag: number;        // Max acceptable lag for promotion
    fencingTimeoutMs: number;         // Time to ensure old primary is fenced
    requireQuorum: boolean;           // Require quorum agreement for failover
}
 
class DatabaseFailoverCoordinator {
    private nodes: Map<string, DatabaseNode> = new Map();
    private config: FailoverConfig;
    private currentPrimary: string | null = null;
    
    constructor(config: FailoverConfig) {
        this.config = config;
    }
    
    /**
     * Detect primary failure and initiate failover
     */
    async checkAndFailover(): Promise<{
        failoverOccurred: boolean;
        newPrimary?: string;
        oldPrimary?: string;
        reason?: string;
    }> {
        const primary = this.getCurrentPrimary();
        
        if (!primary) {
            return { failoverOccurred: false, reason: 'No primary configured' };
        }
        
        // Check primary health
        const isHealthy = await this.checkNodeHealth(primary);
        
        if (isHealthy) {
            return { failoverOccurred: false, reason: 'Primary is healthy' };
        }
        
        // Verify failure with quorum if configured
        if (this.config.requireQuorum) {
            const quorumAgreed = await this.checkQuorumAgreement(primary);
            if (!quorumAgreed) {
                console.log('Quorum does not agree on primary failure - possible network partition');
                return { failoverOccurred: false, reason: 'Quorum disagreement' };
            }
        }
        
        // Select best replica for promotion
        const newPrimary = await this.selectPromotionCandidate();
        
        if (!newPrimary) {
            throw new Error('No suitable replica available for promotion');
        }
        
        // Execute failover
        await this.executeFailover(primary, newPrimary);
        
        return {
            failoverOccurred: true,
            newPrimary: newPrimary.id,
            oldPrimary: primary.id,
            reason: 'Primary failure detected and failover completed'
        };
    }
    
    /**
     * Select the best replica to promote to primary
     */
    private async selectPromotionCandidate(): Promise<DatabaseNode | null> {
        const replicas = Array.from(this.nodes.values())
            .filter(n => n.role === 'replica' && n.isHealthy);
        
        if (replicas.length === 0) {
            return null;
        }
        
        // Sort by replication lag (lowest first)
        replicas.sort((a, b) => a.replicationLag - b.replicationLag);
        
        // Find first replica within acceptable lag
        for (const replica of replicas) {
            if (replica.replicationLag <= this.config.maxReplicationLag) {
                return replica;
            }
        }
        
        // If no replica within threshold, return lowest lag replica with warning
        console.warn(`All replicas exceed max replication lag. Selecting ${replicas[0].id} with lag ${replicas[0].replicationLag}s`);
        return replicas[0];
    }
    
    /**
     * Execute the failover sequence
     */
    private async executeFailover(
        oldPrimary: DatabaseNode, 
        newPrimary: DatabaseNode
    ): Promise<void> {
        console.log(`[Failover] Starting: ${oldPrimary.id} → ${newPrimary.id}`);
        
        // Step 1: Fence the old primary (prevent split-brain)
        await this.fenceNode(oldPrimary);
        console.log(`[Failover] Step 1: Old primary ${oldPrimary.id} fenced`);
        
        // Step 2: Wait for new primary to apply any remaining replication
        await this.waitForReplicationCatchup(newPrimary);
        console.log(`[Failover] Step 2: Replication caught up on ${newPrimary.id}`);
        
        // Step 3: Promote new primary
        await this.promoteToMaster(newPrimary);
        console.log(`[Failover] Step 3: ${newPrimary.id} promoted to primary`);
        
        // Step 4: Update topology/discovery service
        await this.updateTopology(newPrimary);
        console.log(`[Failover] Step 4: Topology updated`);
        
        // Step 5: Redirect replicas to new primary
        await this.redirectReplicas(newPrimary);
        console.log(`[Failover] Step 5: Replicas redirected`);
        
        // Update internal state
        oldPrimary.role = 'unknown';
        newPrimary.role = 'primary';
        this.currentPrimary = newPrimary.id;
        
        console.log(`[Failover] Complete: ${newPrimary.id} is now primary`);
    }
    
    /**
     * Fence a node to prevent it from accepting writes (split-brain prevention)
     */
    private async fenceNode(node: DatabaseNode): Promise<void> {
        // Fencing strategies:
        // 1. STONITH (Shoot The Other Node In The Head) - power off the node
        // 2. Revoke write permissions at network level
        // 3. Use a fencing token that must be held to write
        // 4. Instruct the node to enter read-only mode
        
        // Implementation depends on infrastructure
        await this.sendFencingCommand(node);
        
        // Wait for fencing to take effect
        await new Promise(resolve => 
            setTimeout(resolve, this.config.fencingTimeoutMs)
        );
    }
    
    // Infrastructure-specific implementations
    private async checkNodeHealth(node: DatabaseNode): Promise<boolean> {
        // Implement health check logic
        throw new Error('Implementation required');
    }
    
    private async checkQuorumAgreement(primary: DatabaseNode): Promise<boolean> {
        // Ask majority of nodes if they can reach primary
        throw new Error('Implementation required');
    }
    
    private async waitForReplicationCatchup(node: DatabaseNode): Promise<void> {
        throw new Error('Implementation required');
    }
    
    private async promoteToMaster(node: DatabaseNode): Promise<void> {
        throw new Error('Implementation required');
    }
    
    private async updateTopology(newPrimary: DatabaseNode): Promise<void> {
        throw new Error('Implementation required');
    }
    
    private async redirectReplicas(newPrimary: DatabaseNode): Promise<void> {
        throw new Error('Implementation required');
    }
    
    private async sendFencingCommand(node: DatabaseNode): Promise<void> {
        throw new Error('Implementation required');
    }
    
    private getCurrentPrimary(): DatabaseNode | null {
        if (!this.currentPrimary) return null;
        return this.nodes.get(this.currentPrimary) || null;
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

# AWS Route 53 Health Check and Failover Configuration
 
# Primary endpoint health check
AWSTemplateFormatVersion: '2010-09-09'
Resources:
  PrimaryHealthCheck:
    Type: AWS::Route53::HealthCheck
    Properties:
      HealthCheckConfig:
        IPAddress: 192.0.2.1
        Port: 443
        Type: HTTPS
        ResourcePath: /health/ready
        FullyQualifiedDomainName: api.example.com
        RequestInterval: 10        # Check every 10 seconds
        FailureThreshold: 2        # Fail after 2 consecutive failures
        MeasureLatency: true
        EnableSNI: true
      HealthCheckTags:
        - Key: Name
          Value: Primary-DC-Health
 
  # Secondary endpoint health check
  SecondaryHealthCheck:
    Type: AWS::Route53::HealthCheck
    Properties:
      HealthCheckConfig:
        IPAddress: 192.0.2.2
        Port: 443
        Type: HTTPS
        ResourcePath: /health/ready
        FullyQualifiedDomainName: api-secondary.example.com
        RequestInterval: 10
        FailureThreshold: 2
 
  # Failover DNS record for primary
  PrimaryDNSRecord:
    Type: AWS::Route53::RecordSet
    Properties:
      HostedZoneId: Z1234567890ABC
      Name: api.example.com
      Type: A
      TTL: 60                       # Low TTL for faster failover
      SetIdentifier: primary
      Failover: PRIMARY             # Primary in failover policy
      HealthCheckId: !Ref PrimaryHealthCheck
      ResourceRecords:
        - 192.0.2.1
 
  # Failover DNS record for secondary
  SecondaryDNSRecord:
    Type: AWS::Route53::RecordSet
    Properties:
      HostedZoneId: Z1234567890ABC
      Name: api.example.com
      Type: A
      TTL: 60
      SetIdentifier: secondary
      Failover: SECONDARY           # Secondary in failover policy
      HealthCheckId: !Ref SecondaryHealthCheck
      ResourceRecords:
        - 192.0.2.2
 
---
# Global Accelerator for faster failover (bypasses DNS)
Resources:
  GlobalAccelerator:
    Type: AWS::GlobalAccelerator::Accelerator
    Properties:
      Name: api-accelerator
      Enabled: true
      IpAddressType: IPV4
  
  Listener:
    Type: AWS::GlobalAccelerator::Listener
    Properties:
      AcceleratorArn: !Ref GlobalAccelerator
      Protocol: TCP
      PortRanges:
        - FromPort: 443
          ToPort: 443
  
  EndpointGroup:
    Type: AWS::GlobalAccelerator::EndpointGroup
    Properties:
      ListenerArn: !Ref Listener
      EndpointGroupRegion: us-east-1
      HealthCheckIntervalSeconds: 10
      ThresholdCount: 2
      EndpointConfigurations:
        - EndpointId: !Ref PrimaryALB
          Weight: 100
          ClientIPPreservationEnabled: true
        - EndpointId: !Ref SecondaryALB
          Weight: 0                   # Zero weight = standby

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196

// TypeScript: Failback Procedure Automation
 
interface FailbackConfig {
    verifyHealthDurationMs: number;     // How long recovered primary must be healthy
    syncVerificationQuery: string;       // Query to verify data sync completion
    trafficShiftSteps: number[];         // Traffic percentages: [10, 25, 50, 100]
    stepWaitMs: number;                  // Time between traffic shift steps
    rollbackThreshold: number;           // Error rate to trigger rollback (0-1)
}
 
interface FailbackStatus {
    phase: 'not_started' | 'verifying' | 'syncing' | 'shifting' | 'complete' | 'rolled_back';
    currentTrafficPercent: number;
    startTime: number;
    errors: string[];
}
 
class FailbackController {
    private config: FailbackConfig;
    private status: FailbackStatus;
    
    constructor(config: FailbackConfig) {
        this.config = config;
        this.status = {
            phase: 'not_started',
            currentTrafficPercent: 0,
            startTime: 0,
            errors: []
        };
    }
    
    /**
     * Execute controlled failback procedure
     */
    async executeFailback(
        recoveredPrimary: string,
        currentPrimary: string
    ): Promise<FailbackStatus> {
        this.status = {
            phase: 'verifying',
            currentTrafficPercent: 0,
            startTime: Date.now(),
            errors: []
        };
        
        try {
            // Phase 1: Verify recovered primary health
            console.log('[Failback] Phase 1: Verifying recovered primary health');
            await this.verifyHealth(recoveredPrimary);
            
            // Phase 2: Verify data synchronization
            console.log('[Failback] Phase 2: Verifying data synchronization');
            this.status.phase = 'syncing';
            await this.verifySyncComplete(recoveredPrimary, currentPrimary);
            
            // Phase 3: Gradual traffic shift
            console.log('[Failback] Phase 3: Shifting traffic');
            this.status.phase = 'shifting';
            
            for (const targetPercent of this.config.trafficShiftSteps) {
                await this.shiftTraffic(recoveredPrimary, targetPercent);
                this.status.currentTrafficPercent = targetPercent;
                
                // Monitor for errors
                const isHealthy = await this.monitorAfterShift();
                
                if (!isHealthy) {
                    console.log(`[Failback] Error threshold exceeded at ${targetPercent}% - rolling back`);
                    await this.rollback(currentPrimary);
                    this.status.phase = 'rolled_back';
                    return this.status;
                }
                
                console.log(`[Failback] Traffic at ${targetPercent}% - healthy, continuing`);
            }
            
            // Phase 4: Complete - recovered primary is now current primary
            this.status.phase = 'complete';
            console.log('[Failback] Complete - recovered primary is now serving 100% traffic');
            
            // Update topology to reflect new primary
            await this.updateTopologyAfterFailback(recoveredPrimary);
            
        } catch (error) {
            this.status.errors.push(`Failback error: ${error.message}`);
            console.error('[Failback] Error during failback:', error);
            
            // Attempt rollback on any error
            try {
                await this.rollback(currentPrimary);
                this.status.phase = 'rolled_back';
            } catch (rollbackError) {
                this.status.errors.push(`Rollback error: ${rollbackError.message}`);
            }
        }
        
        return this.status;
    }
    
    /**
     * Verify primary is consistently healthy
     */
    private async verifyHealth(primary: string): Promise<void> {
        const startTime = Date.now();
        
        while (Date.now() - startTime < this.config.verifyHealthDurationMs) {
            const isHealthy = await this.checkHealth(primary);
            
            if (!isHealthy) {
                throw new Error(`Recovered primary ${primary} failed health check`);
            }
            
            await this.sleep(5000);  // Check every 5 seconds
        }
        
        console.log(`[Failback] Recovered primary healthy for ${this.config.verifyHealthDurationMs}ms`);
    }
    
    /**
     * Verify data sync is complete (no replication lag)
     */
    private async verifySyncComplete(
        target: string, 
        source: string
    ): Promise<void> {
        const maxAttempts = 60;
        let attempts = 0;
        
        while (attempts < maxAttempts) {
            const lag = await this.getReplicationLag(target, source);
            
            if (lag === 0) {
                console.log('[Failback] Data synchronization complete');
                return;
            }
            
            console.log(`[Failback] Replication lag: ${lag}s - waiting`);
            attempts++;
            await this.sleep(5000);
        }
        
        throw new Error('Replication sync did not complete within timeout');
    }
    
    /**
     * Monitor error rate after traffic shift
     */
    private async monitorAfterShift(): Promise<boolean> {
        // Wait for traffic to stabilize
        await this.sleep(this.config.stepWaitMs);
        
        // Get current error rate
        const errorRate = await this.getCurrentErrorRate();
        
        if (errorRate > this.config.rollbackThreshold) {
            this.status.errors.push(`Error rate ${errorRate} exceeded threshold ${this.config.rollbackThreshold}`);
            return false;
        }
        
        return true;
    }
    
    /**
     * Rollback all traffic to specified primary
     */
    private async rollback(target: string): Promise<void> {
        console.log(`[Failback] Rolling back all traffic to ${target}`);
        await this.shiftTraffic(target, 100);
        this.status.currentTrafficPercent = 0;
    }
    
    // Implementation stubs
    private async checkHealth(primary: string): Promise<boolean> {
        throw new Error('Implementation required');
    }
    
    private async getReplicationLag(target: string, source: string): Promise<number> {
        throw new Error('Implementation required');
    }
    
    private async shiftTraffic(target: string, percent: number): Promise<void> {
        throw new Error('Implementation required');
    }
    
    private async getCurrentErrorRate(): Promise<number> {
        throw new Error('Implementation required');
    }
    
    private async updateTopologyAfterFailback(primary: string): Promise<void> {
        throw new Error('Implementation required');
    }
    
    private sleep(ms: number): Promise<void> {
        return new Promise(resolve => setTimeout(resolve, ms));
    }
}