Disaster Recovery: Building Resilient Systems That Survive Catastrophe

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169

// RTO Component Analysis
// Breaking down recovery time into measurable phases
 
interface RTOBreakdown {
  phase: string;
  minTime: number;    // minutes, optimistic case
  avgTime: number;    // minutes, typical case
  maxTime: number;    // minutes, worst case
  factors: string[];
  optimizations: string[];
}
 
const rtoComponents: RTOBreakdown[] = [
  {
    phase: 'Detection',
    minTime: 1,
    avgTime: 5,
    maxTime: 30,
    factors: [
      'Monitoring coverage and sensitivity',
      'Alert routing and acknowledgment',
      'False positive rate (alert fatigue)',
      'Time of day (3 AM vs business hours)'
    ],
    optimizations: [
      'Implement comprehensive health checks',
      'Reduce false positives to combat alert fatigue',
      'Use on-call rotation with defined response SLAs',
      'Automated anomaly detection'
    ]
  },
  {
    phase: 'Assessment & Declaration',
    minTime: 5,
    avgTime: 15,
    maxTime: 60,
    factors: [
      'Clarity of disaster declaration criteria',
      'Authority to declare (who can decide)',
      'Communication channel availability',
      'Assessment of failure scope'
    ],
    optimizations: [
      'Pre-define objective declaration criteria',
      'Empower on-call to declare without escalation',
      'Maintain out-of-band communication channels',
      'Regular drills to speed decision-making'
    ]
  },
  {
    phase: 'Recovery Initiation',
    minTime: 5,
    avgTime: 20,
    maxTime: 60,
    factors: [
      'Access to DR systems and credentials',
      'Availability of required personnel',
      'Clarity of initial recovery steps',
      'State of DR infrastructure'
    ],
    optimizations: [
      'Pre-staged credentials in secure vault',
      'Automated DR environment warmup',
      'Runbooks with one-click initiation',
      'Regular DR site health verification'
    ]
  },
  {
    phase: 'Data Recovery / Sync',
    minTime: 10,
    avgTime: 60,
    maxTime: 480, // 8 hours
    factors: [
      'Volume of data to restore',
      'Replication lag at failure time',
      'Backup restore bandwidth',
      'Data validation requirements'
    ],
    optimizations: [
      'Continuous replication vs periodic backup',
      'Parallel restore across multiple streams',
      'Incremental restore for large datasets',
      'Pre-validated data integrity checks'
    ]
  },
  {
    phase: 'Application Startup',
    minTime: 5,
    avgTime: 20,
    maxTime: 120,
    factors: [
      'Application complexity and dependencies',
      'Configuration synchronization',
      'Cache warming requirements',
      'Health check passage'
    ],
    optimizations: [
      'Minimize cold-start dependencies',
      'Infrastructure-as-Code for consistent config',
      'Pre-populated caches or lazy loading',
      'Parallel service startup where possible'
    ]
  },
  {
    phase: 'Verification',
    minTime: 10,
    avgTime: 30,
    maxTime: 120,
    factors: [
      'Comprehensiveness of health checks',
      'Integration test coverage',
      'Manual verification requirements',
      'Smoke testing external integrations'
    ],
    optimizations: [
      'Automated verification playbooks',
      'Synthetic transaction testing',
      'Staged traffic rollout (canary)',
      'Pre-defined verification checklist'
    ]
  },
  {
    phase: 'Traffic Cutover',
    minTime: 1,
    avgTime: 10,
    maxTime: 30,
    factors: [
      'DNS TTL values',
      'Load balancer configuration',
      'Client caching behavior',
      'Geographic distribution of traffic'
    ],
    optimizations: [
      'Low TTL DNS records for DR-critical paths',
      'Global load balancer with health checks',
      'Edge CDN with failover configuration',
      'Mobile app capability for endpoint switching'
    ]
  }
];
 
function calculateTotalRTO(
  scenario: 'optimistic' | 'typical' | 'worst'
): { totalMinutes: number; breakdown: string[] } {
  
  const timeKey = scenario === 'optimistic' ? 'minTime' 
                : scenario === 'worst' ? 'maxTime' 
                : 'avgTime';
  
  let total = 0;
  const breakdown: string[] = [];
  
  for (const component of rtoComponents) {
    const time = component[timeKey];
    total += time;
    breakdown.push(`${component.phase}: ${time} minutes`);
  }
  
  return { totalMinutes: total, breakdown };
}
 
// Example analysis
const typical = calculateTotalRTO('typical');
console.log(`Typical Total RTO: ${typical.totalMinutes} minutes (${(typical.totalMinutes / 60).toFixed(1)} hours)`);
// Output: Typical Total RTO: 160 minutes (2.7 hours)
 
const worst = calculateTotalRTO('worst');
console.log(`Worst-case Total RTO: ${worst.totalMinutes} minutes (${(worst.totalMinutes / 60).toFixed(1)} hours)`);
// Output: Worst-case Total RTO: 900 minutes (15 hours)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

// RPO/RTO Target Setting Framework
// Systematic approach to determining appropriate recovery objectives
 
interface BusinessImpactAssessment {
  systemName: string;
  
  // Financial impact per hour of downtime
  revenueImpactPerHour: number;
  operationalCostPerHour: number;
  slaExposurePerHour: number;  // SLA penalties
  
  // Data loss impact (per hour of lost data)
  transactionValueAtRisk: number;
  reconciliationCostPerHour: number;
  regulatoryExposure: number;
  
  // Qualitative factors (1-5 scale)
  customerVisibility: number;      // How visible is downtime to customers?
  competitiveRisk: number;         // Can customers switch to competitors easily?
  reputationalSensitivity: number; // Brand impact of publicized outage
  regulatoryScrutiny: number;      // Level of regulatory attention
}
 
interface TechnicalFeasibility {
  targetRTO: number;  // hours
  targetRPO: number;  // hours
  
  architectureRequired: string;
  estimatedCost: {
    capex: number;
    monthlyOpex: number;
  };
  complexity: 'low' | 'medium' | 'high' | 'very-high';
  constraints: string[];
}
 
function calculateOptimalTargets(impact: BusinessImpactAssessment): {
  recommendedRTO: number;
  recommendedRPO: number;
  maxAcceptableRTO: number;
  maxAcceptableRPO: number;
  justification: string;
} {
  // Calculate total hourly impact
  const hourlyDowntimeImpact = 
    impact.revenueImpactPerHour + 
    impact.operationalCostPerHour + 
    impact.slaExposurePerHour;
  
  const hourlyDataLossImpact = 
    impact.transactionValueAtRisk + 
    impact.reconciliationCostPerHour + 
    impact.regulatoryExposure;
  
  // Qualitative multiplier based on visibility/risk factors
  const qualitativeFactor = (
    impact.customerVisibility +
    impact.competitiveRisk +
    impact.reputationalSensitivity +
    impact.regulatoryScrutiny
  ) / 20;  // Normalized to 0-1 range
  
  // Determine RTO threshold where impact becomes 'unacceptable'
  // Rule of thumb: If qualitative factors are high, tolerance is lower
  let maxAcceptableRTO: number;
  if (qualitativeFactor > 0.7) {
    // High visibility/risk - aggressive RTO
    maxAcceptableRTO = Math.max(0.25, 50000 / hourlyDowntimeImpact);
  } else if (qualitativeFactor > 0.4) {
    // Moderate visibility/risk
    maxAcceptableRTO = Math.max(1, 100000 / hourlyDowntimeImpact);
  } else {
    // Lower visibility - more flexibility
    maxAcceptableRTO = Math.max(4, 200000 / hourlyDowntimeImpact);
  }
  
  // Determine RPO threshold
  let maxAcceptableRPO: number;
  if (hourlyDataLossImpact > 100000 || impact.regulatoryExposure > 0) {
    maxAcceptableRPO = 0.25;  // 15 minutes
  } else if (hourlyDataLossImpact > 10000) {
    maxAcceptableRPO = 1;     // 1 hour
  } else {
    maxAcceptableRPO = 4;     // 4 hours
  }
  
  // Recommendation is typically more aggressive than maximum acceptable
  // Build in safety margin
  const recommendedRTO = maxAcceptableRTO * 0.5;
  const recommendedRPO = maxAcceptableRPO * 0.5;
  
  return {
    recommendedRTO: Math.round(recommendedRTO * 100) / 100,
    recommendedRPO: Math.round(recommendedRPO * 100) / 100,
    maxAcceptableRTO: Math.round(maxAcceptableRTO * 100) / 100,
    maxAcceptableRPO: Math.round(maxAcceptableRPO * 100) / 100,
    justification: `Based on $${hourlyDowntimeImpact.toLocaleString()}/hour downtime impact, ` +
                   `$${hourlyDataLossImpact.toLocaleString()}/hour data loss risk, ` +
                   `and qualitative risk factor of ${(qualitativeFactor * 100).toFixed(0)}%`
  };
}
 
// Example: E-commerce order management system
const orderSystemImpact: BusinessImpactAssessment = {
  systemName: 'Order Management System',
  revenueImpactPerHour: 150000,
  operationalCostPerHour: 10000,
  slaExposurePerHour: 5000,
  transactionValueAtRisk: 200000,
  reconciliationCostPerHour: 25000,
  regulatoryExposure: 0,
  customerVisibility: 5,
  competitiveRisk: 4,
  reputationalSensitivity: 4,
  regulatoryScrutiny: 2
};
 
const targets = calculateOptimalTargets(orderSystemImpact);
console.log(targets);
// Output:
// {
//   recommendedRTO: 0.15,      // ~9 minutes
//   recommendedRPO: 0.12,      // ~7 minutes
//   maxAcceptableRTO: 0.30,    // ~18 minutes
//   maxAcceptableRPO: 0.25,    // 15 minutes
//   justification: "Based on $165,000/hour downtime impact..."
// }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166

// Comprehensive RPO/RTO Monitoring System
 
interface RPOMetric {
  systemId: string;
  targetRPOSeconds: number;
  currentLagSeconds: number;
  lagTrend: 'improving' | 'stable' | 'degrading';
  lastMeasuredAt: Date;
  measurementMethod: 'replication_lag' | 'synthetic_write' | 'snapshot_age';
}
 
interface RTOMetric {
  systemId: string;
  targetRTOMinutes: number;
  estimatedRTOMinutes: number;  // Based on component readiness
  lastTestedRTOMinutes: number; // From most recent DR test
  lastTestDate: Date;
  componentReadiness: {
    component: string;
    status: 'ready' | 'degraded' | 'unavailable';
    estimatedRecoveryMinutes: number;
  }[];
}
 
class DRCapabilityMonitor {
  private rpoTargets: Map<string, number> = new Map();
  private rtoTargets: Map<string, number> = new Map();
  
  async measureActualRPO(systemId: string): Promise<RPOMetric> {
    // Method 1: Query replication lag directly
    const replicationLag = await this.queryReplicationLag(systemId);
    
    // Method 2: Synthetic write test
    // Write a timestamped record to primary, read from replica
    const syntheticLag = await this.performSyntheticWriteTest(systemId);
    
    // Use the more conservative (higher) measurement
    const actualLag = Math.max(replicationLag, syntheticLag);
    
    return {
      systemId,
      targetRPOSeconds: this.rpoTargets.get(systemId) || 0,
      currentLagSeconds: actualLag,
      lagTrend: this.calculateTrend(systemId, actualLag),
      lastMeasuredAt: new Date(),
      measurementMethod: 'synthetic_write'  // Most accurate method
    };
  }
  
  async assessRTOReadiness(systemId: string): Promise<RTOMetric> {
    // Check each component required for recovery
    const components = await this.getRecoveryComponents(systemId);
    const componentAssessments = await Promise.all(
      components.map(async (component) => ({
        component: component.name,
        status: await this.checkComponentStatus(component),
        estimatedRecoveryMinutes: await this.estimateComponentRecovery(component)
      }))
    );
    
    // RTO is approximately the maximum of component recovery times
    // (assuming some parallelism but sequential critical path)
    const estimatedRTO = this.calculateCriticalPath(componentAssessments);
    
    // Get last actual test result
    const lastTest = await this.getLastDRTestResult(systemId);
    
    return {
      systemId,
      targetRTOMinutes: this.rtoTargets.get(systemId) || 0,
      estimatedRTOMinutes: estimatedRTO,
      lastTestedRTOMinutes: lastTest.actualRTOMinutes,
      lastTestDate: lastTest.testDate,
      componentReadiness: componentAssessments
    };
  }
  
  generateDashboard(): DRDashboard {
    return {
      overallStatus: this.calculateOverallStatus(),
      rpoViolations: this.getCurrentRPOViolations(),
      rtoRisks: this.getCurrentRTORisks(),
      upcomingTests: this.getScheduledDRTests(),
      recommendations: this.generateRecommendations()
    };
  }
  
  // Alert Rules
  checkAlerts(): Alert[] {
    const alerts: Alert[] = [];
    
    for (const [systemId, targetRPO] of this.rpoTargets) {
      const current = this.getLatestRPOMeasurement(systemId);
      
      if (current.currentLagSeconds >= targetRPO) {
        alerts.push({
          severity: 'CRITICAL',
          systemId,
          type: 'RPO_VIOLATION',
          message: `RPO VIOLATED: ${systemId} lag is ${current.currentLagSeconds}s, target is ${targetRPO}s`,
          requiredAction: 'Investigate replication immediately'
        });
      } else if (current.currentLagSeconds >= targetRPO * 0.8) {
        alerts.push({
          severity: 'WARNING',
          systemId,
          type: 'RPO_THRESHOLD',
          message: `RPO at risk: ${systemId} lag is ${(current.currentLagSeconds / targetRPO * 100).toFixed(0)}% of target`,
          requiredAction: 'Monitor closely, prepare for intervention'
        });
      }
    }
    
    // RTO alerts based on stale testing
    for (const [systemId, _] of this.rtoTargets) {
      const readiness = this.getLatestRTOAssessment(systemId);
      const daysSinceTest = this.daysSince(readiness.lastTestDate);
      
      if (daysSinceTest > 90) {
        alerts.push({
          severity: 'WARNING',
          systemId,
          type: 'RTO_STALE',
          message: `RTO unvalidated: No DR test for ${systemId} in ${daysSinceTest} days`,
          requiredAction: 'Schedule DR test'
        });
      }
    }
    
    return alerts;
  }
  
  // Helper methods (implementations omitted for brevity)
  private async queryReplicationLag(systemId: string): Promise<number> { /* ... */ return 0; }
  private async performSyntheticWriteTest(systemId: string): Promise<number> { /* ... */ return 0; }
  private calculateTrend(systemId: string, currentLag: number): 'improving' | 'stable' | 'degrading' { /* ... */ return 'stable'; }
  private async getRecoveryComponents(systemId: string): Promise<any[]> { /* ... */ return []; }
  private async checkComponentStatus(component: any): Promise<'ready' | 'degraded' | 'unavailable'> { /* ... */ return 'ready'; }
  private async estimateComponentRecovery(component: any): Promise<number> { /* ... */ return 0; }
  private calculateCriticalPath(components: any[]): number { /* ... */ return 0; }
  private async getLastDRTestResult(systemId: string): Promise<any> { /* ... */ return { actualRTOMinutes: 0, testDate: new Date() }; }
  private calculateOverallStatus(): string { return 'healthy'; }
  private getCurrentRPOViolations(): any[] { return []; }
  private getCurrentRTORisks(): any[] { return []; }
  private getScheduledDRTests(): any[] { return []; }
  private generateRecommendations(): string[] { return []; }
  private getLatestRPOMeasurement(systemId: string): any { return { currentLagSeconds: 0 }; }
  private getLatestRTOAssessment(systemId: string): any { return { lastTestDate: new Date() }; }
  private daysSince(date: Date): number { return Math.floor((Date.now() - date.getTime()) / 86400000); }
}
 
interface Alert {
  severity: 'CRITICAL' | 'WARNING' | 'INFO';
  systemId: string;
  type: string;
  message: string;
  requiredAction: string;
}
 
interface DRDashboard {
  overallStatus: string;
  rpoViolations: any[];
  rtoRisks: any[];
  upcomingTests: any[];
  recommendations: string[];
}