1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

// Feature flag schema optimized for degradation control
interface DegradationFlag {
  // Identifier
  name: string;
  feature: string;
  
  // Current state
  enabled: boolean;
  
  // Degradation level (for multi-level flags)
  level?: 'full' | 'reduced' | 'minimal' | 'disabled';
  
  // Percentage of traffic to receive degraded experience
  degradationPercentage?: number;
  
  // Conditions that auto-trigger degradation
  autoTriggers?: {
    cpuThreshold?: number;          // Degrade when CPU exceeds
    latencyThreshold?: number;      // Degrade when P99 latency exceeds
    errorRateThreshold?: number;    // Degrade when error rate exceeds
    dependencyHealth?: string[];    // Degrade when these deps are unhealthy
  };
  
  // Who/what can control this flag
  permissions: {
    autoModify: boolean;           // Can system auto-modify
    manualModify: string[];        // Roles that can manually modify
    requiresApproval: boolean;     // Requires second person approval
  };
  
  // Audit trail
  lastModified: Date;
  lastModifiedBy: string;
  modificationReason: string;
}
 
// Example usage
const recommendationFlag: DegradationFlag = {
  name: 'product_recommendations',
  feature: 'recommendations',
  enabled: true,
  level: 'full',
  autoTriggers: {
    cpuThreshold: 85,
    latencyThreshold: 500,
    dependencyHealth: ['ml-model-service', 'user-preference-service']
  },
  permissions: {
    autoModify: true,
    manualModify: ['oncall', 'platform-eng'],
    requiresApproval: false
  },
  lastModified: new Date(),
  lastModifiedBy: 'system',
  modificationReason: 'Initial configuration'
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

class LoadShedder {
  constructor(
    private metrics: MetricsClient,
    private config: LoadShedderConfig
  ) {}
 
  shouldAccept(request: IncomingRequest): SheddingDecision {
    const systemLoad = this.metrics.getCurrentLoad();
    const requestPriority = this.classifyRequest(request);
 
    // Normal operation - accept everything
    if (systemLoad < this.config.normalThreshold) {
      return { accept: true, reason: 'normal_operation' };
    }
 
    // Elevated load - shed lowest priority
    if (systemLoad < this.config.elevatedThreshold) {
      if (requestPriority === 'low') {
        return { accept: false, reason: 'shedding_low_priority', retryAfter: 30 };
      }
      return { accept: true, reason: 'elevated_priority_pass' };
    }
 
    // High load - shed low and medium priority
    if (systemLoad < this.config.criticalThreshold) {
      if (requestPriority === 'low' || requestPriority === 'medium') {
        return { accept: false, reason: 'shedding_medium_priority', retryAfter: 60 };
      }
      return { accept: true, reason: 'high_load_priority_pass' };
    }
 
    // Critical load - only accept highest priority
    if (requestPriority !== 'critical') {
      return { accept: false, reason: 'critical_load_shedding', retryAfter: 120 };
    }
    return { accept: true, reason: 'critical_priority_pass' };
  }
 
  private classifyRequest(request: IncomingRequest): 'critical' | 'high' | 'medium' | 'low' {
    // Critical: Health checks, payment completions
    if (request.path.startsWith('/health') || 
        request.path.includes('/payment/complete')) {
      return 'critical';
    }
 
    // High: Authenticated user core actions
    if (request.authenticated && 
        (request.path.includes('/cart') || request.path.includes('/checkout'))) {
      return 'high';
    }
 
    // Medium: Authenticated user browsing
    if (request.authenticated) {
      return 'medium';
    }
 
    // Low: Anonymous browsing, bots
    return 'low';
  }
}
 
interface SheddingDecision {
  accept: boolean;
  reason: string;
  retryAfter?: number;  // Seconds before client should retry
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

class DegradationController {
  async evaluateDegradation(systemState: SystemState): Promise<DegradationAction[]> {
    const actions: DegradationAction[] = [];
 
    for (const feature of this.features) {
      const shouldDegrade = this.evaluateConditions(feature, systemState);
      
      if (!shouldDegrade) continue;
 
      switch (feature.degradationType) {
        case 'automatic_silent':
          // Tier 4: Just do it, log for audit
          await this.applyDegradation(feature);
          actions.push({ feature: feature.name, action: 'degraded', approval: 'auto' });
          break;
 
        case 'automatic_notify':
          // Tier 3: Do it, but notify operators
          await this.applyDegradation(feature);
          await this.notifyOperators(feature, systemState);
          actions.push({ feature: feature.name, action: 'degraded', approval: 'auto_notified' });
          break;
 
        case 'recommend':
          // Tier 2: Don't do it, but recommend to operators
          await this.createRecommendation(feature, systemState);
          actions.push({ feature: feature.name, action: 'recommended', approval: 'pending' });
          break;
 
        case 'manual_only':
          // Tier 1: Only operators can degrade
          // Log that conditions were met, no action taken
          await this.logConditionMet(feature, systemState);
          break;
      }
    }
 
    return actions;
  }
 
  // Manual override - can be used to force degradation or prevent it
  async manualOverride(featureName: string, action: 'degrade' | 'restore' | 'lock', 
                       operator: string, reason: string): Promise<void> {
    const feature = this.getFeature(featureName);
    
    // Audit trail
    await this.auditLog.record({
      feature: featureName,
      action,
      operator,
      reason,
      timestamp: new Date(),
      previousState: feature.currentState
    });
 
    switch (action) {
      case 'degrade':
        await this.applyDegradation(feature);
        break;
      case 'restore':
        await this.restoreFeature(feature);
        break;
      case 'lock':
        await this.lockFeature(feature);  // Prevent auto-degradation
        break;
    }
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

interface FeatureDependencyConfig {
  featureName: string;
  
  dependencies: {
    service: string;
    required: boolean;  // Feature fails if this does, or degrades?
    healthCheck: string;  // Endpoint or metric to check health
    
    // What to do when this dependency fails
    onFailure: {
      action: 'disable_feature' | 'degrade_feature' | 'use_fallback';
      degradedBehavior?: string;  // Description of degraded mode
      fallbackSource?: string;    // Where to get fallback data
    };
  }[];
}
 
// Example: Product recommendations feature
const recommendationsConfig: FeatureDependencyConfig = {
  featureName: 'personalized_recommendations',
  dependencies: [
    {
      service: 'ml-model-serving',
      required: false,  // Can degrade
      healthCheck: '/health',
      onFailure: {
        action: 'degrade_feature',
        degradedBehavior: 'Show trending items instead of personalized',
        fallbackSource: 'trending-service'
      }
    },
    {
      service: 'user-preferences',
      required: false,
      healthCheck: '/health',
      onFailure: {
        action: 'use_fallback',
        fallbackSource: 'cached-preferences',
        degradedBehavior: 'Use cached preferences up to 24h old'
      }
    },
    {
      service: 'product-catalog',
      required: true,  // Can't show anything without products
      healthCheck: '/health',
      onFailure: {
        action: 'disable_feature',
        degradedBehavior: 'Hide recommendations section entirely'
      }
    }
  ]
};
 
// Degradation controller uses this config
class DependencyAwareDegradation {
  async evaluateFeature(featureConfig: FeatureDependencyConfig): Promise<FeatureState> {
    const unhealthyDeps = await this.checkDependencies(featureConfig.dependencies);
    
    if (unhealthyDeps.length === 0) {
      return { status: 'healthy', mode: 'full' };
    }
 
    // Check if any required deps are unhealthy
    const requiredUnhealthy = unhealthyDeps.filter(d => d.required);
    if (requiredUnhealthy.length > 0) {
      return { status: 'disabled', mode: 'off', reason: `Required dependency ${requiredUnhealthy[0].service} unavailable` };
    }
 
    // Degrade based on optional dependency failures
    const degradations = unhealthyDeps.map(d => d.onFailure.degradedBehavior);
    return { status: 'degraded', mode: 'degraded', degradations };
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

# Feature Degradation Runbook: Personalized Recommendations
 
## Overview
The recommendations feature shows personalized product suggestions based on 
user history and ML models. It can be degraded to show trending items instead,
or disabled entirely.
 
## When to Degrade
- ML model service latency > 500ms P99 for > 5 minutes
- ML model service error rate > 5% for > 2 minutes  
- System CPU > 85% and recommendations contributing to load
- During planned high-traffic events (preemptive degradation)
 
## Degradation Commands
 
### To degraded mode (trending items):
```bash
# Via CLI
feature-flag set recommendations.mode degraded --reason "incident-123"
 
# Via Admin UI
Navigate to: admin.example.com/feature-flags/recommendations
Set Mode: degraded
Add Reason: [your reason]
Click: Apply Changes
```
 
### To disabled:
```bash
feature-flag set recommendations.enabled false --reason "incident-123"
```
 
## Verification
1. Visit any product page as logged-in user
2. Recommendations section should show "Trending Products" header (not "For You")
3. Check metrics: `recommendations.source` should show 'trending' not 'personalized'
4. Verify no errors in logs: `grep "recommendations" /var/log/app.log`
 
## User Impact
- Users see generic trending products instead of personalized
- Expected impact: ~2% reduction in add-to-cart rate from recommendations
- No user complaints expected (section still shows products)
 
## Recovery
1. Verify ML service is healthy (latency < 200ms P99, error rate < 1%)
2. Set flag: `feature-flag set recommendations.mode full`
3. Verify personalized recommendations returning (check metrics)
4. Monitor for 10 minutes for any issues
 
## Escalation
- Primary: #platform-eng Slack channel
- Secondary: @recommendations-team-oncall
- Emergency: Page platform-eng-manager