Error Rate SLIs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163

interface RequestOutcome {
  httpStatus: number;
  latencyMs: number;
  wasTimeout: boolean;
  endpoint: string;
  errorCode?: string;  // Application-level error code
  clientType: 'external' | 'internal' | 'synthetic';
}
 
interface ErrorClassification {
  isError: boolean;
  errorCategory: 'server' | 'client' | 'timeout' | 'semantic' | 'none';
  severity: 'critical' | 'major' | 'minor' | 'cosmetic' | 'none';
  includeInSli: boolean;
  reason: string;
}
 
function classifyError(outcome: RequestOutcome): ErrorClassification {
  // Timeouts are always errors from user perspective
  if (outcome.wasTimeout) {
    return {
      isError: true,
      errorCategory: 'timeout',
      severity: 'critical',
      includeInSli: true,
      reason: 'User waited and received no response',
    };
  }
 
  // Server errors (5xx) are always errors
  if (outcome.httpStatus >= 500) {
    return {
      isError: true,
      errorCategory: 'server',
      severity: determineSeverityByEndpoint(outcome.endpoint),
      includeInSli: true,
      reason: `Server error: HTTP ${outcome.httpStatus}`,
    };
  }
 
  // Rate limiting: track separately, don't count against error rate SLI
  if (outcome.httpStatus === 429) {
    return {
      isError: false,
      errorCategory: 'client',
      severity: 'none',
      includeInSli: false,
      reason: 'Rate limiting is protective behavior',
    };
  }
 
  // Auth errors (401, 403): typically not errors unless unexpected
  if (outcome.httpStatus === 401 || outcome.httpStatus === 403) {
    return {
      isError: false,
      errorCategory: 'client',
      severity: 'none',
      includeInSli: false,
      reason: 'Authentication/authorization working as expected',
    };
  }
 
  // 400 Bad Request: nuanced classification
  if (outcome.httpStatus === 400) {
    // If from our own client app, might be a bug on our side
    if (outcome.clientType === 'internal') {
      return {
        isError: true,
        errorCategory: 'client',
        severity: 'major',
        includeInSli: true,
        reason: 'Bad request from internal client suggests a bug',
      };
    }
    // External clients: don't count against our SLI
    return {
      isError: false,
      errorCategory: 'client',
      severity: 'none',
      includeInSli: false,
      reason: 'Bad request from external client',
    };
  }
 
  // 404: depends on context
  if (outcome.httpStatus === 404) {
    // Check if this endpoint should exist
    if (isKnownEndpoint(outcome.endpoint)) {
      return {
        isError: true,
        errorCategory: 'server',
        severity: 'major',
        includeInSli: true,
        reason: 'Valid endpoint returned 404 - likely a bug or data issue',
      };
    }
    return {
      isError: false,
      errorCategory: 'client',
      severity: 'none',
      includeInSli: false,
      reason: 'Unknown endpoint - likely user error or old link',
    };
  }
 
  // Semantic errors: HTTP 2xx but application-level failure
  if (outcome.httpStatus >= 200 && outcome.httpStatus < 300) {
    if (outcome.errorCode) {
      return {
        isError: true,
        errorCategory: 'semantic',
        severity: determineSeverityByErrorCode(outcome.errorCode),
        includeInSli: true,
        reason: `Semantic error: ${outcome.errorCode}`,
      };
    }
  }
 
  // Success
  return {
    isError: false,
    errorCategory: 'none',
    severity: 'none',
    includeInSli: false,
    reason: 'Request succeeded',
  };
}
 
function calculateErrorRate(outcomes: RequestOutcome[]): {
  totalRequests: number;
  errorCount: number;
  errorRate: number;
  successRate: number;
  byCategory: Record<string, number>;
} {
  let errorCount = 0;
  const byCategory: Record<string, number> = {};
  
  const eligibleOutcomes = outcomes.filter(o => 
    o.clientType !== 'synthetic'  // Exclude synthetic monitoring
  );
 
  for (const outcome of eligibleOutcomes) {
    const classification = classifyError(outcome);
    if (classification.includeInSli && classification.isError) {
      errorCount++;
      byCategory[classification.errorCategory] = 
        (byCategory[classification.errorCategory] || 0) + 1;
    }
  }
 
  const total = eligibleOutcomes.length;
  const errorRate = total > 0 ? (errorCount / total) * 100 : 0;
  const successRate = 100 - errorRate;
 
  return {
    totalRequests: total,
    errorCount,
    errorRate,
    successRate,
    byCategory,
  };
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

from dataclasses import dataclass
from typing import List, Optional, Dict, Any
import json
 
@dataclass
class SemanticValidationResult:
    is_valid: bool
    errors: List[str]
    warnings: List[str]
 
class SemanticValidator:
    """
    Validates responses for semantic correctness beyond HTTP status codes.
    """
    
    def validate_search_response(
        self, 
        query: str, 
        response: Dict[str, Any]
    ) -> SemanticValidationResult:
        """Validate search results are semantically correct."""
        errors = []
        warnings = []
        
        results = response.get('results', [])
        query_terms = query.lower().split()
        
        # Check: Results should be relevant to query
        if results and len(query_terms) > 0:
            relevance_scores = []
            for result in results[:10]:  # Check top 10
                title = result.get('title', '').lower()
                description = result.get('description', '').lower()
                content = f"{title} {description}"
                
                term_matches = sum(1 for term in query_terms if term in content)
                relevance_scores.append(term_matches / len(query_terms))
            
            avg_relevance = sum(relevance_scores) / len(relevance_scores)
            if avg_relevance < 0.2:  # Less than 20% term coverage
                errors.append(
                    f"Search results appear irrelevant: avg_relevance={avg_relevance:.2f}"
                )
        
        # Check: Result count should match total claimed
        claimed_total = response.get('total_results', 0)
        if len(results) > claimed_total:
            errors.append(
                f"Result count mismatch: returned {len(results)} but claimed total {claimed_total}"
            )
        
        # Check: Results should have required fields
        for i, result in enumerate(results):
            if not result.get('id'):
                errors.append(f"Result {i} missing 'id' field")
            if not result.get('title'):
                warnings.append(f"Result {i} missing 'title' field")
        
        return SemanticValidationResult(
            is_valid=len(errors) == 0,
            errors=errors,
            warnings=warnings,
        )
 
    def validate_order_response(
        self, 
        order_request: Dict[str, Any],
        response: Dict[str, Any]
    ) -> SemanticValidationResult:
        """Validate order confirmation is semantically correct."""
        errors = []
        warnings = []
        
        # Check: Order ID should be returned
        if not response.get('order_id'):
            errors.append("Order confirmation missing order_id")
        
        # Check: Totals should be consistent
        items = response.get('items', [])
        calculated_subtotal = sum(
            item.get('price', 0) * item.get('quantity', 0) 
            for item in items
        )
        claimed_subtotal = response.get('subtotal', 0)
        
        if abs(calculated_subtotal - claimed_subtotal) > 0.01:
            errors.append(
                f"Subtotal mismatch: calculated {calculated_subtotal}, claimed {claimed_subtotal}"
            )
        
        # Check: Total = Subtotal + Tax + Shipping - Discounts
        claimed_total = response.get('total', 0)
        expected_total = (
            claimed_subtotal + 
            response.get('tax', 0) + 
            response.get('shipping', 0) - 
            response.get('discount', 0)
        )
        
        if abs(expected_total - claimed_total) > 0.01:
            errors.append(
                f"Total calculation error: expected {expected_total}, got {claimed_total}"
            )
        
        # Check: Items match request
        requested_items = {item['sku']: item['quantity'] 
                         for item in order_request.get('items', [])}
        confirmed_items = {item['sku']: item['quantity'] 
                         for item in items}
        
        if requested_items != confirmed_items:
            errors.append("Confirmed items don't match requested items")
        
        return SemanticValidationResult(
            is_valid=len(errors) == 0,
            errors=errors,
            warnings=warnings,
        )
 
# Usage in request pipeline
validator = SemanticValidator()
 
def track_semantic_error_rate(response, operation_type, request_data=None):
    """
    Validate response and track semantic error rate.
    """
    if operation_type == 'search':
        result = validator.validate_search_response(
            request_data.get('query', ''),
            response
        )
    elif operation_type == 'order':
        result = validator.validate_order_response(request_data, response)
    else:
        return  # No validation for this operation type
    
    if not result.is_valid:
        # Increment semantic error counter
        metrics.increment('semantic_errors', 
                         tags={'operation': operation_type})
        
        # Log for investigation
        logger.error(f"Semantic validation failed: {result.errors}",
                    extra={'response': response, 'errors': result.errors})

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

interface BurnRateAlert {
  shouldAlert: boolean;
  burnRate: number;
  currentErrorRate: number;
  budgetRemaining: number;
  hoursUntilBudgetExhausted: number;
  severity: 'critical' | 'warning' | 'none';
}
 
function evaluateErrorBudgetBurning(
  currentErrorRate: number,      // Current error rate (0-1, e.g., 0.02 = 2%)
  sloTarget: number,             // SLO target (0-1, e.g., 0.999 = 99.9%)
  windowHours: number,           // Current measurement window
  budgetConsumedSoFar: number,   // Portion of monthly budget already used (0-1)
  monthlyBudgetHours: number = 720,  // Hours in SLO period (30 days)
): BurnRateAlert {
  
  // Error budget = allowed error rate = 1 - SLO
  const allowedErrorRate = 1 - sloTarget;  // e.g., 0.001 for 99.9% SLO
  
  // Burn rate = (current error rate) / (allowed error rate)
  // Burn rate of 1 = using budget at exact expected pace
  // Burn rate of 10 = burning 10x faster than sustainable
  const burnRate = currentErrorRate / allowedErrorRate;
  
  // Budget remaining
  const budgetRemaining = 1 - budgetConsumedSoFar;
  
  // At current burn rate, how long until budget exhausted?
  // If burn rate is 10 and we have 30 days budget, exhausted in 3 days
  const hoursUntilExhausted = burnRate > 0 
    ? (budgetRemaining * monthlyBudgetHours) / burnRate 
    : Infinity;
  
  // Alert thresholds based on Google SRE multi-window approach
  let shouldAlert = false;
  let severity: 'critical' | 'warning' | 'none' = 'none';
  
  // Critical: Burning very fast, will exhaust budget soon
  if (burnRate >= 14.4 && windowHours >= 1/60) {  // 14.4x for 1 minute
    shouldAlert = true;
    severity = 'critical';
  } else if (burnRate >= 6 && windowHours >= 0.25) {  // 6x for 15 minutes
    shouldAlert = true;
    severity = 'critical';
  }
  // Warning: Elevated burn rate
  else if (burnRate >= 3 && windowHours >= 1) {  // 3x for 1 hour
    shouldAlert = true;
    severity = 'warning';
  } else if (burnRate >= 1 && windowHours >= 6) {  // 1x for 6 hours
    shouldAlert = true;
    severity = 'warning';
  }
  
  return {
    shouldAlert,
    burnRate,
    currentErrorRate,
    budgetRemaining,
    hoursUntilBudgetExhausted: hoursUntilExhausted,
    severity,
  };
}
 
// Example usage
const result = evaluateErrorBudgetBurning(
  0.02,    // 2% current error rate
  0.999,   // 99.9% SLO target
  0.25,    // 15-minute window
  0.3,     // 30% of monthly budget already used
);
 
if (result.shouldAlert) {
  console.log(`ALERT (${result.severity}): Burn rate ${result.burnRate.toFixed(1)}x`);
  console.log(`Budget exhausted in ${result.hoursUntilBudgetExhausted.toFixed(1)} hours`);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

# Error Rate SLI Specification: Customer-Facing API
sli:
  name: "API Success Rate"
  description: "Proportion of API requests that succeed"
  
  success_definition:
    http_status:
      success: [200, 201, 204, 304]  # Standard success codes
      error: [500, 501, 502, 503, 504]  # Server errors
      excluded:
        - 429  # Rate limiting (tracked separately)
        - 401  # Authentication expected behavior
        - 403  # Authorization expected behavior
        - 404  # Only counted as error for known resource paths
    additional_error_conditions:
      - "Request timeout (client or server)"
      - "Connection reset before response"
      - "Response body contains 'error' with non-null value"
  
  scope:
    endpoints: "/api/v*/**"
    traffic_sources:
      include: ["external_users", "mobile_app", "web_app"]
      exclude: ["synthetic_monitoring", "internal_services"]
  
  target:
    success_rate: 99.9%
    error_rate: 0.1%
    budget_period: "30 days rolling"
    
  alerting:
    burn_rate_alerts:
      - window: "5 minutes"
        burn_rate_threshold: 14.4
        severity: "critical"
      - window: "1 hour"
        burn_rate_threshold: 6
        severity: "critical"
      - window: "6 hours"
        burn_rate_threshold: 3
        severity: "warning"
    minimum_request_count: 100
    
  reporting:
    dashboards: ["service-health", "slo-compliance"]
    weekly_review: true
    monthly_report: true

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

# Error Rate SLI Specification: Payment Transactions
sli:
  name: "Transaction Success Rate"
  description: "Proportion of payment transactions that complete successfully"
  
  success_definition:
    description: >
      A transaction is successful if the payment is captured and the order
      is recorded in our system. Customer payment method issues (insufficient
      funds, card declined by bank) are not counted as system errors.
    
    success_criteria:
      - "Payment captured (payment_status = 'captured')"
      - "Order created in database"
      - "Confirmation event dispatched"
    
    error_criteria:
      - "HTTP 5xx from payment service"
      - "Timeout waiting for payment processor"
      - "Order creation failed after successful payment capture"
      - "Inconsistent state (payment captured but order missing)"
    
    excluded_from_denominator:
      - "Card declined by issuing bank (customer issue)"
      - "Invalid card number (customer input error)"
      - "Fraud detection block (intentional protection)"
      - "3DS authentication failed (customer issue)"
  
  scope:
    events: "payment_initiated"
    payment_types: ["credit_card", "debit_card", "bank_transfer"]
    excludes: ["test_transactions", "internal_orders"]
  
  target:
    success_rate: 99.99%  # Very high target for financial operations
    error_rate: 0.01%
    
  semantic_validation:
    enabled: true
    checks:
      - name: "Amount consistency"
        rule: "captured_amount == authorized_amount"
      - name: "Currency consistency"  
        rule: "captured_currency == order_currency"
      - name: "Order linkage"
        rule: "order_id in orders_database"
    
  alerting:
    immediate_alerts:
      - condition: "Any transaction in inconsistent state"
        severity: "critical"
        notification: ["payments-oncall", "finance-team"]
    rate_based:
      - error_rate_threshold: 0.1%
        window: "5 minutes"
        severity: "critical"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

# Error Rate SLI Specification: Nightly Data Pipeline
sli:
  name: "Data Pipeline Record Processing Success"
  description: "Proportion of records successfully processed through the pipeline"
  
  # For batch processing, we measure at record level, not job level
  # A job can succeed overall while having some record-level failures
  
  success_definition:
    at_record_level:
      success: "Record transformed and loaded to destination"
      error: "Record rejected or failed to load"
      excluded: "Record filtered by business rules (intentional skip)"
    
    at_job_level:
      # Job success doesn't affect SLI, but tracked separately
      job_success: "All records processed (success + error + excluded = input)"
      job_failure: "Job crashed or timed out before completing"
  
  scope:
    pipelines: ["customer-sync", "order-analytics", "inventory-update"]
    record_sources: "All input records from source systems"
  
  targets:
    record_success_rate: 99.5%
    rationale: >
      Batch processing allows for retry and manual remediation.
      0.5% error rate on millions of records is still thousands of
      failures to investigate, so this is reasonably tight.
    
    job_completion_rate: 99.9%  # Separate SLI for job-level reliability
  
  retry_policy:
    automatic_retries: 3
    retry_interval: "exponential backoff, max 5 minutes"
    error_after_retries: true  # Only count as error after all retries fail
  
  error_categorization:
    recoverable:
      - "Transient database connection error"
      - "Temporary API rate limiting"
    permanent:
      - "Schema validation failure"
      - "Missing required field"
      - "Referential integrity violation"
  
  reporting:
    per_job_metrics:
      - "records_processed"
      - "records_succeeded"
      - "records_failed"
      - "error_rate"
      - "error_by_category"
    daily_aggregate: true
    monthly_slo_report: true