Circuit Breaker Pattern

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

// Prometheus metrics setup for circuit breakers
import { Registry, Gauge, Counter, Histogram } from 'prom-client';
 
const registry = new Registry();
 
// State gauge (current state as numeric value)
const circuitState = new Gauge({
  name: 'circuit_breaker_state',
  help: 'Current circuit breaker state (0=closed, 1=open, 2=half_open)',
  labelNames: ['circuit_name', 'downstream_service'],
  registers: [registry],
});
 
// Failure rate gauge
const failureRate = new Gauge({
  name: 'circuit_breaker_failure_rate',
  help: 'Current failure rate percentage in sliding window',
  labelNames: ['circuit_name'],
  registers: [registry],
});
 
// Call counters
const callsTotal = new Counter({
  name: 'circuit_breaker_calls_total',
  help: 'Total number of calls through circuit breaker',
  labelNames: ['circuit_name', 'outcome'],  // outcome: success, failure, not_permitted
  registers: [registry],
});
 
// State transition counter
const transitions = new Counter({
  name: 'circuit_breaker_state_transitions_total',
  help: 'Number of state transitions',
  labelNames: ['circuit_name', 'from_state', 'to_state'],
  registers: [registry],
});
 
// Response time histogram
const responseTime = new Histogram({
  name: 'circuit_breaker_call_duration_seconds',
  help: 'Response time of calls through circuit breaker',
  labelNames: ['circuit_name', 'outcome'],
  buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10],  // 10ms to 10s
  registers: [registry],
});
 
// Usage in circuit breaker
function recordCallOutcome(
  circuitName: string,
  downstream: string,
  outcome: 'success' | 'failure' | 'not_permitted',
  durationMs: number
): void {
  callsTotal.inc({ circuit_name: circuitName, outcome });
  
  if (outcome !== 'not_permitted') {
    responseTime.observe(
      { circuit_name: circuitName, outcome },
      durationMs / 1000
    );
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

// Express endpoint for Prometheus scraping
import express from 'express';
import { circuitBreakerRegistry } from './circuit-breaker-metrics';
 
const app = express();
 
// Metrics endpoint
app.get('/metrics', async (req, res) => {
  try {
    res.set('Content-Type', circuitBreakerRegistry.contentType);
    res.end(await circuitBreakerRegistry.metrics());
  } catch (error) {
    res.status(500).end(error);
  }
});
 
// Health check that includes circuit state summary
app.get('/health', async (req, res) => {
  const circuitSummary = circuitBreakerManager.getAllCircuitStates();
  
  // Include open circuits in health response
  const openCircuits = Object.entries(circuitSummary)
    .filter(([_, state]) => state === 'OPEN')
    .map(([name, _]) => name);
  
  res.json({
    status: openCircuits.length === 0 ? 'healthy' : 'degraded',
    openCircuits,
    circuitStates: circuitSummary,
  });
});

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

// StatsD-based metrics pushing
import StatsD from 'hot-shots';
 
const statsd = new StatsD({
  host: 'statsd.monitoring.local',
  port: 8125,
  prefix: 'circuit_breaker.',
});
 
// Real-time event pushing
class StatsDBreakerMetrics {
  private circuitName: string;
  
  constructor(circuitName: string) {
    this.circuitName = circuitName;
  }
  
  recordSuccess(durationMs: number): void {
    statsd.increment(`${this.circuitName}.success`);
    statsd.timing(`${this.circuitName}.duration`, durationMs);
  }
  
  recordFailure(durationMs: number): void {
    statsd.increment(`${this.circuitName}.failure`);
    statsd.timing(`${this.circuitName}.duration`, durationMs);
  }
  
  recordNotPermitted(): void {
    statsd.increment(`${this.circuitName}.not_permitted`);
  }
  
  recordStateTransition(from: string, to: string): void {
    // Immediate visibility into state changes
    statsd.event(
      `Circuit ${this.circuitName} transition`,
      `State changed from ${from} to ${to}`,
      { alert_type: to === 'OPEN' ? 'warning' : 'info' }
    );
    statsd.increment(`${this.circuitName}.transition.${from}_to_${to}`);
  }
  
  updateGauges(metrics: CircuitMetrics): void {
    statsd.gauge(`${this.circuitName}.failure_rate`, metrics.failureRate);
    statsd.gauge(`${this.circuitName}.slow_call_rate`, metrics.slowCallRate);
    statsd.gauge(`${this.circuitName}.buffered_calls`, metrics.bufferedCalls);
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

# PromQL queries for circuit breaker dashboard
 
# Count of open circuits (single stat panel)
count(circuit_breaker_state{state="1"}) or vector(0)
 
# State map by circuit (table panel)
circuit_breaker_state{} * on(circuit_name) group_left(downstream_service) 
  circuit_breaker_info{}
 
# Failure rate over time (graph panel)
circuit_breaker_failure_rate{circuit_name=~"$circuit"}
 
# Not permitted requests per second (graph panel)
rate(circuit_breaker_calls_total{outcome="not_permitted"}[5m])
 
# State transitions per hour (counter panel)
increase(circuit_breaker_state_transitions_total{}[1h])
 
# Success rate through circuit (graph panel)
sum(rate(circuit_breaker_calls_total{outcome="success"}[5m])) by (circuit_name) /
sum(rate(circuit_breaker_calls_total{outcome=~"success|failure"}[5m])) by (circuit_name)
 
# Time in current state (gauge panel)
time() - (circuit_breaker_state_changed_timestamp_seconds{} or 0)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

# Prometheus Alertmanager rules for circuit breakers
 
groups:
- name: circuit_breaker_alerts
  rules:
    
  # Alert when any circuit opens
  - alert: CircuitBreakerOpened
    expr: circuit_breaker_state == 1
    for: 0s  # Alert immediately
    labels:
      severity: warning
    annotations:
      summary: "Circuit breaker {{ $labels.circuit_name }} is OPEN"
      description: |
        Circuit {{ $labels.circuit_name }} to {{ $labels.downstream_service }}
        has opened due to failure rate exceeding threshold.
        Current failure rate: {{ $value }}%
        
  # Alert when circuit remains open for extended period
  - alert: CircuitBreakerOpenExtended
    expr: |
      circuit_breaker_state == 1 
      and 
      (time() - circuit_breaker_state_changed_timestamp) > 300
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Circuit {{ $labels.circuit_name }} open for >5 minutes"
      description: |
        Circuit has been open for more than 5 minutes.
        Recovery probes may be failing, or downstream service
        is not recovering. Manual investigation required.
 
  # Alert on circuit flapping (multiple transitions in short period)
  - alert: CircuitBreakerFlapping
    expr: |
      increase(circuit_breaker_state_transitions_total[15m]) > 4
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Circuit {{ $labels.circuit_name }} is flapping"
      description: |
        Circuit has transitioned more than 4 times in 15 minutes.
        This indicates either threshold misconfiguration or
        an unstable downstream service. Review circuit config
        and downstream health.
 
  # Alert on high rejection rate
  - alert: HighCircuitRejectionRate
    expr: |
      rate(circuit_breaker_calls_total{outcome="not_permitted"}[5m]) 
      / 
      rate(circuit_breaker_calls_total[5m]) > 0.1
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "Circuit {{ $labels.circuit_name }} rejecting >10% of traffic"
      description: |
        More than 10% of requests to this circuit are being rejected
        due to open circuit state. User impact likely.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

// Structured logging for circuit breaker events
import { Logger } from './logger';
 
class CircuitBreakerLogger {
  private logger: Logger;
  private circuitName: string;
  
  constructor(circuitName: string) {
    this.circuitName = circuitName;
    this.logger = Logger.getLogger('circuit-breaker');
  }
  
  logStateTransition(
    from: State,
    to: State,
    metrics: CircuitMetrics,
    trigger: string
  ): void {
    // State transitions are significant - WARN level
    this.logger.warn({
      event: 'circuit_breaker_state_transition',
      circuit_name: this.circuitName,
      downstream: metrics.downstreamService,
      from_state: from,
      to_state: to,
      trigger: trigger,  // 'failure_threshold_exceeded', 'timeout_expired', etc.
      metrics: {
        failure_rate: metrics.failureRate,
        slow_call_rate: metrics.slowCallRate,
        buffered_calls: metrics.bufferedCalls,
        failed_calls: metrics.failedCalls,
        slow_calls: metrics.slowCalls,
      },
      time_in_previous_state_ms: metrics.timeInPreviousState,
    });
  }
  
  logNotPermitted(requestContext: RequestContext): void {
    // Not-permitted calls might be noisy - INFO or DEBUG level
    this.logger.info({
      event: 'circuit_breaker_call_not_permitted',
      circuit_name: this.circuitName,
      request_id: requestContext.requestId,
      path: requestContext.path,
      method: requestContext.method,
      current_state: 'OPEN',
      reason: 'Circuit is open, request rejected immediately',
    });
  }
  
  logProbeResult(
    probeNumber: number,
    totalProbes: number,
    success: boolean,
    durationMs: number
  ): void {
    this.logger.info({
      event: 'circuit_breaker_probe_result',
      circuit_name: this.circuitName,
      probe_number: probeNumber,
      total_probes: totalProbes,
      success: success,
      duration_ms: durationMs,
      state: 'HALF_OPEN',
    });
  }
  
  logThresholdBreach(
    metricType: 'failure_rate' | 'slow_call_rate',
    currentValue: number,
    threshold: number
  ): void {
    this.logger.warn({
      event: 'circuit_breaker_threshold_breach',
      circuit_name: this.circuitName,
      metric_type: metricType,
      current_value: currentValue,
      threshold: threshold,
      message: `${metricType} (${currentValue}%) exceeded threshold (${threshold}%)`,
    });
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

# Elasticsearch/OpenSearch queries for circuit breaker analysis
 
# Find all circuits that opened in the last hour
event:"circuit_breaker_state_transition" AND to_state:"OPEN" AND @timestamp:[now-1h TO now]
 
# Find circuits that are flapping
event:"circuit_breaker_state_transition" | stats count by circuit_name | where count > 5
 
# Find requests rejected by circuits for a specific service
event:"circuit_breaker_call_not_permitted" AND path:"/api/checkout/*"
 
# Analyze probe success rate during recovery
event:"circuit_breaker_probe_result" AND circuit_name:"payment-service"
  | stats count by success
 
# Correlate circuit opens with error logs from downstream
(event:"circuit_breaker_state_transition" AND to_state:"OPEN") 
  OR 
(service:"payment-service" AND level:"ERROR")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

// OpenTelemetry integration for circuit breakers
import { trace, Span, SpanStatusCode } from '@opentelemetry/api';
 
class TracedCircuitBreaker {
  private breaker: CircuitBreaker;
  private tracer = trace.getTracer('circuit-breaker');
  
  async execute<T>(operation: () => Promise<T>): Promise<T> {
    const span = this.tracer.startSpan('circuit-breaker.execute', {
      attributes: {
        'circuit.name': this.breaker.name,
        'circuit.downstream': this.breaker.downstream,
        'circuit.state': this.breaker.state,
      },
    });
    
    try {
      if (this.breaker.state === 'OPEN') {
        // Request rejected by open circuit
        span.setAttributes({
          'circuit.result': 'not_permitted',
          'circuit.rejection_reason': 'circuit_open',
        });
        span.setStatus({ code: SpanStatusCode.ERROR, message: 'Circuit open' });
        throw new CircuitBreakerOpenException();
      }
      
      const startTime = Date.now();
      const result = await operation();
      const duration = Date.now() - startTime;
      
      span.setAttributes({
        'circuit.result': 'success',
        'circuit.duration_ms': duration,
        'circuit.was_slow': duration > this.breaker.slowCallThreshold,
      });
      
      return result;
      
    } catch (error) {
      span.setAttributes({
        'circuit.result': 'failure',
        'circuit.error_type': error.constructor.name,
      });
      span.recordException(error);
      span.setStatus({ code: SpanStatusCode.ERROR });
      throw error;
      
    } finally {
      // Always include final circuit state
      span.setAttributes({
        'circuit.final_state': this.breaker.state,
        'circuit.failure_rate': this.breaker.metrics.failureRate,
      });
      span.end();
    }
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

# Runbook: Circuit Breaker Opened
 
## 1. Initial Assessment (30 seconds)
- Which circuit opened? Check alert details.
- Is this a critical path circuit? Check criticality labeling.
- Are other circuits to the same downstream also open?
 
## 2. Verify Downstream Health (1-2 minutes)
- Check downstream service dashboard
- Review downstream service logs for errors
- Check downstream service pod/container status
- Verify downstream database/dependencies health
 
## 3. Assess Impact (1 minute)
- What user-facing functionality is affected?
- Is fallback behavior working?
- How many users/requests are affected?
 
## 4. Decide Response
IF downstream is genuinely unhealthy:
  → Let circuit protect the system
  → Focus on downstream recovery
  → Monitor circuit state for auto-recovery
 
IF downstream appears healthy (possible false positive):
  → Review circuit configuration
  → Check for threshold misconfiguration
  → Consider temporarily adjusting thresholds
 
IF circuit is flapping:
  → Investigate intermittent downstream issues
  → Consider increasing wait duration
  → Review minimum calls threshold
 
## 5. Recovery Verification
- Monitor half-open probes in dashboard
- Verify circuit closes after downstream recovery
- Confirm user-facing functionality restored
 
## 6. Post-Incident
- Document root cause if not auto-recovered
- Review if circuit configuration was appropriate
- Update runbook with lessons learned

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

# Runbook: Manual Circuit Override
 
## WARNING
Manual overrides bypass automatic protection. Use only when:
- Circuit is misconfigured and needs emergency correction
- Downstream is confirmed healthy but circuit won't recover
- Testing circuit behavior in staging environment
 
## Force Circuit CLOSED
Use when: Circuit is stuck open despite healthy downstream
 
```bash
# Via admin API (if exposed)
curl -X POST http://localhost:8080/admin/circuits/payment-service/close
 
# Via configuration (requires restart or dynamic config)
export CIRCUIT_payment-service_FORCE_OPEN=false
```
 
## Force Circuit OPEN
Use when: Need to stop traffic to a service immediately
 
```bash
# Via admin API
curl -X POST http://localhost:8080/admin/circuits/payment-service/open
 
# Via configuration
export CIRCUIT_payment-service_FORCE_OPEN=true
```
 
## Reset Circuit Metrics
Use when: Historical failures skewing current evaluation
 
```bash
curl -X POST http://localhost:8080/admin/circuits/payment-service/reset
```
 
## IMPORTANT: Revert Overrides
After emergency override, always:
1. Document the override in incident channel
2. Monitor circuit behavior closely
3. Remove override once issue is resolved
4. Let circuit return to automatic operation