Circuit Breaker with Retries

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

// CORRECT: Retry wraps CircuitBreaker
CircuitBreaker circuitBreaker = CircuitBreaker.ofDefaults("service");
Retry retry = Retry.of("service", RetryConfig.custom()
    .maxAttempts(3)
    .waitDuration(Duration.ofMillis(500))
    // Don't retry when circuit is open!
    .retryOnException(e -> !(e instanceof CallNotPermittedException))
    .build());
 
Supplier<Product> decoratedSupplier = Decorators
    .ofSupplier(() -> productService.getProduct(productId))
    .withCircuitBreaker(circuitBreaker)  // Inner: circuit breaker
    .withRetry(retry)                     // Outer: retry
    .decorate();
 
// The call flows:
// 1. Retry (outer) receives request
// 2. Retry calls CircuitBreaker (inner)
// 3. If circuit closed, call proceeds to service
// 4. If service fails, exception propagates to Retry
// 5. Retry checks if exception is retryable
// 6. If CallNotPermittedException (circuit open), don't retry
// 7. Otherwise, retry with backoff
 
// WRONG (DON'T DO THIS): CircuitBreaker wraps Retry
Supplier<Product> wrongOrder = Decorators
    .ofSupplier(() -> productService.getProduct(productId))
    .withRetry(retry)                     // Inner: retry (WRONG)
    .withCircuitBreaker(circuitBreaker)   // Outer: circuit breaker (WRONG)
    .decorate();
// This counts every retry as a separate failure!

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

// Different jitter strategies in Resilience4j
import io.github.resilience4j.retry.RetryConfig;
import io.github.resilience4j.core.IntervalFunction;
 
// 1. EXPONENTIAL BACKOFF (no jitter)
RetryConfig exponentialOnly = RetryConfig.custom()
    .maxAttempts(4)
    .intervalFunction(IntervalFunction.ofExponentialBackoff(
        Duration.ofMillis(500),  // Initial interval
        2.0                       // Multiplier
    ))
    .build();
// Produces: 500ms, 1000ms, 2000ms, 4000ms
 
// 2. EXPONENTIAL BACKOFF WITH RANDOM JITTER
RetryConfig exponentialWithJitter = RetryConfig.custom()
    .maxAttempts(4)
    .intervalFunction(IntervalFunction.ofExponentialRandomBackoff(
        Duration.ofMillis(500),  // Initial interval
        2.0,                     // Multiplier
        0.5                      // Randomization factor (±50%)
    ))
    .build();
// Produces: 250-750ms, 500-1500ms, 1000-3000ms, 2000-6000ms
 
// 3. DECORRELATED JITTER (AWS recommendation)
// Each retry's delay is random between base delay and 3× previous delay
RetryConfig decorrelatedJitter = RetryConfig.custom()
    .maxAttempts(4)
    .intervalFunction(attempt -> {
        long baseMs = 500;
        long maxMs = 60_000;
        long previousDelay = attempt == 1 ? baseMs : 
            (long)(baseMs * Math.pow(3, attempt - 2));
        long delay = (long)(baseMs + Math.random() * (previousDelay * 3 - baseMs));
        return Duration.ofMillis(Math.min(delay, maxMs));
    })
    .build();
 
// 4. EQUAL JITTER (hybrid approach)
// Half exponential, half random
RetryConfig equalJitter = RetryConfig.custom()
    .maxAttempts(4)
    .intervalFunction(attempt -> {
        long base = 500;
        long exponential = base * (1L << (attempt - 1)); // 2^(attempt-1)
        long halfExponential = exponential / 2;
        long delay = halfExponential + (long)(Math.random() * halfExponential);
        return Duration.ofMillis(Math.min(delay, 60_000));
    })
    .build();
// Provides minimum guaranteed backoff with random component

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

/**
 * A token-bucket based retry budget.
 * Limits retries to a percentage of total traffic.
 */
public class RetryBudget {
    private final double retryRatio;        // Target retry percentage
    private final long windowMs;            // Time window for calculation
    private final AtomicLong attempts;      // Total attempts in window
    private final AtomicLong retries;       // Retries in window
    private final AtomicLong windowStart;
    
    public RetryBudget(double retryRatio, Duration window) {
        this.retryRatio = retryRatio;
        this.windowMs = window.toMillis();
        this.attempts = new AtomicLong(0);
        this.retries = new AtomicLong(0);
        this.windowStart = new AtomicLong(System.currentTimeMillis());
    }
    
    /**
     * Record an original (non-retry) attempt.
     */
    public void recordAttempt() {
        maybeResetWindow();
        attempts.incrementAndGet();
    }
    
    /**
     * Check if a retry is allowed and record it if so.
     */
    public boolean tryAcquireRetry() {
        maybeResetWindow();
        
        long currentAttempts = attempts.get();
        long currentRetries = retries.get();
        long total = currentAttempts + currentRetries;
        
        if (total == 0) {
            // No attempts yet, allow retry
            retries.incrementAndGet();
            return true;
        }
        
        // Check if adding a retry would exceed budget
        double potentialRetryRatio = (currentRetries + 1.0) / (total + 1.0);
        
        if (potentialRetryRatio <= retryRatio) {
            retries.incrementAndGet();
            return true;
        }
        
        return false;  // Budget exhausted
    }
    
    private void maybeResetWindow() {
        long now = System.currentTimeMillis();
        long start = windowStart.get();
        
        if (now - start > windowMs) {
            // Reset window
            if (windowStart.compareAndSet(start, now)) {
                attempts.set(0);
                retries.set(0);
            }
        }
    }
    
    public double getCurrentRetryRatio() {
        long total = attempts.get() + retries.get();
        return total == 0 ? 0.0 : (double) retries.get() / total;
    }
}
 
// Usage
RetryBudget budget = new RetryBudget(0.20, Duration.ofSeconds(10));
 
public CompletableFuture<Response> callWithBudgetedRetry(Request request) {
    budget.recordAttempt();  // Original attempt
    
    return circuitBreaker.executeCompletableFuture(() -> 
        httpClient.send(request)
    ).exceptionallyCompose(error -> {
        if (error instanceof CallNotPermittedException) {
            // Circuit open, no retry
            return fallback(request);
        }
        
        if (budget.tryAcquireRetry()) {
            // Budget allows retry
            return callWithBudgetedRetry(request);  // Recursive retry
        } else {
            // Budget exhausted
            log.warn("Retry budget exhausted, failing request");
            return fallback(request);
        }
    });
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

apiVersion: policy.linkerd.io/v1beta1
kind: HTTPRoute
metadata:
  name: product-service-route
spec:
  parentRefs:
    - name: product-service
      kind: Service
  rules:
    - backendRefs:
        - name: product-service
          port: 80
      timeouts:
        request: 10s
      retry:
        limit:
          # Retry budget: max 20% of traffic can be retries
          retryRatio: 0.2
          # Allow this many retries per request regardless of ratio
          minRetriesPerSecond: 10
        conditions:
          - statusCodes:
              - 502
              - 503
              - 504
          - timeouts: true
        backoff:
          baseInterval: 25ms
          maxInterval: 250ms

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

@RestController
public class PaymentController {
    
    private final PaymentService paymentService;
    private final IdempotencyStore idempotencyStore;
    
    @PostMapping("/payments")
    public ResponseEntity<PaymentResult> createPayment(
            @RequestHeader("Idempotency-Key") String idempotencyKey,
            @RequestBody PaymentRequest request) {
        
        // Check if we've seen this idempotency key before
        Optional<PaymentResult> cachedResult = 
            idempotencyStore.get(idempotencyKey);
        
        if (cachedResult.isPresent()) {
            // Return cached result - this is a retry
            log.info("Returning cached result for idempotency key: {}", 
                     idempotencyKey);
            return ResponseEntity.ok(cachedResult.get());
        }
        
        // Try to acquire lock on idempotency key
        if (!idempotencyStore.tryLock(idempotencyKey, Duration.ofMinutes(5))) {
            // Another request with same key is in progress
            return ResponseEntity.status(HttpStatus.CONFLICT)
                .body(PaymentResult.inProgress());
        }
        
        try {
            // Process the payment
            PaymentResult result = paymentService.processPayment(request);
            
            // Store result for future retries
            idempotencyStore.store(idempotencyKey, result, Duration.ofHours(24));
            
            return ResponseEntity.ok(result);
            
        } finally {
            idempotencyStore.unlock(idempotencyKey);
        }
    }
}
 
@Component
public class RedisIdempotencyStore implements IdempotencyStore {
    
    private final RedisTemplate<String, String> redis;
    private final ObjectMapper objectMapper;
    
    @Override
    public Optional<PaymentResult> get(String key) {
        String value = redis.opsForValue().get("idempotency:" + key);
        if (value == null) return Optional.empty();
        return Optional.of(objectMapper.readValue(value, PaymentResult.class));
    }
    
    @Override
    public boolean tryLock(String key, Duration timeout) {
        return redis.opsForValue()
            .setIfAbsent("lock:" + key, "locked", timeout);
    }
    
    @Override
    public void store(String key, PaymentResult result, Duration ttl) {
        String value = objectMapper.writeValueAsString(result);
        redis.opsForValue().set("idempotency:" + key, value, ttl);
    }
    
    @Override
    public void unlock(String key) {
        redis.delete("lock:" + key);
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

resilience4j:
  # Circuit Breaker Configuration
  circuitbreaker:
    configs:
      default:
        slidingWindowType: COUNT_BASED
        slidingWindowSize: 100
        minimumNumberOfCalls: 10
        failureRateThreshold: 50
        slowCallRateThreshold: 80
        slowCallDurationThreshold: 2s
        waitDurationInOpenState: 30s
        permittedNumberOfCallsInHalfOpenState: 5
        automaticTransitionFromOpenToHalfOpenEnabled: true
        recordExceptions:
          - java.io.IOException
          - java.util.concurrent.TimeoutException
          - org.springframework.web.client.HttpServerErrorException
        ignoreExceptions:
          - org.springframework.web.client.HttpClientErrorException
    instances:
      productService:
        baseConfig: default
      paymentService:
        baseConfig: default
        failureRateThreshold: 30  # More conservative for payments
        waitDurationInOpenState: 60s
 
  # Retry Configuration
  retry:
    configs:
      default:
        maxAttempts: 3
        waitDuration: 500ms
        enableExponentialBackoff: true
        exponentialBackoffMultiplier: 2
        exponentialMaxWaitDuration: 5s
        retryExceptions:
          - java.io.IOException
          - java.util.concurrent.TimeoutException
        ignoreExceptions:
          # CRITICAL: Don't retry when circuit is open
          - io.github.resilience4j.circuitbreaker.CallNotPermittedException
          - org.springframework.web.client.HttpClientErrorException
    instances:
      productService:
        baseConfig: default
      paymentService:
        baseConfig: default
        maxAttempts: 2  # Fewer retries for sensitive operations
 
  # Bulkhead Configuration
  bulkhead:
    configs:
      default:
        maxConcurrentCalls: 25
        maxWaitDuration: 500ms
    instances:
      productService:
        baseConfig: default
      paymentService:
        maxConcurrentCalls: 10  # More restrictive for payments
 
  # Time Limiter Configuration
  timelimiter:
    configs:
      default:
        timeoutDuration: 5s
        cancelRunningFuture: true
    instances:
      productService:
        baseConfig: default
      paymentService:
        timeoutDuration: 10s  # Longer for payment processing

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

@Service
@Slf4j
public class ProductServiceClient {
    
    private final WebClient webClient;
    
    public ProductServiceClient(WebClient.Builder webClientBuilder) {
        this.webClient = webClientBuilder
            .baseUrl("http://product-service")
            .build();
    }
    
    /**
     * Get product with full resilience stack.
     * Decorator order: Retry → CircuitBreaker → Bulkhead → TimeLimiter → Call
     */
    @Retry(name = "productService")
    @CircuitBreaker(name = "productService", fallbackMethod = "getProductFallback")
    @Bulkhead(name = "productService")
    @TimeLimiter(name = "productService")
    public CompletableFuture<Product> getProduct(String productId) {
        return webClient.get()
            .uri("/products/{id}", productId)
            .retrieve()
            .bodyToMono(Product.class)
            .toFuture();
    }
    
    /**
     * Fallback when circuit is open or all retries exhausted.
     */
    private CompletableFuture<Product> getProductFallback(
            String productId, 
            Throwable throwable) {
        
        if (throwable instanceof CallNotPermittedException) {
            log.warn("Circuit open for product service, using fallback");
            // Circuit is open - return cached data
            return CompletableFuture.completedFuture(
                productCache.get(productId).orElse(Product.placeholder(productId))
            );
        }
        
        if (throwable instanceof BulkheadFullException) {
            log.warn("Bulkhead full for product service, using fallback");
            // Too many concurrent requests
            return CompletableFuture.completedFuture(
                Product.placeholder(productId)
            );
        }
        
        if (throwable instanceof TimeoutException) {
            log.warn("Request timed out for product {}", productId);
        }
        
        // Default fallback
        log.error("Product service call failed for {}", productId, throwable);
        return CompletableFuture.completedFuture(
            Product.unavailable(productId)
        );
    }
    
    /**
     * Create product with idempotency key.
     * Uses fewer retries to avoid duplicate creation issues.
     */
    @Retry(name = "productService", fallbackMethod = "createProductFallback")
    @CircuitBreaker(name = "productService")
    @Bulkhead(name = "productService")
    @TimeLimiter(name = "productService")
    public CompletableFuture<Product> createProduct(
            String idempotencyKey, 
            ProductCreateRequest request) {
        
        return webClient.post()
            .uri("/products")
            .header("Idempotency-Key", idempotencyKey)
            .bodyValue(request)
            .retrieve()
            .bodyToMono(Product.class)
            .toFuture();
    }
    
    private CompletableFuture<Product> createProductFallback(
            String idempotencyKey,
            ProductCreateRequest request,
            Throwable throwable) {
        
        // For creates, queue the operation for later processing
        log.error("Product creation failed, queuing for retry", throwable);
        productQueue.enqueue(idempotencyKey, request);
        
        return CompletableFuture.failedFuture(
            new ServiceTemporarilyUnavailableException(
                "Product creation queued for processing"
            )
        );
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

{
  "title": "Resilience Dashboard",
  "widgets": [
    {
      "title": "Circuit Breaker State",
      "query": "resilience4j_circuitbreaker_state{service:product}",
      "visualization": "timeseries"
    },
    {
      "title": "Calls Rejected by Open Circuit",
      "query": "sum:resilience4j_circuitbreaker_not_permitted_calls{*}.as_rate()",
      "visualization": "timeseries"
    },
    {
      "title": "Retry Ratio",
      "query": "sum:resilience4j_retry_calls{kind:retry} / sum:resilience4j_retry_calls{*}",
      "visualization": "query_value",
      "alert_threshold": 0.2
    },
    {
      "title": "Failure Rate by Service",
      "query": "avg:resilience4j_circuitbreaker_failure_rate{*} by {name}",
      "visualization": "timeseries"
    },
    {
      "title": "Retry Exhaustion Rate",
      "query": "sum:resilience4j_retry_calls{kind:failed_with_retry}.as_rate()",
      "visualization": "timeseries"
    }
  ]
}