Phase 8.1: Implement Polly Circuit Breaker Pattern

src/StarGate.Infrastructure/Extensions/ResilienceServiceCollectionExtensions.cs

@@ -3,6 +3,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Polly;
+using Polly.Wrap;
 using StarGate.Infrastructure.Resilience;
 
 namespace StarGate.Infrastructure.Extensions;
@@ -26,50 +27,56 @@ public static IServiceCollection AddResiliencePolicies(
         services.Configure<RetryPolicyConfiguration>(
             configuration.GetSection("Resilience:Retry"));
 
-        // Register database retry policy as singleton
+        // Register circuit breaker configuration
+        services.Configure<CircuitBreakerConfiguration>(
+            configuration.GetSection("Resilience:CircuitBreaker"));
+
+        // Register wrapped resilience policies (circuit breaker + retry)
         services.AddSingleton(provider =>
         {
-            var config = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
+            var retryConfig = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
+            var circuitConfig = provider.GetRequiredService<IOptions<CircuitBreakerConfiguration>>().Value;
             var logger = provider.GetRequiredService<ILogger<RetryPolicyConfiguration>>();
-            return RetryPolicyFactory.CreateDatabaseRetryPolicy(config, logger);
+            return ResiliencePolicyWrapper.CreateDatabaseResiliencePolicy(retryConfig, circuitConfig, logger);
         });
 
-        // Register broker retry policy as singleton
         services.AddSingleton(provider =>
         {
-            var config = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
+            var retryConfig = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
+            var circuitConfig = provider.GetRequiredService<IOptions<CircuitBreakerConfiguration>>().Value;
             var logger = provider.GetRequiredService<ILogger<RetryPolicyConfiguration>>();
-            return RetryPolicyFactory.CreateBrokerRetryPolicy(config, logger);
+            return ResiliencePolicyWrapper.CreateBrokerResiliencePolicy(retryConfig, circuitConfig, logger);
         });
 
-        // Register HTTP retry policy factory as singleton
+        // Register HTTP resilience policy factory as singleton
         services.AddSingleton(provider =>
         {
-            var config = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
+            var retryConfig = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
+            var circuitConfig = provider.GetRequiredService<IOptions<CircuitBreakerConfiguration>>().Value;
             var loggerFactory = provider.GetRequiredService<ILoggerFactory>();
 
-            // Return a factory function that creates HTTP retry policies with appropriate logger
-            return new Func<ILogger, Polly.Retry.AsyncRetryPolicy<HttpResponseMessage>>(
-                logger => RetryPolicyFactory.CreateHttpRetryPolicy(config, logger));
+            // Return a factory function that creates HTTP resilience policies with appropriate logger
+            return new Func<ILogger, AsyncPolicyWrap<HttpResponseMessage>>(
+                logger => ResiliencePolicyWrapper.CreateHttpResiliencePolicy(retryConfig, circuitConfig, logger));
         });
 
         return services;
     }
 
     /// <summary>
-    /// Adds HTTP client without automatic retry policy.
-    /// Consumers should inject AsyncRetryPolicy and wrap calls manually.
+    /// Adds HTTP client without automatic resilience policy.
+    /// Consumers should inject AsyncPolicyWrap&lt;HttpResponseMessage&gt; and wrap calls manually.
     /// </summary>
     /// <typeparam name="TClient">HTTP client interface type.</typeparam>
     /// <param name="services">The service collection.</param>
     /// <param name="name">HTTP client name.</param>
     /// <returns>HTTP client builder for further configuration.</returns>
     /// <remarks>
-    /// Polly v8 removed AddPolicyHandler. To use retry policies:
-    /// 1. Inject AsyncRetryPolicy&lt;HttpResponseMessage&gt; via factory
+    /// To use resilience policies:
+    /// 1. Inject AsyncPolicyWrap&lt;HttpResponseMessage&gt; via factory
     /// 2. Wrap HTTP calls: await policy.ExecuteAsync(() => httpClient.SendAsync(request))
     /// </remarks>
-    public static IHttpClientBuilder AddHttpClientWithRetry<TClient>(
+    public static IHttpClientBuilder AddHttpClientWithResilience<TClient>(
         this IServiceCollection services,
         string name)
         where TClient : class

src/StarGate.Infrastructure/Resilience/CircuitBreakerConfiguration.cs

@@ -0,0 +1,42 @@
+namespace StarGate.Infrastructure.Resilience;
+
+/// <summary>
+/// Configuration for circuit breaker policies.
+/// </summary>
+public class CircuitBreakerConfiguration
+{
+    /// <summary>
+    /// Number of consecutive failures before breaking the circuit.
+    /// </summary>
+    public int FailureThreshold { get; set; } = 5;
+
+    /// <summary>
+    /// Percentage of failures in sampling duration before breaking.
+    /// </summary>
+    public double FailureRateThreshold { get; set; } = 0.5; // 50%
+
+    /// <summary>
+    /// Minimum throughput before considering failure rate.
+    /// </summary>
+    public int MinimumThroughput { get; set; } = 10;
+
+    /// <summary>
+    /// Duration to keep circuit open before testing recovery (seconds).
+    /// </summary>
+    public double BreakDurationSeconds { get; set; } = 30.0;
+
+    /// <summary>
+    /// Duration to sample for failure rate calculation (seconds).
+    /// </summary>
+    public double SamplingDurationSeconds { get; set; } = 60.0;
+
+    /// <summary>
+    /// Gets the break duration as TimeSpan.
+    /// </summary>
+    public TimeSpan BreakDuration => TimeSpan.FromSeconds(BreakDurationSeconds);
+
+    /// <summary>
+    /// Gets the sampling duration as TimeSpan.
+    /// </summary>
+    public TimeSpan SamplingDuration => TimeSpan.FromSeconds(SamplingDurationSeconds);
+}

src/StarGate.Infrastructure/Resilience/CircuitBreakerFactory.cs

@@ -0,0 +1,125 @@
+using Microsoft.Extensions.Logging;
+using Polly;
+using Polly.CircuitBreaker;
+
+namespace StarGate.Infrastructure.Resilience;
+
+/// <summary>
+/// Factory for creating Polly circuit breaker policies.
+/// </summary>
+public static class CircuitBreakerFactory
+{
+    /// <summary>
+    /// Creates a circuit breaker policy for HTTP operations.
+    /// </summary>
+    /// <param name="config">Circuit breaker configuration.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <returns>Configured async circuit breaker policy for HTTP responses.</returns>
+    public static AsyncCircuitBreakerPolicy<HttpResponseMessage> CreateHttpCircuitBreaker(
+        CircuitBreakerConfiguration config,
+        ILogger logger)
+    {
+        return Policy
+            .HandleResult<HttpResponseMessage>(r => !r.IsSuccessStatusCode)
+            .Or<HttpRequestException>()
+            .Or<TimeoutException>()
+            .AdvancedCircuitBreakerAsync(
+                failureThreshold: config.FailureRateThreshold,
+                samplingDuration: config.SamplingDuration,
+                minimumThroughput: config.MinimumThroughput,
+                durationOfBreak: config.BreakDuration,
+                onBreak: (outcome, breakDuration, context) =>
+                {
+                    var statusCode = outcome.Result?.StatusCode.ToString() ?? "N/A";
+                    var exception = outcome.Exception?.GetType().Name ?? "None";
+
+                    logger.LogError(
+                        "HTTP circuit breaker opened: StatusCode={StatusCode}, Exception={Exception}, BreakDuration={BreakDuration}s",
+                        statusCode,
+                        exception,
+                        breakDuration.TotalSeconds);
+                },
+                onReset: context =>
+                {
+                    logger.LogInformation("HTTP circuit breaker reset: Circuit closed");
+                },
+                onHalfOpen: () =>
+                {
+                    logger.LogWarning("HTTP circuit breaker half-open: Testing recovery");
+                });
+    }
+
+    /// <summary>
+    /// Creates a circuit breaker policy for database operations.
+    /// </summary>
+    /// <param name="config">Circuit breaker configuration.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <returns>Configured async circuit breaker policy for database operations.</returns>
+    public static AsyncCircuitBreakerPolicy CreateDatabaseCircuitBreaker(
+        CircuitBreakerConfiguration config,
+        ILogger logger)
+    {
+        return Policy
+            .Handle<TimeoutException>()
+            .Or<IOException>()
+            .Or<InvalidOperationException>(ex => ex.Message.Contains("connection", StringComparison.OrdinalIgnoreCase))
+            .AdvancedCircuitBreakerAsync(
+                failureThreshold: config.FailureRateThreshold,
+                samplingDuration: config.SamplingDuration,
+                minimumThroughput: config.MinimumThroughput,
+                durationOfBreak: config.BreakDuration,
+                onBreak: (exception, breakDuration, context) =>
+                {
+                    logger.LogError(
+                        exception,
+                        "Database circuit breaker opened: Exception={Exception}, BreakDuration={BreakDuration}s",
+                        exception.GetType().Name,
+                        breakDuration.TotalSeconds);
+                },
+                onReset: context =>
+                {
+                    logger.LogInformation("Database circuit breaker reset: Circuit closed");
+                },
+                onHalfOpen: () =>
+                {
+                    logger.LogWarning("Database circuit breaker half-open: Testing recovery");
+                });
+    }
+
+    /// <summary>
+    /// Creates a circuit breaker policy for message broker operations.
+    /// </summary>
+    /// <param name="config">Circuit breaker configuration.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <returns>Configured async circuit breaker policy for broker operations.</returns>
+    public static AsyncCircuitBreakerPolicy CreateBrokerCircuitBreaker(
+        CircuitBreakerConfiguration config,
+        ILogger logger)
+    {
+        return Policy
+            .Handle<TimeoutException>()
+            .Or<IOException>()
+            .Or<InvalidOperationException>(ex => ex.Message.Contains("connection", StringComparison.OrdinalIgnoreCase))
+            .AdvancedCircuitBreakerAsync(
+                failureThreshold: config.FailureRateThreshold,
+                samplingDuration: config.SamplingDuration,
+                minimumThroughput: config.MinimumThroughput,
+                durationOfBreak: config.BreakDuration,
+                onBreak: (exception, breakDuration, context) =>
+                {
+                    logger.LogError(
+                        exception,
+                        "Broker circuit breaker opened: Exception={Exception}, BreakDuration={BreakDuration}s",
+                        exception.GetType().Name,
+                        breakDuration.TotalSeconds);
+                },
+                onReset: context =>
+                {
+                    logger.LogInformation("Broker circuit breaker reset: Circuit closed");
+                },
+                onHalfOpen: () =>
+                {
+                    logger.LogWarning("Broker circuit breaker half-open: Testing recovery");
+                });
+    }
+}

src/StarGate.Infrastructure/Resilience/CircuitBreakerStateService.cs

@@ -0,0 +1,50 @@
+using System.Collections.Concurrent;
+using Polly.CircuitBreaker;
+
+namespace StarGate.Infrastructure.Resilience;
+
+/// <summary>
+/// Service for tracking circuit breaker states.
+/// </summary>
+public class CircuitBreakerStateService
+{
+    private readonly ConcurrentDictionary<string, CircuitState> _states = new();
+
+    /// <summary>
+    /// Records circuit state change.
+    /// </summary>
+    /// <param name="circuitName">Name of the circuit.</param>
+    /// <param name="state">New state of the circuit.</param>
+    public void RecordStateChange(string circuitName, CircuitState state)
+    {
+        _states.AddOrUpdate(circuitName, state, (_, __) => state);
+    }
+
+    /// <summary>
+    /// Gets current state of a circuit.
+    /// </summary>
+    /// <param name="circuitName">Name of the circuit.</param>
+    /// <returns>Current state if circuit exists, null otherwise.</returns>
+    public CircuitState? GetState(string circuitName)
+    {
+        return _states.TryGetValue(circuitName, out var state) ? state : null;
+    }
+
+    /// <summary>
+    /// Gets all circuit states.
+    /// </summary>
+    /// <returns>Dictionary of circuit names and their states.</returns>
+    public Dictionary<string, CircuitState> GetAllStates()
+    {
+        return new Dictionary<string, CircuitState>(_states);
+    }
+
+    /// <summary>
+    /// Checks if any circuit is open.
+    /// </summary>
+    /// <returns>True if at least one circuit is open, false otherwise.</returns>
+    public bool HasOpenCircuit()
+    {
+        return _states.Values.Any(state => state == CircuitState.Open);
+    }
+}

src/StarGate.Infrastructure/Resilience/ResiliencePolicyWrapper.cs

@@ -0,0 +1,66 @@
+using Microsoft.Extensions.Logging;
+using Polly;
+using Polly.Wrap;
+
+namespace StarGate.Infrastructure.Resilience;
+
+/// <summary>
+/// Wraps retry and circuit breaker policies together.
+/// </summary>
+public static class ResiliencePolicyWrapper
+{
+    /// <summary>
+    /// Creates a wrapped policy with retry inside circuit breaker for HTTP.
+    /// </summary>
+    /// <param name="retryConfig">Retry policy configuration.</param>
+    /// <param name="circuitConfig">Circuit breaker configuration.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <returns>Wrapped policy with circuit breaker outer and retry inner.</returns>
+    public static AsyncPolicyWrap<HttpResponseMessage> CreateHttpResiliencePolicy(
+        RetryPolicyConfiguration retryConfig,
+        CircuitBreakerConfiguration circuitConfig,
+        ILogger logger)
+    {
+        var retryPolicy = RetryPolicyFactory.CreateHttpRetryPolicy(retryConfig, logger);
+        var circuitBreaker = CircuitBreakerFactory.CreateHttpCircuitBreaker(circuitConfig, logger);
+
+        // Wrap: Circuit Breaker (outer) -> Retry (inner)
+        return Policy.WrapAsync(circuitBreaker, retryPolicy);
+    }
+
+    /// <summary>
+    /// Creates a wrapped policy with retry inside circuit breaker for database.
+    /// </summary>
+    /// <param name="retryConfig">Retry policy configuration.</param>
+    /// <param name="circuitConfig">Circuit breaker configuration.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <returns>Wrapped policy with circuit breaker outer and retry inner.</returns>
+    public static AsyncPolicyWrap CreateDatabaseResiliencePolicy(
+        RetryPolicyConfiguration retryConfig,
+        CircuitBreakerConfiguration circuitConfig,
+        ILogger logger)
+    {
+        var retryPolicy = RetryPolicyFactory.CreateDatabaseRetryPolicy(retryConfig, logger);
+        var circuitBreaker = CircuitBreakerFactory.CreateDatabaseCircuitBreaker(circuitConfig, logger);
+
+        return Policy.WrapAsync(circuitBreaker, retryPolicy);
+    }
+
+    /// <summary>
+    /// Creates a wrapped policy with retry inside circuit breaker for broker.
+    /// </summary>
+    /// <param name="retryConfig">Retry policy configuration.</param>
+    /// <param name="circuitConfig">Circuit breaker configuration.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <returns>Wrapped policy with circuit breaker outer and retry inner.</returns>
+    public static AsyncPolicyWrap CreateBrokerResiliencePolicy(
+        RetryPolicyConfiguration retryConfig,
+        CircuitBreakerConfiguration circuitConfig,
+        ILogger logger)
+    {
+        var retryPolicy = RetryPolicyFactory.CreateBrokerRetryPolicy(retryConfig, logger);
+        var circuitBreaker = CircuitBreakerFactory.CreateBrokerCircuitBreaker(circuitConfig, logger);
+
+        return Policy.WrapAsync(circuitBreaker, retryPolicy);
+    }
+}