feat: Implement Polly Retry Policies for External Services (Issue #107)

src/StarGate.Api/StarGate.Api.csproj

@@ -6,13 +6,13 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AspNetCore.HealthChecks.MongoDb" Version="8.0.1" />
+    <PackageReference Include="AspNetCore.HealthChecks.MongoDb" Version="8.1.0" />
     <PackageReference Include="AspNetCore.HealthChecks.Rabbitmq" Version="8.0.1" />
     <PackageReference Include="AspNetCore.HealthChecks.Redis" Version="8.0.1" />
     <PackageReference Include="AspNetCore.HealthChecks.UI.Client" Version="8.0.1" />
-    <PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="12.1.1" />
+    <PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.9.2" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.0" />
-    <PackageReference Include="MongoDB.Driver" Version="2.22.0" />
+    <PackageReference Include="MongoDB.Driver" Version="2.28.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
   </ItemGroup>
 

src/StarGate.Infrastructure/Extensions/ResilienceServiceCollectionExtensions.cs

@@ -0,0 +1,79 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Polly;
+using StarGate.Infrastructure.Resilience;
+
+namespace StarGate.Infrastructure.Extensions;
+
+/// <summary>
+/// Extension methods for registering resilience policies.
+/// </summary>
+public static class ResilienceServiceCollectionExtensions
+{
+    /// <summary>
+    /// Adds resilience policies to the service collection.
+    /// </summary>
+    /// <param name="services">The service collection.</param>
+    /// <param name="configuration">Application configuration.</param>
+    /// <returns>The service collection for chaining.</returns>
+    public static IServiceCollection AddResiliencePolicies(
+        this IServiceCollection services,
+        IConfiguration configuration)
+    {
+        // Register retry policy configuration
+        services.Configure<RetryPolicyConfiguration>(
+            configuration.GetSection("Resilience:Retry"));
+
+        // Register database retry policy as singleton
+        services.AddSingleton(provider =>
+        {
+            var config = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
+            var logger = provider.GetRequiredService<ILogger<RetryPolicyConfiguration>>();
+            return RetryPolicyFactory.CreateDatabaseRetryPolicy(config, logger);
+        });
+
+        // Register broker retry policy as singleton
+        services.AddSingleton(provider =>
+        {
+            var config = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
+            var logger = provider.GetRequiredService<ILogger<RetryPolicyConfiguration>>();
+            return RetryPolicyFactory.CreateBrokerRetryPolicy(config, logger);
+        });
+
+        // Register HTTP retry policy factory as singleton
+        services.AddSingleton(provider =>
+        {
+            var config = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
+            var loggerFactory = provider.GetRequiredService<ILoggerFactory>();
+
+            // Return a factory function that creates HTTP retry policies with appropriate logger
+            return new Func<ILogger, Polly.Retry.AsyncRetryPolicy<HttpResponseMessage>>(
+                logger => RetryPolicyFactory.CreateHttpRetryPolicy(config, logger));
+        });
+
+        return services;
+    }
+
+    /// <summary>
+    /// Adds HTTP client without automatic retry policy.
+    /// Consumers should inject AsyncRetryPolicy and wrap calls manually.
+    /// </summary>
+    /// <typeparam name="TClient">HTTP client interface type.</typeparam>
+    /// <param name="services">The service collection.</param>
+    /// <param name="name">HTTP client name.</param>
+    /// <returns>HTTP client builder for further configuration.</returns>
+    /// <remarks>
+    /// Polly v8 removed AddPolicyHandler. To use retry policies:
+    /// 1. Inject AsyncRetryPolicy&lt;HttpResponseMessage&gt; via factory
+    /// 2. Wrap HTTP calls: await policy.ExecuteAsync(() => httpClient.SendAsync(request))
+    /// </remarks>
+    public static IHttpClientBuilder AddHttpClientWithRetry<TClient>(
+        this IServiceCollection services,
+        string name)
+        where TClient : class
+    {
+        return services.AddHttpClient<TClient>(name);
+    }
+}

src/StarGate.Infrastructure/Resilience/RetryPolicyConfiguration.cs

@@ -0,0 +1,53 @@
+namespace StarGate.Infrastructure.Resilience;
+
+/// <summary>
+/// Configuration for retry policies.
+/// </summary>
+public class RetryPolicyConfiguration
+{
+    /// <summary>
+    /// Maximum number of retry attempts.
+    /// </summary>
+    public int MaxRetryAttempts { get; set; } = 3;
+
+    /// <summary>
+    /// Initial delay before first retry (seconds).
+    /// </summary>
+    public double InitialDelaySeconds { get; set; } = 1.0;
+
+    /// <summary>
+    /// Maximum delay between retries (seconds).
+    /// </summary>
+    public double MaxDelaySeconds { get; set; } = 30.0;
+
+    /// <summary>
+    /// Exponential backoff multiplier.
+    /// </summary>
+    public double BackoffMultiplier { get; set; } = 2.0;
+
+    /// <summary>
+    /// Whether to use jitter to prevent thundering herd.
+    /// </summary>
+    public bool UseJitter { get; set; } = true;
+
+    /// <summary>
+    /// Calculates delay for a specific retry attempt.
+    /// </summary>
+    /// <param name="retryAttempt">The retry attempt number (1-based).</param>
+    /// <returns>Time span representing the delay before next retry.</returns>
+    public TimeSpan CalculateDelay(int retryAttempt)
+    {
+        var exponentialDelay = InitialDelaySeconds * Math.Pow(BackoffMultiplier, retryAttempt - 1);
+        var delay = Math.Min(exponentialDelay, MaxDelaySeconds);
+
+        if (UseJitter)
+        {
+            var random = new Random();
+            // Generate jitter between -10% and +10%
+            var jitter = delay * 0.2 * (random.NextDouble() - 0.5);
+            delay += jitter;
+        }
+
+        return TimeSpan.FromSeconds(Math.Max(delay, 0));
+    }
+}

src/StarGate.Infrastructure/Resilience/RetryPolicyFactory.cs

@@ -0,0 +1,136 @@
+using Microsoft.Extensions.Logging;
+using Polly;
+using Polly.Retry;
+
+namespace StarGate.Infrastructure.Resilience;
+
+/// <summary>
+/// Factory for creating Polly retry policies.
+/// </summary>
+public static class RetryPolicyFactory
+{
+    /// <summary>
+    /// Creates a retry policy for HTTP operations.
+    /// </summary>
+    /// <param name="config">Retry policy configuration.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <returns>Configured async retry policy for HTTP responses.</returns>
+    public static AsyncRetryPolicy<HttpResponseMessage> CreateHttpRetryPolicy(
+        RetryPolicyConfiguration config,
+        ILogger logger)
+    {
+        return Policy
+            .HandleResult<HttpResponseMessage>(r => !r.IsSuccessStatusCode)
+            .Or<HttpRequestException>()
+            .Or<TimeoutException>()
+            .WaitAndRetryAsync(
+                retryCount: config.MaxRetryAttempts,
+                sleepDurationProvider: retryAttempt => config.CalculateDelay(retryAttempt),
+                onRetry: (outcome, timespan, retryAttempt, context) =>
+                {
+                    var statusCode = outcome.Result?.StatusCode.ToString() ?? "N/A";
+                    var exception = outcome.Exception?.GetType().Name ?? "None";
+
+                    logger.LogWarning(
+                        "HTTP retry attempt {RetryAttempt}/{MaxRetries}: StatusCode={StatusCode}, Exception={Exception}, Delay={Delay}ms",
+                        retryAttempt,
+                        config.MaxRetryAttempts,
+                        statusCode,
+                        exception,
+                        timespan.TotalMilliseconds);
+                });
+    }
+
+    /// <summary>
+    /// Creates a retry policy for database operations.
+    /// </summary>
+    /// <param name="config">Retry policy configuration.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <returns>Configured async retry policy for database operations.</returns>
+    public static AsyncRetryPolicy CreateDatabaseRetryPolicy(
+        RetryPolicyConfiguration config,
+        ILogger logger)
+    {
+        return Policy
+            .Handle<TimeoutException>()
+            .Or<IOException>()
+            .Or<InvalidOperationException>(ex => ex.Message.Contains("connection", StringComparison.OrdinalIgnoreCase))
+            .WaitAndRetryAsync(
+                retryCount: config.MaxRetryAttempts,
+                sleepDurationProvider: retryAttempt => config.CalculateDelay(retryAttempt),
+                onRetry: (exception, timespan, retryAttempt, context) =>
+                {
+                    logger.LogWarning(
+                        exception,
+                        "Database retry attempt {RetryAttempt}/{MaxRetries}: Exception={Exception}, Delay={Delay}ms",
+                        retryAttempt,
+                        config.MaxRetryAttempts,
+                        exception.GetType().Name,
+                        timespan.TotalMilliseconds);
+                });
+    }
+
+    /// <summary>
+    /// Creates a retry policy for message broker operations.
+    /// </summary>
+    /// <param name="config">Retry policy configuration.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <returns>Configured async retry policy for broker operations.</returns>
+    public static AsyncRetryPolicy CreateBrokerRetryPolicy(
+        RetryPolicyConfiguration config,
+        ILogger logger)
+    {
+        return Policy
+            .Handle<TimeoutException>()
+            .Or<IOException>()
+            .Or<InvalidOperationException>(ex => ex.Message.Contains("connection", StringComparison.OrdinalIgnoreCase))
+            .WaitAndRetryAsync(
+                retryCount: config.MaxRetryAttempts,
+                sleepDurationProvider: retryAttempt => config.CalculateDelay(retryAttempt),
+                onRetry: (exception, timespan, retryAttempt, context) =>
+                {
+                    logger.LogWarning(
+                        exception,
+                        "Broker retry attempt {RetryAttempt}/{MaxRetries}: Exception={Exception}, Delay={Delay}ms",
+                        retryAttempt,
+                        config.MaxRetryAttempts,
+                        exception.GetType().Name,
+                        timespan.TotalMilliseconds);
+                });
+    }
+
+    /// <summary>
+    /// Creates a generic retry policy for any async operation.
+    /// </summary>
+    /// <param name="config">Retry policy configuration.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <returns>Configured async retry policy for generic operations.</returns>
+    public static AsyncRetryPolicy CreateGenericRetryPolicy(
+        RetryPolicyConfiguration config,
+        ILogger logger)
+    {
+        return Policy
+            .Handle<Exception>(ex => IsTransientException(ex))
+            .WaitAndRetryAsync(
+                retryCount: config.MaxRetryAttempts,
+                sleepDurationProvider: retryAttempt => config.CalculateDelay(retryAttempt),
+                onRetry: (exception, timespan, retryAttempt, context) =>
+                {
+                    logger.LogWarning(
+                        exception,
+                        "Generic retry attempt {RetryAttempt}/{MaxRetries}: Exception={Exception}, Delay={Delay}ms",
+                        retryAttempt,
+                        config.MaxRetryAttempts,
+                        exception.GetType().Name,
+                        timespan.TotalMilliseconds);
+                });
+    }
+
+    private static bool IsTransientException(Exception ex)
+    {
+        return ex is TimeoutException
+            || ex is HttpRequestException
+            || ex is IOException
+            || (ex is InvalidOperationException && ex.Message.Contains("connection", StringComparison.OrdinalIgnoreCase));
+    }
+}

src/StarGate.Infrastructure/StarGate.Infrastructure.csproj

@@ -2,36 +2,30 @@
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
-    <RootNamespace>StarGate.Infrastructure</RootNamespace>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\StarGate.Core\StarGate.Core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <!-- MongoDB -->
-    <PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="12.1.1" />
-    <PackageReference Include="MongoDB.Driver" Version="2.22.0" />
-
-    <!-- Redis -->
-    <PackageReference Include="StackExchange.Redis" Version="2.8.16" />
-
-    <!-- RabbitMQ -->
-    <PackageReference Include="RabbitMQ.Client" Version="6.8.1" />
-
-    <!-- Diagnostics and Metrics -->
-    <PackageReference Include="System.Diagnostics.DiagnosticSource" Version="8.0.1" />
-
-    <!-- Microsoft.Extensions (ASP.NET Core) - Using 8.0.0 (proven compatible) -->
+    <PackageReference Include="FluentValidation" Version="11.9.2" />
+    <PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.9.2" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
     <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.3" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
     <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
+    <PackageReference Include="MongoDB.Driver" Version="2.28.0" />
+    <PackageReference Include="Polly" Version="8.4.2" />
+    <PackageReference Include="Polly.Extensions.Http" Version="3.0.0" />
+    <PackageReference Include="RabbitMQ.Client" Version="6.8.1" />
+    <PackageReference Include="StackExchange.Redis" Version="2.7.4" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\StarGate.Core\StarGate.Core.csproj" />
   </ItemGroup>
 
 </Project>

src/StarGate.Server/Program.cs

@@ -1,6 +1,7 @@
 using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.Extensions.Hosting;
 using StarGate.Core.Configuration;
+using StarGate.Infrastructure.Extensions;
 using StarGate.Server.HealthChecks;
 using StarGate.Server.Workers;
 
@@ -17,6 +18,9 @@
 builder.Services.Configure<RetryConfiguration>(
     builder.Configuration.GetSection("Retry"));
 
+// Add resilience policies
+builder.Services.AddResiliencePolicies(builder.Configuration);
+
 // Register ProcessWorker as singleton to allow health check injection
 builder.Services.AddSingleton<ProcessWorker>();
 builder.Services.AddHostedService(sp => sp.GetRequiredService<ProcessWorker>());

src/StarGate.Server/appsettings.Development.json

@@ -11,5 +11,14 @@
     "MaxDelaySeconds": 60,
     "BackoffMultiplier": 2.0,
     "UseJitter": true
+  },
+  "Resilience": {
+    "Retry": {
+      "MaxRetryAttempts": 2,
+      "InitialDelaySeconds": 0.5,
+      "MaxDelaySeconds": 10.0,
+      "BackoffMultiplier": 2.0,
+      "UseJitter": true
+    }
   }
 }

src/StarGate.Server/appsettings.json

@@ -11,5 +11,14 @@
     "MaxDelaySeconds": 300,
     "BackoffMultiplier": 2.0,
     "UseJitter": true
+  },
+  "Resilience": {
+    "Retry": {
+      "MaxRetryAttempts": 3,
+      "InitialDelaySeconds": 1.0,
+      "MaxDelaySeconds": 30.0,
+      "BackoffMultiplier": 2.0,
+      "UseJitter": true
+    }
   }
 }