Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
492 changes: 492 additions & 0 deletions docs/RESILIENCE-STRATEGY.md

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ public static IServiceCollection AddResiliencePolicies(
this IServiceCollection services,
IConfiguration configuration)
{
// Register timeout configuration
services.Configure<TimeoutConfiguration>(
configuration.GetSection("Resilience:Timeout"));

// Register retry policy configuration
services.Configure<RetryPolicyConfiguration>(
configuration.GetSection("Resilience:Retry"));
Expand All @@ -31,49 +35,55 @@ public static IServiceCollection AddResiliencePolicies(
services.Configure<CircuitBreakerConfiguration>(
configuration.GetSection("Resilience:CircuitBreaker"));

// Register wrapped resilience policies (circuit breaker + retry)
// Register complete wrapped resilience policies (timeout + circuit breaker + retry)
services.AddSingleton(provider =>
{
var timeoutConfig = provider.GetRequiredService<IOptions<TimeoutConfiguration>>().Value;
var retryConfig = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
var circuitConfig = provider.GetRequiredService<IOptions<CircuitBreakerConfiguration>>().Value;
var logger = provider.GetRequiredService<ILogger<RetryPolicyConfiguration>>();
return ResiliencePolicyWrapper.CreateDatabaseResiliencePolicy(retryConfig, circuitConfig, logger);
return ResiliencePolicyWrapper.CreateCompleteDatabaseResiliencePolicy(
timeoutConfig, retryConfig, circuitConfig, logger);
});

services.AddSingleton(provider =>
{
var timeoutConfig = provider.GetRequiredService<IOptions<TimeoutConfiguration>>().Value;
var retryConfig = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
var circuitConfig = provider.GetRequiredService<IOptions<CircuitBreakerConfiguration>>().Value;
var logger = provider.GetRequiredService<ILogger<RetryPolicyConfiguration>>();
return ResiliencePolicyWrapper.CreateBrokerResiliencePolicy(retryConfig, circuitConfig, logger);
return ResiliencePolicyWrapper.CreateCompleteBrokerResiliencePolicy(
timeoutConfig, retryConfig, circuitConfig, logger);
});

// Register HTTP resilience policy factory as singleton
// Register HTTP complete resilience policy factory as singleton
services.AddSingleton(provider =>
{
var timeoutConfig = provider.GetRequiredService<IOptions<TimeoutConfiguration>>().Value;
var retryConfig = provider.GetRequiredService<IOptions<RetryPolicyConfiguration>>().Value;
var circuitConfig = provider.GetRequiredService<IOptions<CircuitBreakerConfiguration>>().Value;
var loggerFactory = provider.GetRequiredService<ILoggerFactory>();

// Return a factory function that creates HTTP resilience policies with appropriate logger
return new Func<ILogger, AsyncPolicyWrap<HttpResponseMessage>>(
logger => ResiliencePolicyWrapper.CreateHttpResiliencePolicy(retryConfig, circuitConfig, logger));
// Return a factory function that creates HTTP complete resilience policies with appropriate logger
return new Func<ILogger, CompleteHttpResiliencePolicy>(
logger => ResiliencePolicyWrapper.CreateCompleteHttpResiliencePolicy(
timeoutConfig, retryConfig, circuitConfig, logger));
});

return services;
}

/// <summary>
/// Adds HTTP client without automatic resilience policy.
/// Consumers should inject AsyncPolicyWrap&lt;HttpResponseMessage&gt; and wrap calls manually.
/// Consumers should inject CompleteHttpResiliencePolicy and wrap calls manually.
/// </summary>
/// <typeparam name="TClient">HTTP client interface type.</typeparam>
/// <param name="services">The service collection.</param>
/// <param name="name">HTTP client name.</param>
/// <returns>HTTP client builder for further configuration.</returns>
/// <remarks>
/// To use resilience policies:
/// 1. Inject AsyncPolicyWrap&lt;HttpResponseMessage&gt; via factory
/// 1. Inject CompleteHttpResiliencePolicy via factory
/// 2. Wrap HTTP calls: await policy.ExecuteAsync(() => httpClient.SendAsync(request))
/// </remarks>
public static IHttpClientBuilder AddHttpClientWithResilience<TClient>(
Expand Down
110 changes: 110 additions & 0 deletions src/StarGate.Infrastructure/Resilience/ResiliencePolicyWrapper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,114 @@ public static AsyncPolicyWrap CreateBrokerResiliencePolicy(

return Policy.WrapAsync(circuitBreaker, retryPolicy);
}

/// <summary>
/// Creates a complete resilience policy with timeout, circuit breaker, and retry for HTTP.
/// Note: Timeout is applied as an outer wrapper via ExecuteAsync pattern.
/// </summary>
/// <param name="timeoutConfig">Timeout configuration.</param>
/// <param name="retryConfig">Retry policy configuration.</param>
/// <param name="circuitConfig">Circuit breaker configuration.</param>
/// <param name="logger">Logger instance.</param>
/// <returns>Wrapped policy combining circuit breaker and retry. Apply timeout via WrapWithTimeoutAsync extension.</returns>
public static CompleteHttpResiliencePolicy CreateCompleteHttpResiliencePolicy(
TimeoutConfiguration timeoutConfig,
RetryPolicyConfiguration retryConfig,
CircuitBreakerConfiguration circuitConfig,
ILogger logger)
{
var timeoutPolicy = TimeoutPolicyFactory.CreateHttpTimeoutPolicy(timeoutConfig, logger);
var retryPolicy = RetryPolicyFactory.CreateHttpRetryPolicy(retryConfig, logger);
var circuitBreaker = CircuitBreakerFactory.CreateHttpCircuitBreaker(circuitConfig, logger);

// Wrap circuit breaker and retry
var innerPolicy = Policy.WrapAsync(circuitBreaker, retryPolicy);

return new CompleteHttpResiliencePolicy(timeoutPolicy, innerPolicy);
}

/// <summary>
/// Creates a complete resilience policy for database operations.
/// </summary>
/// <param name="timeoutConfig">Timeout configuration.</param>
/// <param name="retryConfig">Retry policy configuration.</param>
/// <param name="circuitConfig">Circuit breaker configuration.</param>
/// <param name="logger">Logger instance.</param>
/// <returns>Complete wrapped policy with timeout (outer), circuit breaker, and retry (inner).</returns>
public static AsyncPolicyWrap CreateCompleteDatabaseResiliencePolicy(
TimeoutConfiguration timeoutConfig,
RetryPolicyConfiguration retryConfig,
CircuitBreakerConfiguration circuitConfig,
ILogger logger)
{
var timeoutPolicy = TimeoutPolicyFactory.CreateDatabaseTimeoutPolicy(timeoutConfig, logger);
var retryPolicy = RetryPolicyFactory.CreateDatabaseRetryPolicy(retryConfig, logger);
var circuitBreaker = CircuitBreakerFactory.CreateDatabaseCircuitBreaker(circuitConfig, logger);

return Policy.WrapAsync(timeoutPolicy, circuitBreaker, retryPolicy);
}

/// <summary>
/// Creates a complete resilience policy for broker operations.
/// </summary>
/// <param name="timeoutConfig">Timeout configuration.</param>
/// <param name="retryConfig">Retry policy configuration.</param>
/// <param name="circuitConfig">Circuit breaker configuration.</param>
/// <param name="logger">Logger instance.</param>
/// <returns>Complete wrapped policy with timeout (outer), circuit breaker, and retry (inner).</returns>
public static AsyncPolicyWrap CreateCompleteBrokerResiliencePolicy(
TimeoutConfiguration timeoutConfig,
RetryPolicyConfiguration retryConfig,
CircuitBreakerConfiguration circuitConfig,
ILogger logger)
{
var timeoutPolicy = TimeoutPolicyFactory.CreateBrokerTimeoutPolicy(timeoutConfig, logger);
var retryPolicy = RetryPolicyFactory.CreateBrokerRetryPolicy(retryConfig, logger);
var circuitBreaker = CircuitBreakerFactory.CreateBrokerCircuitBreaker(circuitConfig, logger);

return Policy.WrapAsync(timeoutPolicy, circuitBreaker, retryPolicy);
}
}

/// <summary>
/// Wrapper for complete HTTP resilience policy with timeout, circuit breaker, and retry.
/// </summary>
public class CompleteHttpResiliencePolicy
{
private readonly Polly.Timeout.AsyncTimeoutPolicy _timeoutPolicy;
private readonly AsyncPolicyWrap<HttpResponseMessage> _innerPolicy;

public CompleteHttpResiliencePolicy(
Polly.Timeout.AsyncTimeoutPolicy timeoutPolicy,
AsyncPolicyWrap<HttpResponseMessage> innerPolicy)
{
_timeoutPolicy = timeoutPolicy ?? throw new ArgumentNullException(nameof(timeoutPolicy));
_innerPolicy = innerPolicy ?? throw new ArgumentNullException(nameof(innerPolicy));
}

/// <summary>
/// Executes the operation with timeout, circuit breaker, and retry policies.
/// </summary>
public async Task<HttpResponseMessage> ExecuteAsync(
Func<Task<HttpResponseMessage>> operation,
CancellationToken cancellationToken = default)
{
return await _timeoutPolicy.ExecuteAsync(async (ct) =>
{
return await _innerPolicy.ExecuteAsync(() => operation());
}, cancellationToken);
}

/// <summary>
/// Executes the operation with timeout, circuit breaker, and retry policies.
/// </summary>
public async Task<HttpResponseMessage> ExecuteAsync(
Func<CancellationToken, Task<HttpResponseMessage>> operation,
CancellationToken cancellationToken = default)
{
return await _timeoutPolicy.ExecuteAsync(async (ct) =>
{
return await _innerPolicy.ExecuteAsync(() => operation(ct));
}, cancellationToken);
}
}
43 changes: 43 additions & 0 deletions src/StarGate.Infrastructure/Resilience/TimeoutConfiguration.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
namespace StarGate.Infrastructure.Resilience;

/// <summary>
/// Configuration for timeout policies.
/// </summary>
public class TimeoutConfiguration
{
/// <summary>
/// Timeout for HTTP requests (seconds).
/// </summary>
public double HttpTimeoutSeconds { get; set; } = 30.0;

/// <summary>
/// Timeout for database operations (seconds).
/// </summary>
public double DatabaseTimeoutSeconds { get; set; } = 10.0;

/// <summary>
/// Timeout for message broker operations (seconds).
/// </summary>
public double BrokerTimeoutSeconds { get; set; } = 5.0;

/// <summary>
/// Whether to use pessimistic timeout (cancels operation).
/// If false, uses optimistic timeout (monitors but doesn't cancel).
/// </summary>
public bool UsePessimisticTimeout { get; set; } = true;

/// <summary>
/// Gets HTTP timeout as TimeSpan.
/// </summary>
public TimeSpan HttpTimeout => TimeSpan.FromSeconds(HttpTimeoutSeconds);

/// <summary>
/// Gets database timeout as TimeSpan.
/// </summary>
public TimeSpan DatabaseTimeout => TimeSpan.FromSeconds(DatabaseTimeoutSeconds);

/// <summary>
/// Gets broker timeout as TimeSpan.
/// </summary>
public TimeSpan BrokerTimeout => TimeSpan.FromSeconds(BrokerTimeoutSeconds);
}
83 changes: 83 additions & 0 deletions src/StarGate.Infrastructure/Resilience/TimeoutPolicyFactory.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
using Microsoft.Extensions.Logging;
using Polly;
using Polly.Timeout;

namespace StarGate.Infrastructure.Resilience;

/// <summary>
/// Factory for creating Polly timeout policies.
/// </summary>
public static class TimeoutPolicyFactory
{
/// <summary>
/// Creates a timeout policy for HTTP operations.
/// </summary>
public static AsyncTimeoutPolicy CreateHttpTimeoutPolicy(
TimeoutConfiguration config,
ILogger logger)
{
return Policy
.TimeoutAsync(
timeout: config.HttpTimeout,
timeoutStrategy: config.UsePessimisticTimeout
? TimeoutStrategy.Pessimistic
: TimeoutStrategy.Optimistic,
onTimeoutAsync: (context, timespan, task) =>
{
logger.LogError(
"HTTP operation timed out: Timeout={Timeout}s, Strategy={Strategy}",
timespan.TotalSeconds,
config.UsePessimisticTimeout ? "Pessimistic" : "Optimistic");

return Task.CompletedTask;
});
}

/// <summary>
/// Creates a timeout policy for database operations.
/// </summary>
public static AsyncTimeoutPolicy CreateDatabaseTimeoutPolicy(
TimeoutConfiguration config,
ILogger logger)
{
return Policy
.TimeoutAsync(
timeout: config.DatabaseTimeout,
timeoutStrategy: config.UsePessimisticTimeout
? TimeoutStrategy.Pessimistic
: TimeoutStrategy.Optimistic,
onTimeoutAsync: (context, timespan, task) =>
{
logger.LogError(
"Database operation timed out: Timeout={Timeout}s, Strategy={Strategy}",
timespan.TotalSeconds,
config.UsePessimisticTimeout ? "Pessimistic" : "Optimistic");

return Task.CompletedTask;
});
}

/// <summary>
/// Creates a timeout policy for message broker operations.
/// </summary>
public static AsyncTimeoutPolicy CreateBrokerTimeoutPolicy(
TimeoutConfiguration config,
ILogger logger)
{
return Policy
.TimeoutAsync(
timeout: config.BrokerTimeout,
timeoutStrategy: config.UsePessimisticTimeout
? TimeoutStrategy.Pessimistic
: TimeoutStrategy.Optimistic,
onTimeoutAsync: (context, timespan, task) =>
{
logger.LogError(
"Broker operation timed out: Timeout={Timeout}s, Strategy={Strategy}",
timespan.TotalSeconds,
config.UsePessimisticTimeout ? "Pessimistic" : "Optimistic");

return Task.CompletedTask;
});
}
}
6 changes: 6 additions & 0 deletions src/StarGate.Server/appsettings.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@
"UseJitter": true
},
"Resilience": {
"Timeout": {
"HttpTimeoutSeconds": 30.0,
"DatabaseTimeoutSeconds": 10.0,
"BrokerTimeoutSeconds": 5.0,
"UsePessimisticTimeout": true
},
"Retry": {
"MaxRetryAttempts": 3,
"InitialDelaySeconds": 1.0,
Expand Down
Loading
Loading