diff --git a/src/COMP.API/COMP.API.csproj b/src/COMP.API/COMP.API.csproj index a30b77d..6931b26 100644 --- a/src/COMP.API/COMP.API.csproj +++ b/src/COMP.API/COMP.API.csproj @@ -5,14 +5,16 @@ enable enable COMP.API - Comp + COMP.API + + diff --git a/src/COMP.API/Configuration/GatewayOptions.cs b/src/COMP.API/Configuration/GatewayOptions.cs new file mode 100644 index 0000000..57bd817 --- /dev/null +++ b/src/COMP.API/Configuration/GatewayOptions.cs @@ -0,0 +1,9 @@ +namespace COMP.API.Configuration; + +public class GatewayOptions +{ + public const string SectionName = "Gateway"; + + public string? Ipfs { get; set; } + public string? Arweave { get; set; } +} diff --git a/src/COMP.API/Configuration/S3Options.cs b/src/COMP.API/Configuration/S3Options.cs new file mode 100644 index 0000000..8eefa4b --- /dev/null +++ b/src/COMP.API/Configuration/S3Options.cs @@ -0,0 +1,15 @@ +namespace COMP.API.Configuration; + +public class S3Options +{ + public const string SectionName = "S3"; + + public bool Enabled { get; set; } + public string BucketName { get; set; } = string.Empty; + public string Region { get; set; } = "us-east-1"; + public string? AccessKey { get; set; } + public string? SecretKey { get; set; } + public string? ServiceUrl { get; set; } + public string? PublicBaseUrl { get; set; } + public int MaxParallelUploads { get; set; } = 4; +} diff --git a/src/COMP.API/Extensions/ServiceCollectionExtensions.cs b/src/COMP.API/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..35a98cf --- /dev/null +++ b/src/COMP.API/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,45 @@ +using Amazon.S3; +using COMP.API.Configuration; +using COMP.API.Services; + +namespace COMP.API.Extensions; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddS3ImageUpload(this IServiceCollection services, IConfiguration configuration) + { + services.Configure(configuration.GetSection(S3Options.SectionName)); + services.Configure(configuration.GetSection(GatewayOptions.SectionName)); + + S3Options s3Options = configuration.GetSection(S3Options.SectionName).Get() ?? new(); + + if (!s3Options.Enabled) + return services; + + services.AddSingleton(_ => + { + AmazonS3Config config = new() + { + RegionEndpoint = Amazon.RegionEndpoint.GetBySystemName(s3Options.Region) + }; + + if (!string.IsNullOrWhiteSpace(s3Options.ServiceUrl)) + { + config.ServiceURL = s3Options.ServiceUrl; + config.ForcePathStyle = true; + } + + return new AmazonS3Client(s3Options.AccessKey, s3Options.SecretKey, config); + }); + + services.AddSingleton(); + services.AddSingleton(); + services.AddHttpClient("ImageFetch", client => + { + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.Add("User-Agent", "COMP-API/1.0"); + }); + + return services; + } +} diff --git a/src/COMP.API/Modules/Handlers/MetadataHandler.cs b/src/COMP.API/Modules/Handlers/MetadataHandler.cs index c469182..8120db2 100644 --- a/src/COMP.API/Modules/Handlers/MetadataHandler.cs +++ b/src/COMP.API/Modules/Handlers/MetadataHandler.cs @@ -2,12 +2,14 @@ using Microsoft.EntityFrameworkCore; using LinqKit; using COMP.Data.Data; +using COMP.API.Services; namespace COMP.API.Modules.Handlers; public class MetadataHandler ( - IDbContextFactory _dbContextFactory + IDbContextFactory _dbContextFactory, + ImageUploadService? _imageUploadService = null ) { // Fetch data by subject (checks both registry and on-chain tables) @@ -29,6 +31,13 @@ public async Task GetTokenMetadataAsync(string subject) return Results.NotFound(); // Prioritize on-chain data, fall back to registry + string? logo = onChainToken?.Logo ?? registryToken?.Logo; + string? cachedLogo = _imageUploadService is not null + ? await _imageUploadService.GetCachedUrlAsync(logo) + : null; + + if (cachedLogo is null) + _imageUploadService?.TryEnqueueUpload(subject, logo); return Results.Ok(new { @@ -36,7 +45,7 @@ public async Task GetTokenMetadataAsync(string subject) policyId = onChainToken?.PolicyId ?? registryToken?.PolicyId ?? "", name = onChainToken?.Name ?? registryToken?.Name, ticker = registryToken?.Ticker, - logo = onChainToken?.Logo ?? registryToken?.Logo, + logo = cachedLogo ?? logo, description = onChainToken?.Description ?? registryToken?.Description, decimals = onChainToken?.Decimals ?? registryToken?.Decimals ?? 0, quantity = onChainToken?.Quantity, @@ -79,7 +88,7 @@ public async Task BatchTokenMetadataAsync( { string lowerPolicyId = policyId.ToLowerInvariant(); registryPredicate = registryPredicate.And(token => - token.Subject.Substring(0, 56).Equals(lowerPolicyId, StringComparison.CurrentCultureIgnoreCase)); + token.Subject.Substring(0, 56).ToLower() == lowerPolicyId); } if (requireName) registryPredicate = registryPredicate.And(token => !string.IsNullOrEmpty(token.Name)); diff --git a/src/COMP.API/Program.cs b/src/COMP.API/Program.cs index 8739614..bf9ff26 100644 --- a/src/COMP.API/Program.cs +++ b/src/COMP.API/Program.cs @@ -3,6 +3,7 @@ using Scalar.AspNetCore; using Microsoft.EntityFrameworkCore; using COMP.Data.Data; +using COMP.API.Extensions; using COMP.API.Modules.Handlers; WebApplicationBuilder builder = WebApplication.CreateBuilder(args); @@ -10,7 +11,8 @@ builder.Services.AddDbContextFactory(options => options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection"))); -builder.Services.AddSingleton(); +builder.Services.AddScoped(); +builder.Services.AddS3ImageUpload(builder.Configuration); builder.Services.AddFastEndpoints(); builder.Services.SwaggerDocument(o => @@ -45,4 +47,4 @@ .WithOpenApiRoutePattern("/swagger/{documentName}/swagger.json") ); -app.Run(); \ No newline at end of file +app.Run(); diff --git a/src/COMP.API/Services/AwsS3Service.cs b/src/COMP.API/Services/AwsS3Service.cs new file mode 100644 index 0000000..4355d09 --- /dev/null +++ b/src/COMP.API/Services/AwsS3Service.cs @@ -0,0 +1,61 @@ +using Amazon.S3; +using Amazon.S3.Model; +using COMP.API.Configuration; +using Microsoft.Extensions.Options; + +namespace COMP.API.Services; + +public class AwsS3Service( + IAmazonS3 s3Client, + IOptions s3Options, + ILogger logger) +{ + private readonly S3Options _options = s3Options.Value; + + public async Task UploadAsync(string key, byte[] content, string? contentType = null, CancellationToken ct = default) + { + using MemoryStream stream = new(content); + PutObjectRequest request = new() + { + BucketName = _options.BucketName, + Key = key, + InputStream = stream, + ContentType = contentType ?? "application/octet-stream", + CannedACL = S3CannedACL.PublicRead + }; + + await s3Client.PutObjectAsync(request, ct); + + string publicUrl = GetPublicUrl(key); + logger.LogInformation("Uploaded to S3: {Key} ({Size} bytes)", key, content.Length); + + return publicUrl; + } + + public async Task ExistsAsync(string key, CancellationToken ct = default) + { + try + { + GetObjectMetadataRequest request = new() + { + BucketName = _options.BucketName, + Key = key + }; + + await s3Client.GetObjectMetadataAsync(request, ct); + return true; + } + catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) + { + return false; + } + } + + public string GetPublicUrl(string key) + { + if (!string.IsNullOrEmpty(_options.PublicBaseUrl)) + return $"{_options.PublicBaseUrl.TrimEnd('/')}/{key}"; + + return $"https://{_options.BucketName}.s3.{_options.Region}.amazonaws.com/{key}"; + } +} diff --git a/src/COMP.API/Services/ImageUploadService.cs b/src/COMP.API/Services/ImageUploadService.cs new file mode 100644 index 0000000..121e648 --- /dev/null +++ b/src/COMP.API/Services/ImageUploadService.cs @@ -0,0 +1,224 @@ +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using COMP.API.Configuration; +using COMP.API.Services; +using HeyRed.Mime; +using Microsoft.Extensions.Options; + +namespace COMP.API.Services; + +public partial class ImageUploadService( + IOptions s3Options, + IOptions gatewayOptions, + IHttpClientFactory httpClientFactory, + ILogger logger, + AwsS3Service? s3Service = null) +{ + private readonly S3Options _s3Options = s3Options.Value; + private readonly GatewayOptions _gatewayOptions = gatewayOptions.Value; + private readonly SemaphoreSlim _semaphore = new(s3Options.Value.MaxParallelUploads > 0 ? s3Options.Value.MaxParallelUploads : 4); + + public void TryEnqueueUpload(string subject, string? logoUri) + { + if (!_s3Options.Enabled || s3Service is null || string.IsNullOrEmpty(logoUri)) + return; + + _ = UploadAsync(subject, logoUri); + } + + public async Task GetCachedUrlAsync(string? logoUri) + { + if (!_s3Options.Enabled || s3Service is null || string.IsNullOrEmpty(logoUri)) + return null; + + string key = GetKeyFromUri(logoUri); + if (string.IsNullOrEmpty(key)) + return null; + + if (await s3Service.ExistsAsync(key)) + return s3Service.GetPublicUrl(key); + + return null; + } + + private async Task UploadAsync(string subject, string logoUri) + { + await _semaphore.WaitAsync(); + try + { + (byte[]? imageData, string? contentType, string? error) = await ResolveImageAsync(logoUri, CancellationToken.None); + + if (imageData is null || error is not null) + { + logger.LogWarning("Failed to resolve image for {Subject}: {Error}", subject, error); + return; + } + + string s3Key = GetKeyFromUri(logoUri, imageData); + + // Check if already exists before uploading + if (await s3Service!.ExistsAsync(s3Key)) + { + logger.LogDebug("Image already exists for {Subject}: {Key}", subject, s3Key); + return; + } + + await s3Service.UploadAsync(s3Key, imageData, contentType); + logger.LogInformation("Uploaded image for {Subject}: {S3Key}", subject, s3Key); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to upload image for subject {Subject}", subject); + } + finally + { + _semaphore.Release(); + } + } + + private async Task<(byte[]? Data, string? ContentType, string? Error)> ResolveImageAsync(string logoUri, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(logoUri)) + return (null, null, "Empty logo URI"); + + if (logoUri.StartsWith("data:image", StringComparison.OrdinalIgnoreCase)) + return DecodeBase64DataUri(logoUri); + + if (logoUri.StartsWith("ipfs://", StringComparison.OrdinalIgnoreCase)) + return await FetchFromIpfsAsync(logoUri, ct); + + if (logoUri.StartsWith("ar://", StringComparison.OrdinalIgnoreCase)) + return await FetchFromArweaveAsync(logoUri, ct); + + if (logoUri.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + logoUri.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + return await FetchFromUrlAsync(logoUri, ct); + + return (null, null, $"Unsupported URI scheme: {logoUri}"); + } + + private static (byte[]? Data, string? ContentType, string? Error) DecodeBase64DataUri(string dataUri) + { + Match match = DataUriRegex().Match(dataUri); + if (!match.Success) + return (null, null, "Invalid data URI format"); + + string contentType = match.Groups[1].Value; + string base64Data = match.Groups[2].Value; + + try + { + byte[] data = Convert.FromBase64String(base64Data); + return (data, contentType, null); + } + catch (FormatException) + { + return (null, null, "Invalid base64 data"); + } + } + + private async Task<(byte[]? Data, string? ContentType, string? Error)> FetchFromIpfsAsync(string ipfsUri, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(_gatewayOptions.Ipfs)) + return (null, null, "IPFS gateway not configured"); + + string cid = ipfsUri[7..]; + string gatewayUrl = _gatewayOptions.Ipfs.TrimEnd('/') + "/" + cid; + + return await FetchFromUrlAsync(gatewayUrl, ct); + } + + private async Task<(byte[]? Data, string? ContentType, string? Error)> FetchFromArweaveAsync(string arUri, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(_gatewayOptions.Arweave)) + return (null, null, "Arweave gateway not configured"); + + string txId = arUri[5..]; + string gatewayUrl = _gatewayOptions.Arweave.TrimEnd('/') + "/" + txId; + + return await FetchFromUrlAsync(gatewayUrl, ct); + } + + private async Task<(byte[]? Data, string? ContentType, string? Error)> FetchFromUrlAsync(string url, CancellationToken ct) + { + try + { + HttpClient client = httpClientFactory.CreateClient("ImageFetch"); + using HttpResponseMessage response = await client.GetAsync(url, ct); + + if (!response.IsSuccessStatusCode) + return (null, null, $"HTTP {(int)response.StatusCode}: {response.ReasonPhrase}"); + + byte[] data = await response.Content.ReadAsByteArrayAsync(ct); + string? contentType = response.Content.Headers.ContentType?.MediaType; + + // Fallback to byte detection if content type is missing or generic + if (string.IsNullOrWhiteSpace(contentType) || contentType == "application/octet-stream") + contentType = MimeGuesser.GuessMimeType(data); + + return (data, contentType, null); + } + catch (HttpRequestException ex) + { + return (null, null, $"HTTP error: {ex.Message}"); + } + catch (TaskCanceledException) + { + return (null, null, "Request timed out"); + } + } + + private static string GetKeyFromUri(string logoUri) + { + // Use the CID directly from IPFS URIs + if (logoUri.StartsWith("ipfs://", StringComparison.OrdinalIgnoreCase)) + return logoUri[7..]; + + // Use the transaction ID directly from Arweave URIs + if (logoUri.StartsWith("ar://", StringComparison.OrdinalIgnoreCase)) + return logoUri[5..]; + + // For base64, hash the data URI string + if (logoUri.StartsWith("data:image", StringComparison.OrdinalIgnoreCase)) + { + byte[] hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(logoUri)); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + // For http URLs, we can't determine key without fetching + return string.Empty; + } + + private static string GetKeyFromUri(string logoUri, byte[] imageData) + { + // Use the CID directly from IPFS URIs + if (logoUri.StartsWith("ipfs://", StringComparison.OrdinalIgnoreCase)) + return logoUri[7..]; + + // Use the transaction ID directly from Arweave URIs + if (logoUri.StartsWith("ar://", StringComparison.OrdinalIgnoreCase)) + return logoUri[5..]; + + // For other sources (base64, http), use content hash + byte[] hash = SHA256.HashData(imageData); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static string GetExtensionFromContentType(string? contentType) => + contentType?.ToLowerInvariant() switch + { + "image/png" => ".png", + "image/jpeg" => ".jpg", + "image/jpg" => ".jpg", + "image/gif" => ".gif", + "image/webp" => ".webp", + "image/svg+xml" => ".svg", + "image/bmp" => ".bmp", + "image/x-icon" => ".ico", + "image/avif" => ".avif", + _ => string.Empty + }; + + [GeneratedRegex(@"^data:(image/[^;]+);base64,(.+)$", RegexOptions.Singleline)] + private static partial Regex DataUriRegex(); +}