diff --git a/README.md b/README.md index 0283866..8b23a32 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@
- .NET + .NET PostgreSQL @@ -32,7 +32,7 @@ ## 📖 Overview -The Cardano Open Metadata Project (COMP) is a unified metadata specification and high-performance reference implementation for the Cardano blockchain. COMP defines standardized REST APIs that aggregate all Cardano metadata types - tokens, NFTs, stake pools, and DReps - from multiple sources into a single, queryable interface. Built with modern .NET 9.0 and PostgreSQL, our reference implementation demonstrates sub-millisecond query performance while establishing COMP as the definitive metadata standard for the Cardano ecosystem. +The Cardano Open Metadata Project (COMP) is a unified metadata specification and high-performance reference implementation for the Cardano blockchain. COMP defines standardized REST APIs that aggregate Cardano metadata from multiple sources into a single, queryable interface. Built with modern .NET 10.0 and PostgreSQL, the current implementation focuses on token registry and on-chain token metadata (CIP-25/CIP-68). **Key Features:** @@ -82,7 +82,7 @@ graph LR ### Prerequisites -- .NET 9.0 SDK +- .NET 10.0 SDK - PostgreSQL 15+ - GitHub Personal Access Token (for API rate limits) @@ -112,18 +112,31 @@ cd COMP } ``` -3. **Run database migrations** +3. **Start local dependencies (why `docker-compose.yml` is in repo root)** +```bash +docker compose up -d db +``` + +The root `docker-compose.yml` is used for local development orchestration. It starts the shared PostgreSQL service and the API container from the same repository context, so Docker can access `src/` and `infrastructure/` paths directly. + +4. **Run database migrations** (optional if running `COMP.Sync`, which applies migrations on startup) ```bash dotnet ef database update ``` -4. **Start the service** +5. **Start the API service** ```bash -dotnet run +dotnet run --project src/COMP.API ``` -The service will start syncing with the GitHub Token Registry and expose APIs on: -- `https://localhost:7276` (HTTPS) or `http://localhost:5146` (HTTP) +6. **(Optional) Start the sync worker** +```bash +dotnet run --project src/COMP.Sync +``` + +API docs in development: +- `http://localhost:5001/scalar` +- `http://localhost:5001/openapi/v1.json` ## 📡 API Endpoints @@ -268,7 +281,7 @@ COMP implementations must handle: Our reference implementation demonstrates the COMP specification using: ### Core Technologies -- **.NET 9.0** - High-performance runtime with modern C# features +- **.NET 10.0** - High-performance runtime with modern C# features - **PostgreSQL** - Scalable database for metadata storage - **FastEndpoints** - Lightweight API framework for REST endpoints @@ -326,6 +339,14 @@ COMP implementations must include: - **Secrets Management** - Secure storage for credentials and keys - **CORS Policy** - Configure appropriate cross-origin policies +## 🖼️ Image Upload Service Status + +Image upload/caching is implemented on the feature branch `feat/api/image-upload-service`. + +- Reference: `docs/IMAGE_UPLOAD_SERVICE.md` +- Implementation branch: `feat/api/image-upload-service` +- Mainline note: if this branch is not merged into your current checkout, image upload will not be active at runtime. + ## 🤝 Contributing We welcome contributions! Please follow these steps: @@ -351,4 +372,4 @@ We welcome contributions! Please follow these steps: Request Feature • Documentation

- \ No newline at end of file + diff --git a/docker-compose.yml b/docker-compose.yml index c5bb2d4..8bc1400 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: app: build: context: . - dockerfile: Dockerfile + dockerfile: infrastructure/comp-api/Dockerfile container_name: comp-app restart: unless-stopped depends_on: diff --git a/infrastructure/comp-api/Dockerfile b/infrastructure/comp-api/Dockerfile index ff1ccd3..50f9f5c 100644 --- a/infrastructure/comp-api/Dockerfile +++ b/infrastructure/comp-api/Dockerfile @@ -18,7 +18,7 @@ COPY infrastructure/comp-api/production-config.json ./COMP.API/appsettings.json RUN dotnet publish COMP.API/COMP.API.csproj -c Release -o /app/publish # Runtime stage -FROM mcr.microsoft.com/dotnet/aspnet:9.0 +FROM mcr.microsoft.com/dotnet/aspnet:10.0 WORKDIR /app # Copy published app @@ -32,4 +32,4 @@ ENV ASPNETCORE_ENVIRONMENT=Production EXPOSE 8080 # Run the application -ENTRYPOINT ["dotnet", "COMP.API.dll"] \ No newline at end of file +ENTRYPOINT ["dotnet", "COMP.API.dll"] diff --git a/infrastructure/comp-sync/Dockerfile b/infrastructure/comp-sync/Dockerfile index fc0d974..827dc80 100644 --- a/infrastructure/comp-sync/Dockerfile +++ b/infrastructure/comp-sync/Dockerfile @@ -4,7 +4,6 @@ WORKDIR /src # Copy project files and restore COPY src/COMP.Data/COMP.Data.csproj src/COMP.Data/ -COPY src/COMP.API/COMP.API.csproj src/COMP.API/ COPY src/COMP.Sync/COMP.Sync.csproj src/COMP.Sync/ RUN dotnet restore "src/COMP.Sync/COMP.Sync.csproj" diff --git a/src/COMP.API/COMP.API.csproj b/src/COMP.API/COMP.API.csproj index a30b77d..f0bebf3 100644 --- a/src/COMP.API/COMP.API.csproj +++ b/src/COMP.API/COMP.API.csproj @@ -5,15 +5,17 @@ enable enable COMP.API - Comp + COMP.API - - + + + - + + diff --git a/src/COMP.API/Endpoints/BatchTokenMetadataEndpoint.cs b/src/COMP.API/Endpoints/BatchTokenMetadataEndpoint.cs index 72e7640..f738a52 100644 --- a/src/COMP.API/Endpoints/BatchTokenMetadataEndpoint.cs +++ b/src/COMP.API/Endpoints/BatchTokenMetadataEndpoint.cs @@ -1,5 +1,5 @@ using FastEndpoints; -using COMP.API.Modules.Handlers; +using COMP.API.Handlers; using COMP.Data.Models.Request; namespace COMP.API.Endpoints; @@ -14,8 +14,7 @@ public override void Configure() AllowAnonymous(); Description(b => b .WithName("BatchTokenMetadata") - .WithSummary("Retrieve token metadata for a batch of subjects") - .WithTags("Metadata")); + .WithSummary("Retrieve token metadata for a batch of subjects")); } public override async Task HandleAsync(BatchTokenMetadataRequest req, CancellationToken ct) @@ -29,11 +28,9 @@ public override async Task HandleAsync(BatchTokenMetadataRequest req, Cancellati req.Offset, req.IncludeEmptyName, req.IncludeEmptyLogo, - req.IncludeEmptyTicker); - - if (result is IResult httpResult) - { - await SendResultAsync(httpResult); - } + req.IncludeEmptyTicker, + ct); + + await result.ExecuteAsync(HttpContext); } } diff --git a/src/COMP.API/Endpoints/GetTokenMetadataEndpoint.cs b/src/COMP.API/Endpoints/GetTokenMetadataEndpoint.cs index c388166..488e692 100644 --- a/src/COMP.API/Endpoints/GetTokenMetadataEndpoint.cs +++ b/src/COMP.API/Endpoints/GetTokenMetadataEndpoint.cs @@ -1,5 +1,5 @@ using FastEndpoints; -using COMP.API.Modules.Handlers; +using COMP.API.Handlers; using COMP.Data.Models.Request; namespace COMP.API.Endpoints; @@ -14,17 +14,12 @@ public override void Configure() AllowAnonymous(); Description(b => b .WithName("GetTokenMetadata") - .WithSummary("Retrieve token metadata by subject") - .WithTags("Metadata")); + .WithSummary("Retrieve token metadata by subject")); } public override async Task HandleAsync(GetTokenMetadataRequest req, CancellationToken ct) { - IResult result = await _metadataHandler.GetTokenMetadataAsync(req.Subject); - - if (result is IResult httpResult) - { - await SendResultAsync(httpResult); - } + IResult result = await _metadataHandler.GetTokenMetadataAsync(req.Subject, ct); + await result.ExecuteAsync(HttpContext); } } diff --git a/src/COMP.API/Modules/Handlers/MetadataHandler.cs b/src/COMP.API/Handlers/MetadataHandler.cs similarity index 71% rename from src/COMP.API/Modules/Handlers/MetadataHandler.cs rename to src/COMP.API/Handlers/MetadataHandler.cs index c469182..d9a4ca3 100644 --- a/src/COMP.API/Modules/Handlers/MetadataHandler.cs +++ b/src/COMP.API/Handlers/MetadataHandler.cs @@ -1,34 +1,32 @@ +using COMP.Data.Data; using COMP.Data.Models.Entity; -using Microsoft.EntityFrameworkCore; using LinqKit; -using COMP.Data.Data; +using Microsoft.EntityFrameworkCore; -namespace COMP.API.Modules.Handlers; +namespace COMP.API.Handlers; -public class MetadataHandler -( - IDbContextFactory _dbContextFactory -) +public class MetadataHandler(IDbContextFactory _dbContextFactory) { // Fetch data by subject (checks both registry and on-chain tables) - public async Task GetTokenMetadataAsync(string subject) + public async Task GetTokenMetadataAsync(string subject, CancellationToken cancellationToken = default) { - await using MetadataDbContext db = await _dbContextFactory.CreateDbContextAsync(); + await using MetadataDbContext db = await _dbContextFactory.CreateDbContextAsync(cancellationToken); // Query both tables sequentially - TokenMetadata? registryToken = await db.TokenMetadata + TokenMetadataRegistry? registryToken = await db.TokenMetadataRegistry .AsNoTracking() - .FirstOrDefaultAsync(t => t.Subject == subject); + .FirstOrDefaultAsync(t => t.Subject == subject, cancellationToken); TokenMetadataOnChain? onChainToken = await db.TokenMetadataOnChain .AsNoTracking() - .FirstOrDefaultAsync(t => t.Subject == subject); + .FirstOrDefaultAsync(t => t.Subject == subject, cancellationToken); // If both are null, return 404 if (registryToken is null && onChainToken is null) return Results.NotFound(); // Prioritize on-chain data, fall back to registry + string? logo = onChainToken?.Logo ?? registryToken?.Logo; return Results.Ok(new { @@ -36,7 +34,7 @@ public async Task GetTokenMetadataAsync(string subject) policyId = onChainToken?.PolicyId ?? registryToken?.PolicyId ?? "", name = onChainToken?.Name ?? registryToken?.Name, ticker = registryToken?.Ticker, - logo = onChainToken?.Logo ?? registryToken?.Logo, + logo, description = onChainToken?.Description ?? registryToken?.Description, decimals = onChainToken?.Decimals ?? registryToken?.Decimals ?? 0, quantity = onChainToken?.Quantity, @@ -59,7 +57,8 @@ public async Task BatchTokenMetadataAsync( int? offset, bool? includeEmptyName, bool? includeEmptyLogo, - bool? includeEmptyTicker) + bool? includeEmptyTicker, + CancellationToken cancellationToken = default) { if (subjects == null || subjects.Count == 0) return Results.BadRequest("No subjects provided."); @@ -72,15 +71,15 @@ public async Task BatchTokenMetadataAsync( List distinctSubjects = [.. subjects.Distinct()]; // Build predicate for registry metadata - ExpressionStarter registryPredicate = PredicateBuilder.New(false); + ExpressionStarter registryPredicate = PredicateBuilder.New(false); registryPredicate = registryPredicate.Or(token => distinctSubjects.Contains(token.Subject)); - if (!string.IsNullOrWhiteSpace(policyId)) - { - string lowerPolicyId = policyId.ToLowerInvariant(); - registryPredicate = registryPredicate.And(token => - token.Subject.Substring(0, 56).Equals(lowerPolicyId, StringComparison.CurrentCultureIgnoreCase)); - } + string? normalizedPolicyId = string.IsNullOrWhiteSpace(policyId) + ? null + : policyId.Trim().ToLowerInvariant(); + + if (!string.IsNullOrWhiteSpace(normalizedPolicyId)) + registryPredicate = registryPredicate.And(token => token.Subject.StartsWith(normalizedPolicyId)); if (requireName) registryPredicate = registryPredicate.And(token => !string.IsNullOrEmpty(token.Name)); @@ -105,9 +104,9 @@ public async Task BatchTokenMetadataAsync( ExpressionStarter onChainPredicate = PredicateBuilder.New(false); onChainPredicate = onChainPredicate.Or(token => distinctSubjects.Contains(token.Subject)); - if (!string.IsNullOrWhiteSpace(policyId)) + if (!string.IsNullOrWhiteSpace(normalizedPolicyId)) { - onChainPredicate = onChainPredicate.And(token => token.PolicyId == policyId); + onChainPredicate = onChainPredicate.And(token => token.PolicyId == normalizedPolicyId); } if (requireName) onChainPredicate = onChainPredicate.And(token => !string.IsNullOrEmpty(token.Name)); @@ -122,32 +121,36 @@ public async Task BatchTokenMetadataAsync( (token.Description != null && EF.Functions.ILike(token.Description, $"%{searchText}%"))); } - await using MetadataDbContext db = await _dbContextFactory.CreateDbContextAsync(); + await using MetadataDbContext db = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - // Query both tables sequentially - List registryTokens = await db.TokenMetadata + // Query both tables sequentially (DbContext is not thread-safe) + List registryTokens = await db.TokenMetadataRegistry .AsNoTracking() .Where(registryPredicate) - .ToListAsync(); + .ToListAsync(cancellationToken); List onChainTokens = await db.TokenMetadataOnChain .AsNoTracking() .Where(onChainPredicate) - .ToListAsync(); + .ToListAsync(cancellationToken); + + // Convert to dictionaries for O(1) lookups instead of O(n) FirstOrDefault + Dictionary registryBySubject = registryTokens.ToDictionary(t => t.Subject); + Dictionary onChainBySubject = onChainTokens.ToDictionary(t => t.Subject); // Merge results - prioritize on-chain, combine with registry var mergedResults = distinctSubjects .Select(subject => { - TokenMetadata? registry = registryTokens.FirstOrDefault(t => t.Subject == subject); - TokenMetadataOnChain? onChain = onChainTokens.FirstOrDefault(t => t.Subject == subject); + registryBySubject.TryGetValue(subject, out TokenMetadataRegistry? registry); + onChainBySubject.TryGetValue(subject, out TokenMetadataOnChain? onChain); if (registry is null && onChain is null) return null; return new { - subject = subject, + subject, policyId = onChain?.PolicyId ?? registry?.PolicyId ?? "", name = onChain?.Name ?? registry?.Name, ticker = registry?.Ticker, @@ -164,15 +167,18 @@ public async Task BatchTokenMetadataAsync( }; }) .Where(t => t is not null) + .Select(t => t!) .ToList(); + if (requireTicker) + mergedResults = [.. mergedResults.Where(t => !string.IsNullOrEmpty(t.ticker))]; + int total = mergedResults.Count; - // Apply pagination + // Apply pagination: offset always applies, limit is optional + mergedResults = [.. mergedResults.Skip(effectiveOffset)]; if (limit.HasValue) - { - mergedResults = mergedResults.Skip(effectiveOffset).Take(limit.Value).ToList(); - } + mergedResults = [.. mergedResults.Take(limit.Value)]; if (mergedResults.Count == 0) return Results.NotFound("No tokens found for the given subjects."); diff --git a/src/COMP.API/Program.cs b/src/COMP.API/Program.cs index 8739614..8145ac8 100644 --- a/src/COMP.API/Program.cs +++ b/src/COMP.API/Program.cs @@ -3,7 +3,7 @@ using Scalar.AspNetCore; using Microsoft.EntityFrameworkCore; using COMP.Data.Data; -using COMP.API.Modules.Handlers; +using COMP.API.Handlers; WebApplicationBuilder builder = WebApplication.CreateBuilder(args); @@ -11,38 +11,25 @@ options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection"))); builder.Services.AddSingleton(); - builder.Services.AddFastEndpoints(); -builder.Services.SwaggerDocument(o => -{ - o.DocumentSettings = s => - { - s.Title = "COMP API"; - s.Version = "v1"; - }; -}); - -builder.Services.AddOpenApi(); +builder.Services.SwaggerDocument(); WebApplication app = builder.Build(); +app.UseFastEndpoints(); + if (app.Environment.IsDevelopment()) { - app.MapOpenApi(); + app.UseSwaggerGen(options => + { + options.Path = "/openapi/{documentName}.json"; + }); + app.MapScalarApiReference(options => + { + options.Title = "COMP API Documentation"; + options.Theme = ScalarTheme.Default; + options.ShowSidebar = true; + }); } -app.UseHttpsRedirection(); - -app.UseFastEndpoints(); - -app.UseSwaggerGen(); - -app.MapScalarApiReference(options => - options - .WithTitle("COMP API") - .WithTheme(ScalarTheme.Purple) - .WithDefaultHttpClient(ScalarTarget.CSharp, ScalarClient.HttpClient) - .WithOpenApiRoutePattern("/swagger/{documentName}/swagger.json") -); - app.Run(); \ No newline at end of file diff --git a/src/COMP.API/Properties/launchSettings.json b/src/COMP.API/Properties/launchSettings.json new file mode 100644 index 0000000..27465e3 --- /dev/null +++ b/src/COMP.API/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "COMP.API": { + "commandName": "Project", + "launchBrowser": false, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "http://localhost:5001" + } + } +} diff --git a/src/COMP.API/appsettings.example.json b/src/COMP.API/appsettings.example.json new file mode 100644 index 0000000..6d79722 --- /dev/null +++ b/src/COMP.API/appsettings.example.json @@ -0,0 +1,24 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "ConnectionStrings": { + "DefaultConnection": "Host=localhost;Port=5432;Database=cardano_metadata;Username=postgres;Password=postgres" + }, + "AllowedHosts": "*", + "S3": { + "Enabled": false, + "BucketName": "my-token-logos", + "Region": "us-east-1", + "AccessKey": "", + "SecretKey": "", + "ServiceUrl": null, + "PublicBaseUrl": null, + "IpfsGateway": null, + "ArweaveGateway": null, + "MaxParallelUploads": 4 + } +} diff --git a/src/COMP.Data/Data/MetadataDbContext.cs b/src/COMP.Data/Data/MetadataDbContext.cs index 6bde345..c14ed9a 100644 --- a/src/COMP.Data/Data/MetadataDbContext.cs +++ b/src/COMP.Data/Data/MetadataDbContext.cs @@ -7,20 +7,20 @@ namespace COMP.Data.Data; public class MetadataDbContext(DbContextOptions options, IConfiguration configuration) : CardanoDbContext(options, configuration) { - public DbSet TokenMetadata => Set(); - public DbSet SyncState => Set(); + public DbSet TokenMetadataRegistry => Set(); + public DbSet RegistrySyncState => Set(); public DbSet TokenMetadataOnChain => Set(); protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); - modelBuilder.Entity().HasKey(tmd => tmd.Subject); - modelBuilder.Entity() + modelBuilder.Entity().HasKey(tmd => tmd.Subject); + modelBuilder.Entity() .HasIndex(tmd => new { tmd.Name, tmd.Description, tmd.Ticker }) - .HasDatabaseName("IX_TokenMetadata_Name_Description_Ticker"); + .HasDatabaseName("IX_TokenMetadataRegistry_Name_Description_Ticker"); - modelBuilder.Entity().HasKey(ss => ss.Hash); + modelBuilder.Entity().HasKey(ss => ss.Hash); modelBuilder.Entity(entity => { diff --git a/src/COMP.Data/Migrations/20260130162627_RenameRegistryTables.Designer.cs b/src/COMP.Data/Migrations/20260130162627_RenameRegistryTables.Designer.cs new file mode 100644 index 0000000..c437db0 --- /dev/null +++ b/src/COMP.Data/Migrations/20260130162627_RenameRegistryTables.Designer.cs @@ -0,0 +1,146 @@ +// +using System; +using COMP.Data.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Comp.Migrations +{ + [DbContext(typeof(MetadataDbContext))] + [Migration("20260130162627_RenameRegistryTables")] + partial class RenameRegistryTables + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("public") + .HasAnnotation("ProductVersion", "10.0.2") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Argus.Sync.Data.Models.ReducerState", b => + { + b.Property("Name") + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LatestIntersectionsJson") + .IsRequired() + .HasColumnType("text"); + + b.Property("StartIntersectionJson") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Name"); + + b.ToTable("ReducerStates", "public"); + }); + + modelBuilder.Entity("COMP.Data.Models.Entity.RegistrySyncState", b => + { + b.Property("Hash") + .HasColumnType("text"); + + b.Property("Date") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Hash"); + + b.ToTable("RegistrySyncState", "public"); + }); + + modelBuilder.Entity("COMP.Data.Models.Entity.TokenMetadataOnChain", b => + { + b.Property("Subject") + .HasColumnType("text"); + + b.Property("AssetName") + .IsRequired() + .HasColumnType("text"); + + b.Property("Decimals") + .HasColumnType("integer"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("Logo") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("PolicyId") + .IsRequired() + .HasColumnType("text"); + + b.Property("Quantity") + .HasColumnType("bigint"); + + b.Property("TokenType") + .HasColumnType("integer"); + + b.HasKey("Subject"); + + b.HasIndex("PolicyId"); + + b.ToTable("TokenMetadataOnChain", "public"); + }); + + modelBuilder.Entity("COMP.Data.Models.Entity.TokenMetadataRegistry", b => + { + b.Property("Subject") + .HasColumnType("text"); + + b.Property("Decimals") + .HasColumnType("integer"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Logo") + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Policy") + .HasColumnType("text"); + + b.Property("PolicyId") + .IsRequired() + .HasColumnType("text"); + + b.Property("Ticker") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .HasColumnType("text"); + + b.HasKey("Subject"); + + b.HasIndex("Name", "Description", "Ticker") + .HasDatabaseName("IX_TokenMetadataRegistry_Name_Description_Ticker"); + + b.ToTable("TokenMetadataRegistry", "public"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/COMP.Data/Migrations/20260130162627_RenameRegistryTables.cs b/src/COMP.Data/Migrations/20260130162627_RenameRegistryTables.cs new file mode 100644 index 0000000..b57c5e4 --- /dev/null +++ b/src/COMP.Data/Migrations/20260130162627_RenameRegistryTables.cs @@ -0,0 +1,84 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Comp.Migrations +{ + /// + public partial class RenameRegistryTables : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + // Rename tables + migrationBuilder.RenameTable( + name: "SyncState", + schema: "public", + newName: "RegistrySyncState", + newSchema: "public"); + + migrationBuilder.RenameTable( + name: "TokenMetadata", + schema: "public", + newName: "TokenMetadataRegistry", + newSchema: "public"); + + // Rename primary key constraints + migrationBuilder.RenameIndex( + name: "PK_SyncState", + schema: "public", + table: "RegistrySyncState", + newName: "PK_RegistrySyncState"); + + migrationBuilder.RenameIndex( + name: "PK_TokenMetadata", + schema: "public", + table: "TokenMetadataRegistry", + newName: "PK_TokenMetadataRegistry"); + + // Rename existing index + migrationBuilder.RenameIndex( + name: "IX_TokenMetadata_Name_Description_Ticker", + schema: "public", + table: "TokenMetadataRegistry", + newName: "IX_TokenMetadataRegistry_Name_Description_Ticker"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + // Rename tables back + migrationBuilder.RenameTable( + name: "RegistrySyncState", + schema: "public", + newName: "SyncState", + newSchema: "public"); + + migrationBuilder.RenameTable( + name: "TokenMetadataRegistry", + schema: "public", + newName: "TokenMetadata", + newSchema: "public"); + + // Rename primary key constraints back + migrationBuilder.RenameIndex( + name: "PK_RegistrySyncState", + schema: "public", + table: "SyncState", + newName: "PK_SyncState"); + + migrationBuilder.RenameIndex( + name: "PK_TokenMetadataRegistry", + schema: "public", + table: "TokenMetadata", + newName: "PK_TokenMetadata"); + + // Rename index back + migrationBuilder.RenameIndex( + name: "IX_TokenMetadataRegistry_Name_Description_Ticker", + schema: "public", + table: "TokenMetadata", + newName: "IX_TokenMetadata_Name_Description_Ticker"); + } + } +} diff --git a/src/COMP.Data/Migrations/MetadataDbContextModelSnapshot.cs b/src/COMP.Data/Migrations/MetadataDbContextModelSnapshot.cs index 63d8a0e..1cf2353 100644 --- a/src/COMP.Data/Migrations/MetadataDbContextModelSnapshot.cs +++ b/src/COMP.Data/Migrations/MetadataDbContextModelSnapshot.cs @@ -18,7 +18,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) #pragma warning disable 612, 618 modelBuilder .HasDefaultSchema("public") - .HasAnnotation("ProductVersion", "10.0.0") + .HasAnnotation("ProductVersion", "10.0.2") .HasAnnotation("Relational:MaxIdentifierLength", 63); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); @@ -44,7 +44,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("ReducerStates", "public"); }); - modelBuilder.Entity("COMP.Data.Models.Entity.SyncState", b => + modelBuilder.Entity("COMP.Data.Models.Entity.RegistrySyncState", b => { b.Property("Hash") .HasColumnType("text"); @@ -54,88 +54,88 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Hash"); - b.ToTable("SyncState", "public"); + b.ToTable("RegistrySyncState", "public"); }); - modelBuilder.Entity("COMP.Data.Models.Entity.TokenMetadata", b => + modelBuilder.Entity("COMP.Data.Models.Entity.TokenMetadataOnChain", b => { b.Property("Subject") .HasColumnType("text"); + b.Property("AssetName") + .IsRequired() + .HasColumnType("text"); + b.Property("Decimals") .HasColumnType("integer"); b.Property("Description") + .IsRequired() .HasColumnType("text"); b.Property("Logo") + .IsRequired() .HasColumnType("text"); b.Property("Name") .IsRequired() .HasColumnType("text"); - b.Property("Policy") - .HasColumnType("text"); - b.Property("PolicyId") .IsRequired() .HasColumnType("text"); - b.Property("Ticker") - .IsRequired() - .HasColumnType("text"); + b.Property("Quantity") + .HasColumnType("bigint"); - b.Property("Url") - .HasColumnType("text"); + b.Property("TokenType") + .HasColumnType("integer"); b.HasKey("Subject"); - b.HasIndex("Name", "Description", "Ticker") - .HasDatabaseName("IX_TokenMetadata_Name_Description_Ticker"); + b.HasIndex("PolicyId"); - b.ToTable("TokenMetadata", "public"); + b.ToTable("TokenMetadataOnChain", "public"); }); - modelBuilder.Entity("COMP.Data.Models.Entity.TokenMetadataOnChain", b => + modelBuilder.Entity("COMP.Data.Models.Entity.TokenMetadataRegistry", b => { b.Property("Subject") .HasColumnType("text"); - b.Property("AssetName") - .IsRequired() - .HasColumnType("text"); - b.Property("Decimals") .HasColumnType("integer"); b.Property("Description") - .IsRequired() .HasColumnType("text"); b.Property("Logo") - .IsRequired() .HasColumnType("text"); b.Property("Name") .IsRequired() .HasColumnType("text"); + b.Property("Policy") + .HasColumnType("text"); + b.Property("PolicyId") .IsRequired() .HasColumnType("text"); - b.Property("Quantity") - .HasColumnType("bigint"); + b.Property("Ticker") + .IsRequired() + .HasColumnType("text"); - b.Property("TokenType") - .HasColumnType("integer"); + b.Property("Url") + .HasColumnType("text"); b.HasKey("Subject"); - b.HasIndex("PolicyId"); + b.HasIndex("Name", "Description", "Ticker") + .HasDatabaseName("IX_TokenMetadataRegistry_Name_Description_Ticker"); - b.ToTable("TokenMetadataOnChain", "public"); + b.ToTable("TokenMetadataRegistry", "public"); }); #pragma warning restore 612, 618 } diff --git a/src/COMP.Data/Models/Entity/SyncState.cs b/src/COMP.Data/Models/Entity/RegistrySyncState.cs similarity index 70% rename from src/COMP.Data/Models/Entity/SyncState.cs rename to src/COMP.Data/Models/Entity/RegistrySyncState.cs index 8aec3b7..51cb4b5 100644 --- a/src/COMP.Data/Models/Entity/SyncState.cs +++ b/src/COMP.Data/Models/Entity/RegistrySyncState.cs @@ -1,6 +1,6 @@ namespace COMP.Data.Models.Entity; -public record SyncState( +public record RegistrySyncState( string Hash, DateTimeOffset Date ); diff --git a/src/COMP.Data/Models/Entity/TokenMetadata.cs b/src/COMP.Data/Models/Entity/TokenMetadataRegistry.cs similarity index 85% rename from src/COMP.Data/Models/Entity/TokenMetadata.cs rename to src/COMP.Data/Models/Entity/TokenMetadataRegistry.cs index 5e17172..eed50b2 100644 --- a/src/COMP.Data/Models/Entity/TokenMetadata.cs +++ b/src/COMP.Data/Models/Entity/TokenMetadataRegistry.cs @@ -1,6 +1,6 @@ namespace COMP.Data.Models.Entity; -public record TokenMetadata( +public record TokenMetadataRegistry( string Subject, string Name, string Ticker, diff --git a/src/COMP.Sync/COMP.Sync.csproj b/src/COMP.Sync/COMP.Sync.csproj index 0c5a3ce..73175ff 100644 --- a/src/COMP.Sync/COMP.Sync.csproj +++ b/src/COMP.Sync/COMP.Sync.csproj @@ -8,13 +8,14 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + - - contentfiles - diff --git a/src/COMP.Sync/Program.cs b/src/COMP.Sync/Program.cs index 4dba79e..71cce5e 100644 --- a/src/COMP.Sync/Program.cs +++ b/src/COMP.Sync/Program.cs @@ -3,7 +3,6 @@ using Microsoft.EntityFrameworkCore; using COMP.Data.Data; using COMP.Sync.Services; -using COMP.API.Modules.Handlers; using COMP.Sync.Reducers; using System.Net.Http.Headers; using System.Reflection; @@ -15,7 +14,6 @@ // Register reducers - the assembly scanning will find CIP25Reducer builder.Services.AddReducers(builder.Configuration); -builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(); diff --git a/src/COMP.Sync/Reducers/CIP25Reducer.cs b/src/COMP.Sync/Reducers/CIP25Reducer.cs index 7083388..6c2fc19 100644 --- a/src/COMP.Sync/Reducers/CIP25Reducer.cs +++ b/src/COMP.Sync/Reducers/CIP25Reducer.cs @@ -20,9 +20,12 @@ ILogger logger private static readonly string[] ValidSchemes = ["https://", "http://", "ipfs://", "ar://", "data:"]; private static readonly Regex DataUriRegex = MyRegex(); + // Rollback is intentionally not implemented - this metadata indexer is append-only + // and doesn't track historical state. On-chain metadata updates are rare and + // rollbacks would require re-syncing from scratch. public async Task RollBackwardAsync(ulong slot) { - logger.LogWarning("Rollback requested to slot {Slot}. Manual resync may be required as we don't maintain historical state.", slot); + logger.LogWarning("Rollback requested to slot {Slot}. Manual resync may be required.", slot); await Task.CompletedTask; } @@ -31,78 +34,61 @@ public async Task RollForwardAsync(Block block) List txBodies = [.. block.TransactionBodies()]; Dictionary auxiliaryDataDict = block.AuxiliaryDataSet().ToDictionary(kvp => kvp.Key, kvp => kvp.Value); - Dictionary tokensWithMetadata = []; - - for (int i = 0; i < txBodies.Count; i++) - { - TransactionBody tx = txBodies[i]; - Dictionary? mint = tx.Mint(); - if (mint == null || mint.Count == 0) - continue; - - if (!auxiliaryDataDict.TryGetValue(i, out AuxiliaryData? auxData)) - continue; - - Metadata? metadata = auxData.Metadata(); - if (metadata == null) - continue; - - if (!metadata.Value().TryGetValue(721, out TransactionMetadatum? cip25Value)) - continue; - - if (cip25Value is not MetadatumMap rootMap) - continue; - - foreach (KeyValuePair policyEntry in mint) + // Extract all potential tokens with CIP-25 metadata + List allTokens = [.. txBodies + .Select((tx, index) => (tx, index)) + .Where(x => x.tx.Mint() is { Count: > 0 }) + .Where(x => auxiliaryDataDict.TryGetValue(x.index, out AuxiliaryData? aux) + && aux.Metadata()?.Value().TryGetValue(721, out TransactionMetadatum? cip25) == true + && cip25 is MetadatumMap) + .SelectMany(x => { - string policyId = Convert.ToHexString(policyEntry.Key).ToLowerInvariant(); - - foreach (KeyValuePair assetEntry in policyEntry.Value.Value.Where(a => a.Value > 0)) + MetadatumMap rootMap = (MetadatumMap)auxiliaryDataDict[x.index].Metadata()!.Value()[721]; + return x.tx.Mint()!.SelectMany(policyEntry => { - string assetNameHex = Convert.ToHexString(assetEntry.Key).ToLowerInvariant(); - string subject = $"{policyId}{assetNameHex}"; - - TokenMetadataOnChain? tokenMetadata = ExtractMetadata(rootMap, policyId, assetNameHex, assetEntry.Value); - - if (tokenMetadata == null) - continue; - - if (string.IsNullOrEmpty(tokenMetadata.Name) || string.IsNullOrEmpty(tokenMetadata.Logo)) - { - logger.LogWarning("Skipping {Subject} - CIP-25 requires name and image fields", subject); - continue; - } - - tokensWithMetadata[subject] = tokenMetadata; - } - } - } + string policyId = Convert.ToHexString(policyEntry.Key).ToLowerInvariant(); + return policyEntry.Value.Value + .Where(a => a.Value > 0) + .Select(assetEntry => + { + string assetNameHex = Convert.ToHexString(assetEntry.Key).ToLowerInvariant(); + return ExtractMetadata(rootMap, policyId, assetNameHex, assetEntry.Value); + }); + }); + })]; + + // Log skipped tokens + allTokens + .Where(t => t is not null && (string.IsNullOrEmpty(t.Name) || string.IsNullOrEmpty(t.Logo))) + .ToList() + .ForEach(t => logger.LogWarning("Skipping {Subject} - CIP-25 requires name and image fields", t!.Subject)); + + // Filter valid tokens + Dictionary tokensWithMetadata = allTokens + .Where(t => t is not null && !string.IsNullOrEmpty(t.Name) && !string.IsNullOrEmpty(t.Logo)) + .GroupBy(t => t!.Subject) + .ToDictionary(g => g.Key, g => g.Last()!); if (tokensWithMetadata.Count == 0) return; await using MetadataDbContext db = await dbContextFactory.CreateDbContextAsync(); - List existingSubjects = [.. tokensWithMetadata.Keys]; Dictionary existingRecords = await db.TokenMetadataOnChain - .Where(t => existingSubjects.Contains(t.Subject)) + .Where(t => tokensWithMetadata.Keys.Contains(t.Subject)) .AsNoTracking() .ToDictionaryAsync(t => t.Subject); - foreach ((string? subject, TokenMetadataOnChain? tokenData) in tokensWithMetadata) - { - if (existingRecords.TryGetValue(subject, out TokenMetadataOnChain? existing)) - { - db.TokenMetadataOnChain.Update(tokenData); - logger.LogInformation("Updated {Subject} - quantity: {OldQty} -> {NewQty}", - subject, existing.Quantity, tokenData.Quantity); - } - else - { - db.TokenMetadataOnChain.Add(tokenData); - logger.LogInformation("Inserted {Subject} with quantity: {Quantity}", subject, tokenData.Quantity); - } - } + List toInsert = [.. tokensWithMetadata.Values.Where(t => !existingRecords.ContainsKey(t.Subject))]; + List toUpdate = [.. tokensWithMetadata.Values.Where(t => existingRecords.TryGetValue(t.Subject, out TokenMetadataOnChain? existing) && !TokensAreEqual(t, existing))]; + + db.TokenMetadataOnChain.AddRange(toInsert); + db.TokenMetadataOnChain.UpdateRange(toUpdate); + + toInsert.ForEach(t => logger.LogInformation("Inserted {Subject} with quantity: {Quantity}", t.Subject, t.Quantity)); + toUpdate.ForEach(t => logger.LogInformation("Updated {Subject} - quantity: {OldQty} -> {NewQty}", + t.Subject, existingRecords[t.Subject].Quantity, t.Quantity)); + await db.SaveChangesAsync(); } @@ -157,45 +143,36 @@ public async Task RollForwardAsync(Block block) if (assetKvp.Value is not MetadatumMap assetMap) return null; - TokenMetadataOnChain metadata = new( + Dictionary fields = assetMap.Value + .Where(f => f.Key is MetadataText) + .ToDictionary(f => ((MetadataText)f.Key).Value ?? string.Empty, f => f.Value); + + string name = fields.TryGetValue("name", out TransactionMetadatum? n) && n is MetadataText nt + ? nt.Value ?? string.Empty : string.Empty; + + string imageUri = fields.TryGetValue("image", out TransactionMetadatum? img) + ? img switch + { + MetadataText imageText => imageText.Value ?? string.Empty, + MetadatumList imageList => string.Join(string.Empty, imageList.Value.OfType().Select(t => t.Value)), + _ => string.Empty + } + : string.Empty; + + string description = fields.TryGetValue("description", out TransactionMetadatum? d) && d is MetadataText dt + ? dt.Value ?? string.Empty : string.Empty; + + return new TokenMetadataOnChain( Subject: $"{policyId}{assetNameHex}", PolicyId: policyId, AssetName: assetNameHex, - Name: "", - Logo: "", - Description: "", + Name: name, + Logo: IsValidUri(imageUri) ? imageUri : string.Empty, + Description: description, Quantity: quantity, Decimals: 0, TokenType: TokenType.CIP25 ); - - foreach (KeyValuePair field in assetMap.Value) - { - if (field.Key is MetadataText keyText) - { - switch (keyText.Value) - { - case "name" when field.Value is MetadataText nameText: - metadata = metadata with { Name = nameText.Value ?? "" }; - break; - case "image": - string imageUri = field.Value switch - { - MetadataText imageText => imageText.Value ?? "", - MetadatumList imageList => string.Join("", imageList.Value.OfType().Select(t => t.Value)), - _ => "" - }; - if (IsValidUri(imageUri)) - metadata = metadata with { Logo = imageUri }; - break; - case "description" when field.Value is MetadataText descText: - metadata = metadata with { Description = descText.Value ?? "" }; - break; - } - } - } - - return metadata; } private static bool IsValidUri(string uri) @@ -216,6 +193,13 @@ private static bool IsValidUri(string uri) return uri.Length > ValidSchemes.First(s => uri.StartsWith(s, StringComparison.OrdinalIgnoreCase)).Length; } + private static bool TokensAreEqual(TokenMetadataOnChain a, TokenMetadataOnChain b) => + a.Name == b.Name && + a.Logo == b.Logo && + a.Description == b.Description && + a.Quantity == b.Quantity && + a.Decimals == b.Decimals; + [GeneratedRegex(@"^data:image\/[a-zA-Z0-9]+(?:\+[a-zA-Z0-9]+)?;base64,", RegexOptions.Compiled)] private static partial Regex MyRegex(); } \ No newline at end of file diff --git a/src/COMP.Sync/Reducers/CIP68Reducer.cs b/src/COMP.Sync/Reducers/CIP68Reducer.cs index 3ccbd87..0c351de 100644 --- a/src/COMP.Sync/Reducers/CIP68Reducer.cs +++ b/src/COMP.Sync/Reducers/CIP68Reducer.cs @@ -30,10 +30,10 @@ ILogger logger { "RFT", "001bc280" } }; - public async Task RollBackwardAsync(ulong slot) - { - await Task.CompletedTask; - } + // Rollback is intentionally not implemented - this metadata indexer is append-only + // and doesn't track historical state. On-chain metadata updates are rare and + // rollbacks would require re-syncing from scratch. + public async Task RollBackwardAsync(ulong slot) => await Task.CompletedTask; public async Task RollForwardAsync(Block block) { @@ -52,244 +52,155 @@ public async Task RollForwardAsync(Block block) private static Dictionary ExtractAllReferenceTokens( List txBodies, List witnessSets, - ILogger logger) - { - Dictionary referenceTokens = []; - - for (int i = 0; i < txBodies.Count; i++) - { - List? outputs = txBodies[i].Outputs()?.ToList(); - if (outputs == null || outputs.Count == 0) - continue; - - TransactionWitnessSet? witnessSet = i < witnessSets.Count ? witnessSets[i] : null; - ExtractReferenceTokensFromOutputs(outputs, referenceTokens, witnessSet, logger); - } - - return referenceTokens; - } - - private static void ExtractReferenceTokensFromOutputs( - List outputs, - Dictionary referenceTokens, - TransactionWitnessSet? witnessSet, - ILogger logger) - { - foreach (TransactionOutput output in outputs) - { - Dictionary? multiAsset = output.Amount()?.MultiAsset(); - if (multiAsset == null) - continue; - - foreach (KeyValuePair policy in multiAsset) + ILogger logger) => + txBodies + .Select((tx, i) => (outputs: tx.Outputs()?.ToList(), witnessSet: i < witnessSets.Count ? witnessSets[i] : null)) + .Where(x => x.outputs is { Count: > 0 }) + .SelectMany(x => x.outputs!.SelectMany(output => { - string policyId = Convert.ToHexString(policy.Key).ToLowerInvariant(); - ProcessPolicyAssets(policy.Value.Value, policyId, output, referenceTokens, witnessSet, logger); - } - } - } - - private static void ProcessPolicyAssets( - Dictionary assets, - string policyId, - TransactionOutput output, - Dictionary referenceTokens, - TransactionWitnessSet? witnessSet, - ILogger logger) - { - foreach (KeyValuePair asset in assets) - { - if (asset.Value == 0) - continue; - - string assetNameHex = Convert.ToHexString(asset.Key).ToLowerInvariant(); - if (!assetNameHex.StartsWith(REFERENCE_PREFIX)) - continue; + Dictionary? multiAsset = output.Amount()?.MultiAsset(); + if (multiAsset is null) return []; - string baseName = assetNameHex[8..]; - string referenceSubject = $"{policyId}{assetNameHex}"; - - if (referenceTokens.ContainsKey(referenceSubject)) - continue; - - byte[]? datumBytes = ExtractDatum(output, witnessSet); - if (datumBytes == null) - { - logger.LogDebug("No datum found for reference token {Subject}", referenceSubject); - continue; - } - - referenceTokens[referenceSubject] = (policyId, baseName, datumBytes); - } - } + return multiAsset.SelectMany(policy => + { + string policyId = Convert.ToHexString(policy.Key).ToLowerInvariant(); + return policy.Value.Value + .Where(asset => asset.Value > 0) + .Select(asset => Convert.ToHexString(asset.Key).ToLowerInvariant()) + .Where(assetNameHex => assetNameHex.StartsWith(REFERENCE_PREFIX)) + .Select(assetNameHex => + { + byte[]? datumBytes = ExtractDatum(output, x.witnessSet); + if (datumBytes is null) + { + logger.LogDebug("No datum found for reference token {Subject}", $"{policyId}{assetNameHex}"); + return ((string policyId, string baseName, byte[] datumBytes)?)null; + } + return (policyId, baseName: assetNameHex[8..], datumBytes); + }) + .Where(r => r is not null) + .Select(r => r!.Value); + }); + })) + .GroupBy(r => $"{r.policyId}{REFERENCE_PREFIX}{r.baseName}") + .ToDictionary(g => g.Key, g => g.First()); private async Task> ProcessReferenceTokensAsync( Dictionary referenceTokens, List txBodies) { Dictionary mintedUserTokens = ExtractMintedUserTokens(txBodies); - Dictionary tokensToProcess = []; - foreach ((string _, (string? policyId, string? baseName, byte[]? datumBytes)) in referenceTokens) - { - string? userTokenSubject = await DetermineUserTokenSubjectAsync(baseName, policyId, mintedUserTokens); - if (userTokenSubject == null) - continue; + // Build all potential user token subjects for single batch DB query + List allPotentialSubjects = referenceTokens.Values + .SelectMany(r => UserTokenPrefixes.Values.Select(prefix => $"{r.policyId}{prefix}{r.baseName}")) + .ToList(); - long quantity = await GetTokenQuantityAsync(userTokenSubject, mintedUserTokens); - string assetName = userTokenSubject[policyId.Length..]; - (string? name, string? image, string? description, int? decimals) = ExtractCIP68Metadata(datumBytes); + // Single DB query to get all existing tokens + await using MetadataDbContext db = await dbContextFactory.CreateDbContextAsync(); + Dictionary existingTokens = await db.TokenMetadataOnChain + .Where(t => allPotentialSubjects.Contains(t.Subject)) + .AsNoTracking() + .ToDictionaryAsync(t => t.Subject); - if (string.IsNullOrEmpty(name) || string.IsNullOrEmpty(image)) + return referenceTokens.Values + .Select(r => { - logger.LogWarning("Skipping {Subject} - CIP-68 requires name and image fields", userTokenSubject); - continue; - } - - tokensToProcess[userTokenSubject] = new TokenMetadataOnChain( - Subject: userTokenSubject, - PolicyId: policyId, - AssetName: assetName, - Name: name, - Logo: image, - Description: description, - Quantity: quantity, - Decimals: decimals ?? 0, - TokenType: TokenType.CIP68 - ); - - logger.LogDebug("Processed CIP-68 token: {Subject} (qty: {Quantity})", userTokenSubject, quantity); - } + // Determine user token subject in prefix priority order + string? userTokenSubject = UserTokenPrefixes.Values + .Select(prefix => $"{r.policyId}{prefix}{r.baseName}") + .FirstOrDefault(s => mintedUserTokens.ContainsKey(s) || existingTokens.ContainsKey(s)); - return tokensToProcess; - } + if (userTokenSubject == null) + return null; - private static Dictionary ExtractMintedUserTokens(List txBodies) - { - Dictionary mintedUserTokens = []; + // Get quantity from minted or existing + long quantity = mintedUserTokens.GetValueOrDefault(userTokenSubject, 0); + if (quantity == 0) + quantity = existingTokens.GetValueOrDefault(userTokenSubject)?.Quantity ?? 0; - foreach (TransactionBody tx in txBodies) - { - Dictionary? mint = tx.Mint(); - if (mint == null) - continue; + (string name, string image, string description, int? decimals) = ExtractCIP68Metadata(r.datumBytes); - foreach (KeyValuePair policyEntry in mint) - { - string policyId = Convert.ToHexString(policyEntry.Key).ToLowerInvariant(); - foreach (KeyValuePair asset in policyEntry.Value.Value) + if (string.IsNullOrEmpty(name) || string.IsNullOrEmpty(image)) { - if (asset.Value <= 0) - continue; - - string assetNameHex = Convert.ToHexString(asset.Key).ToLowerInvariant(); - if (UserTokenPrefixes.Values.Any(assetNameHex.StartsWith)) - { - string subject = $"{policyId}{assetNameHex}"; - mintedUserTokens[subject] = asset.Value; - } + logger.LogWarning("Skipping {Subject} - CIP-68 requires name and image fields", userTokenSubject); + return null; } - } - } - return mintedUserTokens; + logger.LogDebug("Processed CIP-68 token: {Subject} (qty: {Quantity})", userTokenSubject, quantity); + + return new TokenMetadataOnChain( + Subject: userTokenSubject, + PolicyId: r.policyId, + AssetName: userTokenSubject[r.policyId.Length..], + Name: name, + Logo: image, + Description: description, + Quantity: quantity, + Decimals: decimals ?? 0, + TokenType: TokenType.CIP68 + ); + }) + .Where(t => t is not null) + .ToDictionary(t => t!.Subject, t => t!); } - private async Task GetTokenQuantityAsync(string userTokenSubject, Dictionary mintedUserTokens) - { - long quantity = mintedUserTokens.GetValueOrDefault(userTokenSubject, 0); - if (quantity > 0) - return quantity; - - await using MetadataDbContext db = await dbContextFactory.CreateDbContextAsync(); - TokenMetadataOnChain? existing = await db.TokenMetadataOnChain - .AsNoTracking() - .FirstOrDefaultAsync(t => t.Subject == userTokenSubject); - return existing?.Quantity ?? 0; - } - - private async Task DetermineUserTokenSubjectAsync( - string baseName, - string policyId, - Dictionary mintedUserTokens) + private static Dictionary ExtractMintedUserTokens(List txBodies) => + txBodies + .Select(tx => tx.Mint()) + .Where(mint => mint is not null) + .SelectMany(mint => mint!) + .SelectMany(policyEntry => + { + string policyId = Convert.ToHexString(policyEntry.Key).ToLowerInvariant(); + return policyEntry.Value.Value + .Where(asset => asset.Value > 0) + .Select(asset => ( + subject: $"{policyId}{Convert.ToHexString(asset.Key).ToLowerInvariant()}", + quantity: asset.Value + )) + .Where(x => UserTokenPrefixes.Values.Any(x.subject[(policyId.Length)..].StartsWith)); + }) + .GroupBy(x => x.subject) + .ToDictionary(g => g.Key, g => g.Sum(x => x.quantity)); + + private static (string name, string image, string description, int? decimals) ExtractCIP68Metadata(byte[] datumBytes) { - foreach ((string _, string? prefix) in UserTokenPrefixes) + try { - string subject = $"{policyId}{prefix}{baseName}"; - if (mintedUserTokens.ContainsKey(subject)) - return subject; - } + Cip68 datum = CborSerializer.Deserialize>(datumBytes); + if (datum?.Metadata is not PlutusMap map) + return (string.Empty, string.Empty, string.Empty, null); - await using MetadataDbContext db = await dbContextFactory.CreateDbContextAsync(); - foreach ((string _, string? prefix) in UserTokenPrefixes) - { - string subject = $"{policyId}{prefix}{baseName}"; - bool exists = await db.TokenMetadataOnChain.AnyAsync(t => t.Subject == subject); - if (exists) - return subject; - } + Dictionary fields = map.PlutusData + .Where(kvp => kvp.Key is PlutusBoundedBytes) + .ToDictionary( + kvp => System.Text.Encoding.UTF8.GetString(((PlutusBoundedBytes)kvp.Key).Value), + kvp => kvp.Value); - return null; - } + string name = fields.TryGetValue("name", out PlutusData? n) && n is PlutusBoundedBytes nb + ? System.Text.Encoding.UTF8.GetString(nb.Value) : string.Empty; - private (string name, string image, string description, int? decimals) ExtractCIP68Metadata(byte[] datumBytes) - { - try - { - Cip68 datum = CborSerializer.Deserialize>(datumBytes); - if (datum?.Metadata == null) - return ("", "", "", null); + string image = fields.TryGetValue("image", out PlutusData? i) && i is PlutusBoundedBytes ib + ? System.Text.Encoding.UTF8.GetString(ib.Value) : string.Empty; - string? name = null, image = null, description = null; - int? decimals = null; + string description = fields.TryGetValue("description", out PlutusData? d) && d is PlutusBoundedBytes db + ? System.Text.Encoding.UTF8.GetString(db.Value) : string.Empty; - if (datum.Metadata is PlutusMap map) - { - foreach (KeyValuePair kvp in map.PlutusData) + int? decimals = fields.TryGetValue("decimals", out PlutusData? dec) + ? dec switch { - if (kvp.Key is PlutusBoundedBytes keyBytes) - { - string key = System.Text.Encoding.UTF8.GetString(keyBytes.Value); - - switch (key) - { - case "name": - if (kvp.Value is PlutusBoundedBytes nameBytes) - { - name = System.Text.Encoding.UTF8.GetString(nameBytes.Value); - } - break; - case "image": - if (kvp.Value is PlutusBoundedBytes imageBytes) - { - image = System.Text.Encoding.UTF8.GetString(imageBytes.Value); - } - break; - case "description": - if (kvp.Value is PlutusBoundedBytes descBytes) - { - description = System.Text.Encoding.UTF8.GetString(descBytes.Value); - } - break; - case "decimals": - if (kvp.Value is PlutusInt64 decInt) - { - decimals = (int)decInt.Value; - } - else if (kvp.Value is PlutusUint64 decUint) - { - decimals = (int)decUint.Value; - } - break; - } - } + PlutusInt64 decInt => (int)decInt.Value, + PlutusUint64 decUint => (int)decUint.Value, + _ => null } - } - return (name ?? "", image ?? "", description ?? "", decimals); + : null; + + return (name, image, description, decimals); } catch { - return ("", "", "", null); + return (string.Empty, string.Empty, string.Empty, null); } } @@ -318,47 +229,42 @@ private async Task GetTokenQuantityAsync(string userTokenSubject, Dictiona return ResolveDatumFromWitnessSet(datumHash, plutusDataSet); } - private static byte[]? ResolveDatumFromWitnessSet(byte[] datumHash, IEnumerable plutusDataSet) - { - foreach (PlutusData plutusData in plutusDataSet) - { - byte[] datumBytes = plutusData.Raw(); - byte[] calculatedHash = HashUtil.Blake2b256(datumBytes); - - if (calculatedHash.SequenceEqual(datumHash)) - { - return datumBytes; - } - } - return null; - } + private static byte[]? ResolveDatumFromWitnessSet(byte[] datumHash, IEnumerable plutusDataSet) => + plutusDataSet + .Select(pd => pd.Raw()) + .FirstOrDefault(datumBytes => HashUtil.Blake2b256(datumBytes).SequenceEqual(datumHash)); private async Task SaveTokensAsync(Dictionary tokensToProcess) { await using MetadataDbContext db = await dbContextFactory.CreateDbContextAsync(); - List subjects = [.. tokensToProcess.Keys]; - Dictionary existingRecords = await db.TokenMetadataOnChain - .Where(t => subjects.Contains(t.Subject)) + Dictionary existingTokens = await db.TokenMetadataOnChain + .Where(t => tokensToProcess.Keys.Contains(t.Subject)) .AsNoTracking() .ToDictionaryAsync(t => t.Subject); - foreach ((string? subject, TokenMetadataOnChain? tokenData) in tokensToProcess) - { - if (existingRecords.ContainsKey(subject)) - { - db.TokenMetadataOnChain.Update(tokenData); - logger.LogInformation("Updated CIP-68 token {Subject} with quantity {Quantity}", - subject, tokenData.Quantity); - } - else - { - db.TokenMetadataOnChain.Add(tokenData); - logger.LogInformation("Inserted CIP-68 token {Subject} with quantity {Quantity}", - subject, tokenData.Quantity); - } - } + List toInsert = tokensToProcess.Values + .Where(t => !existingTokens.ContainsKey(t.Subject)) + .ToList(); + + List toUpdate = tokensToProcess.Values + .Where(t => existingTokens.TryGetValue(t.Subject, out TokenMetadataOnChain? existing) && !TokensAreEqual(t, existing)) + .ToList(); + + db.TokenMetadataOnChain.AddRange(toInsert); + db.TokenMetadataOnChain.UpdateRange(toUpdate); + + toInsert.ForEach(t => logger.LogInformation("Inserted CIP-68 token {Subject} with quantity {Quantity}", t.Subject, t.Quantity)); + toUpdate.ForEach(t => logger.LogInformation("Updated CIP-68 token {Subject} - quantity: {OldQty} -> {NewQty}", + t.Subject, existingTokens[t.Subject].Quantity, t.Quantity)); await db.SaveChangesAsync(); } -} \ No newline at end of file + + private static bool TokensAreEqual(TokenMetadataOnChain a, TokenMetadataOnChain b) => + a.Name == b.Name && + a.Logo == b.Logo && + a.Description == b.Description && + a.Quantity == b.Quantity && + a.Decimals == b.Decimals; +} diff --git a/src/COMP.Sync/Reducers/GithubReducer.cs b/src/COMP.Sync/Reducers/GithubReducer.cs index c6c82d1..441cfcd 100644 --- a/src/COMP.Sync/Reducers/GithubReducer.cs +++ b/src/COMP.Sync/Reducers/GithubReducer.cs @@ -19,7 +19,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { logger.LogInformation("Starting metadata synchronization cycle"); - SyncState? syncState = await metadataDbService.GetSyncStateAsync(stoppingToken); + RegistrySyncState? syncState = await metadataDbService.GetRegistrySyncStateAsync(stoppingToken); if (syncState is null) { @@ -38,28 +38,44 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) logger.LogError("Tree response is null"); break; } - foreach (GitTreeItem item in treeResponse.Tree) - { - if (item.Path?.StartsWith("mappings/") == true && item.Path.EndsWith(".json")) - { - string subject = ExtractSubjectFromPath(item.Path); + // Filter mapping files and extract subjects in single pass + List<(GitTreeItem item, string subject)> mappingFiles = [.. treeResponse.Tree + .Where(item => item.Path?.StartsWith("mappings/") == true && item.Path.EndsWith(".json")) + .Select(item => (item, subject: ExtractSubjectFromPath(item.Path)))]; - bool exist = await metadataDbService.SubjectExistsAsync(subject, stoppingToken); - if (exist) continue; + // Batch check existing subjects to avoid N+1 queries + HashSet existingSubjects = await metadataDbService.GetExistingSubjectsAsync( + mappingFiles.Select(m => m.subject), stoppingToken); - MetadataResponse? mapping = await githubService.GetMappingJsonAsync(latestCommit.Sha, item.Path, stoppingToken); - TokenMetadata? token = MapTokenMetadata(mapping); + foreach ((GitTreeItem item, string subject) in mappingFiles) + { + if (existingSubjects.Contains(subject)) continue; + try + { + MetadataResponse? mapping = await githubService.GetMappingJsonAsync(latestCommit.Sha, item.Path!, stoppingToken); + TokenMetadataRegistry? token = MapTokenMetadataRegistry(mapping); if (token == null) continue; await metadataDbService.AddTokenAsync(token, stoppingToken); } + catch (HttpRequestException httpEx) + { + logger.LogError(httpEx, "Network error while fetching metadata for subject {Subject}. Skipping this update.", subject); + } + catch (JsonException jsonEx) + { + logger.LogError(jsonEx, "JSON parsing error for subject {Subject}. File may be malformed. Skipping this update.", subject); + } + catch (Exception ex) + { + logger.LogError(ex, "Unexpected error processing metadata for subject {Subject}. Skipping this update.", subject); + } } - await metadataDbService.UpsertSyncStateAsync(latestCommit, stoppingToken); + await metadataDbService.UpsertRegistrySyncStateAsync(latestCommit, stoppingToken); } else { List latestCommitsSince = await GetLatestCommitsSinceAsync(syncState.Date, stoppingToken); - foreach (GitCommit commit in latestCommitsSince) { @@ -68,52 +84,61 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) GitCommit? resolvedCommit = await githubService.GetMappingJsonAsync(commit.Url, cancellationToken: stoppingToken); if (resolvedCommit is null || string.IsNullOrEmpty(resolvedCommit.Sha) || resolvedCommit.Files is null) continue; - foreach (GitCommitFile file in resolvedCommit.Files) + // Filter and extract subjects for changed mapping files in single pass + List<(GitCommitFile file, string subject)> filesToProcess = [.. resolvedCommit.Files + .Where(f => f.Filename?.StartsWith("mappings/") == true + && f.Filename.EndsWith(".json", StringComparison.OrdinalIgnoreCase) + && !string.Equals(f.Status, "removed", StringComparison.OrdinalIgnoreCase)) + .Select(f => (file: f, subject: ExtractSubjectFromPath(f.Filename)))]; + + // Log removed mapping files (deletes are intentionally not applied locally) + foreach (GitCommitFile removedFile in resolvedCommit.Files.Where(f => + f.Filename?.StartsWith("mappings/") == true + && f.Filename.EndsWith(".json", StringComparison.OrdinalIgnoreCase) + && string.Equals(f.Status, "removed", StringComparison.OrdinalIgnoreCase))) { - if (file.Filename is not null) + logger.LogInformation("Skipping removed file {Filename} in commit {Sha}", removedFile.Filename, resolvedCommit.Sha); + } + + // Batch check existing subjects + HashSet existingSubjects = await metadataDbService.GetExistingSubjectsAsync( + filesToProcess.Select(f => f.subject), stoppingToken); + + foreach ((GitCommitFile file, string subject) in filesToProcess) + { + try { - if (string.Equals(file.Status, "removed", StringComparison.OrdinalIgnoreCase)) + MetadataResponse? mapping = await githubService.GetMappingJsonAsync(resolvedCommit.Sha, file.Filename!, stoppingToken); + TokenMetadataRegistry? token = MapTokenMetadataRegistry(mapping); + if (token is null) { - logger.LogInformation("Skipping removed file {Filename} in commit {Sha}", file.Filename, resolvedCommit.Sha); + logger.LogWarning("Failed to map registry item for subject {Subject} in commit {Sha}", subject, resolvedCommit.Sha); continue; } - string subject = ExtractSubjectFromPath(file.Filename); - try + if (existingSubjects.Contains(subject)) { - MetadataResponse? mapping = await githubService.GetMappingJsonAsync(resolvedCommit.Sha, file.Filename, stoppingToken); - TokenMetadata? token = MapTokenMetadata(mapping); - if (token is null) - { - logger.LogWarning("Failed to map registry item for subject {Subject} in commit {Sha}", subject, resolvedCommit.Sha); - continue; - } - - bool exists = await metadataDbService.SubjectExistsAsync(subject, stoppingToken); - if (exists) - { - await metadataDbService.UpdateTokenAsync(token, stoppingToken); - } - else - { - await metadataDbService.AddTokenAsync(token, stoppingToken); - } + await metadataDbService.UpdateTokenAsync(token, stoppingToken); } - catch (HttpRequestException httpEx) + else { - logger.LogError(httpEx, "Network error while fetching metadata for subject {Subject}. Skipping this update.", subject); - } - catch (JsonException jsonEx) - { - logger.LogError(jsonEx, "JSON parsing error for subject {Subject}. File may be malformed. Skipping this update.", subject); - } - catch (Exception ex) - { - logger.LogError(ex, "Unexpected error processing metadata for subject {Subject}. Skipping this update.", subject); + await metadataDbService.AddTokenAsync(token, stoppingToken); } } + catch (HttpRequestException httpEx) + { + logger.LogError(httpEx, "Network error while fetching metadata for subject {Subject}. Skipping this update.", subject); + } + catch (JsonException jsonEx) + { + logger.LogError(jsonEx, "JSON parsing error for subject {Subject}. File may be malformed. Skipping this update.", subject); + } + catch (Exception ex) + { + logger.LogError(ex, "Unexpected error processing metadata for subject {Subject}. Skipping this update.", subject); + } } - await metadataDbService.UpsertSyncStateAsync(resolvedCommit, stoppingToken); + await metadataDbService.UpsertRegistrySyncStateAsync(resolvedCommit, stoppingToken); } } @@ -127,7 +152,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } - public TokenMetadata? MapTokenMetadata(MetadataResponse? resp) + public TokenMetadataRegistry? MapTokenMetadataRegistry(MetadataResponse? resp) { if (resp is null) { @@ -158,7 +183,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) return null; } - return new TokenMetadata( + return new TokenMetadataRegistry( Subject: subject, Name: name, Ticker: ticker, @@ -190,8 +215,17 @@ private static string ExtractSubjectFromPath(string? path) { if (string.IsNullOrEmpty(path)) return string.Empty; - - return path.Replace("mappings/", string.Empty) - .Replace(".json", string.Empty); + + const string mappingsPrefix = "mappings/"; + if (!path.StartsWith(mappingsPrefix, StringComparison.Ordinal)) + return string.Empty; + + if (!path.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + return string.Empty; + + string relativePath = path[mappingsPrefix.Length..]; + return relativePath.Length <= ".json".Length + ? string.Empty + : relativePath[..^".json".Length]; } } diff --git a/src/COMP.Sync/Services/MetadataDbService.cs b/src/COMP.Sync/Services/MetadataDbService.cs index 855dace..f760420 100644 --- a/src/COMP.Sync/Services/MetadataDbService.cs +++ b/src/COMP.Sync/Services/MetadataDbService.cs @@ -10,7 +10,7 @@ public class MetadataDbService ILogger logger, IDbContextFactory _dbContextFactory) { - public async Task AddTokenAsync(TokenMetadata token, CancellationToken cancellationToken) + public async Task AddTokenAsync(TokenMetadataRegistry token, CancellationToken cancellationToken) { using MetadataDbContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); @@ -23,20 +23,20 @@ public class MetadataDbService return null; } - await dbContext.TokenMetadata.AddAsync(token, cancellationToken); + await dbContext.TokenMetadataRegistry.AddAsync(token, cancellationToken); await dbContext.SaveChangesAsync(cancellationToken); return token; } - public async Task GetSyncStateAsync(CancellationToken cancellationToken) + public async Task GetRegistrySyncStateAsync(CancellationToken cancellationToken) { using MetadataDbContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - return await dbContext.SyncState + return await dbContext.RegistrySyncState .OrderByDescending(ss => ss.Date) .FirstOrDefaultAsync(cancellationToken); } - public async Task UpsertSyncStateAsync(GitCommit latestCommit, CancellationToken cancellationToken) + public async Task UpsertRegistrySyncStateAsync(GitCommit latestCommit, CancellationToken cancellationToken) { using MetadataDbContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); @@ -44,61 +44,39 @@ public async Task UpsertSyncStateAsync(GitCommit latestCommit, CancellationToken DateTimeOffset newDate = latestCommit.Commit?.Author?.Date ?? DateTimeOffset.UtcNow; // Delete existing sync state if it exists - SyncState? existingSyncState = await dbContext.SyncState + RegistrySyncState? existingRegistrySyncState = await dbContext.RegistrySyncState .FirstOrDefaultAsync(cancellationToken); - if (existingSyncState is not null) + if (existingRegistrySyncState is not null) { - string oldHash = existingSyncState.Hash; - dbContext.SyncState.Remove(existingSyncState); + string oldHash = existingRegistrySyncState.Hash; + dbContext.RegistrySyncState.Remove(existingRegistrySyncState); logger.LogInformation("Removed old sync state with hash: {OldHash}", oldHash); } // Insert new sync state - SyncState syncState = new( + RegistrySyncState syncState = new( Hash: newSha, Date: newDate ); - await dbContext.SyncState.AddAsync(syncState, cancellationToken); + await dbContext.RegistrySyncState.AddAsync(syncState, cancellationToken); await dbContext.SaveChangesAsync(cancellationToken); logger.LogInformation("Sync state created with hash: {Hash}", newSha); } - public async Task SubjectExistsAsync(string subject, CancellationToken cancellationToken) + public async Task> GetExistingSubjectsAsync(IEnumerable subjects, CancellationToken cancellationToken) { using MetadataDbContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - return await dbContext.TokenMetadata - .AnyAsync(t => t.Subject == subject, cancellationToken); + List subjectList = subjects.ToList(); + List existing = await dbContext.TokenMetadataRegistry + .Where(t => subjectList.Contains(t.Subject)) + .Select(t => t.Subject) + .ToListAsync(cancellationToken); + return [.. existing]; } - public async Task DeleteTokenAsync(string subject, CancellationToken cancellationToken) - { - using MetadataDbContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - - TokenMetadata? existingMetadata = await dbContext.TokenMetadata - .Where(tm => tm.Subject == subject) - .FirstOrDefaultAsync(cancellationToken); - if (existingMetadata != null) - { - dbContext.TokenMetadata.Remove(existingMetadata); - await dbContext.SaveChangesAsync(cancellationToken); - } - } - - public async Task UpdateTokenAsync(TokenMetadata updated, CancellationToken cancellationToken) + public async Task UpdateTokenAsync(TokenMetadataRegistry updated, CancellationToken cancellationToken) { - using MetadataDbContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - - TokenMetadata? existingMetadata = await dbContext.TokenMetadata - .AsNoTracking() - .FirstOrDefaultAsync(t => t.Subject == updated.Subject, cancellationToken); - - if (existingMetadata is null) - { - logger.LogWarning("Token metadata not found for subject {Subject}", updated.Subject); - return null; - } - if (string.IsNullOrEmpty(updated.Subject) || string.IsNullOrEmpty(updated.Name) || string.IsNullOrEmpty(updated.Description) || @@ -108,9 +86,10 @@ public async Task DeleteTokenAsync(string subject, CancellationToken cancellatio return null; } - // Preserve immutable fields from existing if needed; here we accept the provided updated entity - dbContext.TokenMetadata.Update(updated); + using MetadataDbContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); + dbContext.TokenMetadataRegistry.Update(updated); await dbContext.SaveChangesAsync(cancellationToken); return updated; } + } diff --git a/src/COMP.Sync/appsettings.example.json b/src/COMP.Sync/appsettings.example.json new file mode 100644 index 0000000..0589304 --- /dev/null +++ b/src/COMP.Sync/appsettings.example.json @@ -0,0 +1,42 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning", + "Microsoft.EntityFrameworkCore": "Warning", + "Comp.Sync.Reducers": "Debug" + } + }, + "AllowedHosts": "*", + "ConnectionStrings": { + "CardanoContext": "Host=localhost;Port=5432;Database=cardano_metadata;Username=postgres;Password=postgres", + "CardanoContextSchema": "public" + }, + "CardanoNodeConnection": { + "ConnectionType": "UnixSocket", + "UnixSocket": { + "Path": "/path/to/cardano/node.socket" + }, + "NetworkMagic": 764824073, + "MaxRollbackSlots": 1000, + "RollbackBuffer": 10, + "Slot": 0, + "Hash": "" + }, + "Sync": { + "Dashboard": { + "TuiMode": false, + "RefreshInterval": 5000, + "DisplayType": "sync" + } + }, + "GithubPAT": "YOUR_GITHUB_PAT", + "RegistryOwner": "cardano-foundation", + "RegistryRepo": "cardano-token-registry", + "Github": { + "UserAgent": { + "ProductName": "CardanoMetadataService", + "ProductUrl": "(+https://github.com/SAIB-Inc/COMP)" + } + } +}