diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71520dd..386ac06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: if: github.event_name == 'pull_request' - uses: actions/setup-dotnet@v3 with: - dotnet-version: 7.0.305 + dotnet-version: 9.0.102 - uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e20041a..d43ff9c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v3 with: - dotnet-version: 7.0.305 + dotnet-version: 9.0.102 - name: Checkout source uses: actions/checkout@v3 - name: Prepare publish artifacts diff --git a/CentCom.API/CentCom.API.csproj b/CentCom.API/CentCom.API.csproj index dd2db13..093182b 100644 --- a/CentCom.API/CentCom.API.csproj +++ b/CentCom.API/CentCom.API.csproj @@ -7,17 +7,17 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - + + + + + diff --git a/CentCom.API/Startup.cs b/CentCom.API/Startup.cs index d05b3c1..092c681 100644 --- a/CentCom.API/Startup.cs +++ b/CentCom.API/Startup.cs @@ -15,14 +15,9 @@ namespace CentCom.API; -public class Startup +public class Startup(IConfiguration configuration) { - public Startup(IConfiguration configuration) - { - Configuration = configuration; - } - - public IConfiguration Configuration { get; } + public IConfiguration Configuration { get; } = configuration; // This method gets called by the runtime. Use this method to add services to the container. public void ConfigureServices(IServiceCollection services) diff --git a/CentCom.Common/CentCom.Common.csproj b/CentCom.Common/CentCom.Common.csproj index d5c8f3f..606643d 100644 --- a/CentCom.Common/CentCom.Common.csproj +++ b/CentCom.Common/CentCom.Common.csproj @@ -1,18 +1,18 @@  - - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - + + + + diff --git a/CentCom.Exporter/CentCom.Exporter.csproj b/CentCom.Exporter/CentCom.Exporter.csproj index 4aa648d..ad2358a 100644 --- a/CentCom.Exporter/CentCom.Exporter.csproj +++ b/CentCom.Exporter/CentCom.Exporter.csproj @@ -5,10 +5,10 @@ - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/CentCom.Server/BanSources/BanParser.cs b/CentCom.Server/BanSources/BanParser.cs index ef45882..3e18c95 100644 --- a/CentCom.Server/BanSources/BanParser.cs +++ b/CentCom.Server/BanSources/BanParser.cs @@ -125,7 +125,7 @@ private async Task ParseBans(IJobExecutionContext context) }; // Get stored bans from the database - List storedBans = null; + List storedBans; try { storedBans = await DbContext.Bans @@ -143,7 +143,7 @@ private async Task ParseBans(IJobExecutionContext context) // Get bans from the source var isCompleteRefresh = context.MergedJobDataMap.GetBoolean("completeRefresh") || !storedBans.Any(); history.CompleteRefresh = isCompleteRefresh; - IEnumerable bans = null; + List bans; try { bans = await (isCompleteRefresh ? FetchAllBansAsync() : FetchNewBansAsync()); @@ -178,24 +178,25 @@ private async Task ParseBans(IJobExecutionContext context) } // Remove and report any invalid data from the parsed data - var dirtyBans = bans.Where(x => x.CKey == null || (SourceSupportsBanIDs && x.BanID == null)); - if (dirtyBans.Any()) + var dirtyBans = bans.Where(x => x.CKey == null || (SourceSupportsBanIDs && x.BanID == null)).ToList(); + if (dirtyBans.Count != 0) { - bans = bans.Except(dirtyBans); - history.Erroneous = dirtyBans.Count(); + bans = bans.Except(dirtyBans).ToList(); + history.Erroneous = dirtyBans.Count; Logger.LogWarning( - $"Removed {history.Erroneous} erroneous bans from parsed data. This shouldn't happen!"); + "Removed {Erroneous} erroneous bans from parsed data. This shouldn't happen!", history.Erroneous); } // Remove erronenous duplicates from source var sourceDupes = bans.GroupBy(x => x, BanEqualityComparer.Instance) .Where(x => x.Count() > 1) - .SelectMany(x => x.OrderBy(y => y.Id).Skip(1)); - if (sourceDupes.Any()) + .SelectMany(x => x.OrderBy(y => y.Id).Skip(1)) + .ToList(); + if (sourceDupes.Count != 0) { Logger.LogWarning( - $"Removing {sourceDupes.Count()} duplicated bans from source, this indicates an issue with the source data!"); - bans = bans.Except(sourceDupes); + "Removing {SourceDupes} duplicated bans from source, this indicates an issue with the source data!", sourceDupes.Count); + bans = bans.Except(sourceDupes).ToList(); } // Check for ban updates @@ -207,7 +208,7 @@ private async Task ParseBans(IJobExecutionContext context) b.MakeKeysCanonical(); // Attempt to find matching bans in the database - Ban matchedBan = null; + Ban matchedBan; if (SourceSupportsBanIDs) { matchedBan = storedBans.FirstOrDefault(x => @@ -274,7 +275,7 @@ private async Task ParseBans(IJobExecutionContext context) storedBans.AddRange(toInsert); } - Logger.LogInformation($"Inserting {toInsert.Count} new bans, updating {updated} modified bans..."); + Logger.LogInformation("Inserting {NewBans} new bans, updating {Updated} modified bans...", toInsert.Count, updated); history.Added = toInsert.Count; history.Updated = updated; await DbContext.SaveChangesAsync(); @@ -282,7 +283,7 @@ private async Task ParseBans(IJobExecutionContext context) // No need to continue unless this is a complete refresh if (!isCompleteRefresh) { - Logger.LogInformation("Completed ban parsing. Partial refresh complete."); + Logger.LogInformation("Completed ban parsing. Partial refresh complete"); history.CompletedUpload = DateTimeOffset.UtcNow; return history; } @@ -298,9 +299,11 @@ private async Task ParseBans(IJobExecutionContext context) } // Apply deletions - Logger.LogInformation(missingBans.Count > 0 - ? $"Removing {missingBans.Count} deleted bans..." - : "Found no deleted bans to remove"); + if (missingBans.Count > 0) + Logger.LogInformation("Removing {MissingBans} deleted bans...", missingBans.Count); + else + Logger.LogInformation("Found no deleted bans to remove"); + history.Deleted = missingBans.Count; if (missingBans.Count > 0) { @@ -316,16 +319,17 @@ private async Task ParseBans(IJobExecutionContext context) // Delete any accidental duplications var duplicates = storedBans.GroupBy(x => x, BanEqualityComparer.Instance) .Where(x => x.Count() > 1) - .SelectMany(x => x.OrderBy(x => x.Id).Skip(1)); - if (duplicates.Any()) + .SelectMany(x => x.OrderBy(y => y.Id).Skip(1)) + .ToList(); + if (duplicates.Count != 0) { - Logger.LogWarning($"Removing {duplicates.Count()} duplicated bans from the database"); + Logger.LogWarning("Removing {Duplicates} duplicated bans from the database", duplicates.Count); DbContext.RemoveRange(duplicates); await DbContext.SaveChangesAsync(); } history.CompletedUpload = DateTimeOffset.UtcNow; - Logger.LogInformation("Completed ban parsing. Complete refresh complete."); + Logger.LogInformation("Completed ban parsing. Complete refresh complete"); return history; } @@ -367,7 +371,7 @@ public async Task> GetSourcesAsync() /// /// A collection of bans to have their source objects assigned /// A collection of bans which have correct database-backed BanSource objects assigned - public async Task> AssignBanSources(IEnumerable bans) + public async Task> AssignBanSources(List bans) { var sources = await GetSourcesAsync(); foreach (var b in bans) @@ -387,11 +391,11 @@ public async Task> AssignBanSources(IEnumerable bans) /// just to limit the response size /// /// A collection of bans found from the source - public abstract Task> FetchNewBansAsync(); + public abstract Task> FetchNewBansAsync(); /// /// Attempts to fetch all bans from the ban source /// /// A collection of bans found from the source - public abstract Task> FetchAllBansAsync(); + public abstract Task> FetchAllBansAsync(); } \ No newline at end of file diff --git a/CentCom.Server/BanSources/BeeBanParser.cs b/CentCom.Server/BanSources/BeeBanParser.cs index 0637739..cc2b0fa 100644 --- a/CentCom.Server/BanSources/BeeBanParser.cs +++ b/CentCom.Server/BanSources/BeeBanParser.cs @@ -19,7 +19,7 @@ public BeeBanParser(DatabaseContext dbContext, BeeBanService banService, ILogger _banService = banService; } - protected override Dictionary Sources => new Dictionary + protected override Dictionary Sources => new() { { "bee-lrp", new BanSource { @@ -38,7 +38,7 @@ public BeeBanParser(DatabaseContext dbContext, BeeBanService banService, ILogger protected override bool SourceSupportsBanIDs => true; protected override string Name => "Beestation"; - public override async Task> FetchNewBansAsync() + public override async Task> FetchNewBansAsync() { Logger.LogInformation("Getting new bans for Beestation..."); var recent = await DbContext.Bans @@ -65,7 +65,7 @@ public override async Task> FetchNewBansAsync() return foundBans; } - public override async Task> FetchAllBansAsync() + public override async Task> FetchAllBansAsync() { Logger.LogInformation("Getting all bans for Beestation..."); return await _banService.GetBansBatchedAsync(); diff --git a/CentCom.Server/BanSources/FulpBanParser.cs b/CentCom.Server/BanSources/FulpBanParser.cs index 76719f9..afa7a3e 100644 --- a/CentCom.Server/BanSources/FulpBanParser.cs +++ b/CentCom.Server/BanSources/FulpBanParser.cs @@ -11,7 +11,7 @@ namespace CentCom.Server.BanSources; public class FulpBanParser : BanParser { - private const int PAGES_PER_BATCH = 12; + private const int PagesPerBatch = 12; private readonly FulpBanService _banService; public FulpBanParser(DatabaseContext dbContext, FulpBanService banService, ILogger logger) : base(dbContext, logger) @@ -20,7 +20,7 @@ public FulpBanParser(DatabaseContext dbContext, FulpBanService banService, ILogg Logger = logger; } - protected override Dictionary Sources => new Dictionary + protected override Dictionary Sources => new() { { "fulp", new BanSource { @@ -33,13 +33,13 @@ public FulpBanParser(DatabaseContext dbContext, FulpBanService banService, ILogg protected override bool SourceSupportsBanIDs => false; protected override string Name => "Fulpstation"; - public override async Task> FetchAllBansAsync() + public override async Task> FetchAllBansAsync() { Logger.LogInformation("Getting all bans for Fulpstation..."); return await _banService.GetBansBatchedAsync(); } - public override async Task> FetchNewBansAsync() + public override async Task> FetchNewBansAsync() { Logger.LogInformation("Getting new bans for Fulpstation..."); var recent = await DbContext.Bans @@ -54,13 +54,13 @@ public override async Task> FetchNewBansAsync() while (true) { - var batch = await _banService.GetBansBatchedAsync(page, PAGES_PER_BATCH); + var batch = await _banService.GetBansBatchedAsync(page, PagesPerBatch); foundBans.AddRange(batch); - if (!batch.Any() || batch.Any(x => recent.Any(y => y.BannedOn == x.BannedOn && y.CKey == y.CKey))) + if (batch.Count == 0 || batch.Any(x => recent.Any(y => y.BannedOn == x.BannedOn && y.CKey == x.CKey))) { break; } - page += PAGES_PER_BATCH; + page += PagesPerBatch; } return foundBans; diff --git a/CentCom.Server/BanSources/StandardBanParser.cs b/CentCom.Server/BanSources/StandardBanParser.cs index 8a86d84..2fc81cc 100644 --- a/CentCom.Server/BanSources/StandardBanParser.cs +++ b/CentCom.Server/BanSources/StandardBanParser.cs @@ -13,23 +13,18 @@ namespace CentCom.Server.BanSources; -public class StandardBanParser : BanParser +public class StandardBanParser( + DatabaseContext dbContext, + ILogger logger, + StandardProviderService banService, + IConfiguration config) + : BanParser(dbContext, logger) { - private readonly StandardProviderService _banService; - private readonly List _providerConfigs; + private readonly List _providerConfigs = config.GetSection("standardSources").Get>(); private string _name; private Dictionary _sources; - public StandardBanParser(DatabaseContext dbContext, ILogger logger, - StandardProviderService banService, - IConfiguration config) : - base(dbContext, logger) - { - _banService = banService; - _providerConfigs = config.GetSection("standardSources").Get>(); - } - protected override Dictionary Sources => _sources; protected override bool SourceSupportsBanIDs => true; @@ -45,18 +40,18 @@ protected override Task Configure(IJobExecutionContext context) // Configure the ban service for this source _name = source.Display; - _banService.Configure(source); + banService.Configure(source); // Ensure source is set _sources = new Dictionary { - { _banService.Source.Name, _banService.Source } + { banService.Source.Name, banService.Source } }; return Task.CompletedTask; } - public override async Task> FetchNewBansAsync() + public override async Task> FetchNewBansAsync() { Logger.LogInformation("Fetching new bans for {Name}...", Name); var recent = await DbContext.Bans @@ -66,12 +61,12 @@ public override async Task> FetchNewBansAsync() .Include(x => x.JobBans) .Include(x => x.SourceNavigation) .ToListAsync(); - return await _banService.GetBansBatchedAsync(searchFor: recent.Select(x => int.Parse(x.BanID))); + return await banService.GetBansBatchedAsync(searchFor: recent.Select(x => int.Parse(x.BanID)).ToList()); } - public override async Task> FetchAllBansAsync() + public override async Task> FetchAllBansAsync() { Logger.LogInformation("Fetching all bans for {Name}...", Name); - return await _banService.GetBansBatchedAsync(); + return await banService.GetBansBatchedAsync(); } } \ No newline at end of file diff --git a/CentCom.Server/BanSources/TGMCBanParser.cs b/CentCom.Server/BanSources/TGMCBanParser.cs index 4138848..83c15b0 100644 --- a/CentCom.Server/BanSources/TGMCBanParser.cs +++ b/CentCom.Server/BanSources/TGMCBanParser.cs @@ -9,17 +9,12 @@ namespace CentCom.Server.BanSources; -public class TGMCBanParser : BanParser +public class TGMCBanParser(DatabaseContext dbContext, TGMCBanService banService, ILogger logger) + : BanParser(dbContext, logger) { - private const int PAGES_PER_BATCH = 3; - private readonly TGMCBanService _banService; + private const int PagesPerBatch = 3; - public TGMCBanParser(DatabaseContext dbContext, TGMCBanService banService, ILogger logger) : base(dbContext, logger) - { - _banService = banService; - } - - protected override Dictionary Sources => new Dictionary + protected override Dictionary Sources => new() { { "tgmc", new BanSource { @@ -32,13 +27,13 @@ public TGMCBanParser(DatabaseContext dbContext, TGMCBanService banService, ILogg protected override bool SourceSupportsBanIDs => true; protected override string Name => "TGMC"; - public override async Task> FetchAllBansAsync() + public override async Task> FetchAllBansAsync() { Logger.LogInformation("Getting all bans for TGMC..."); - return await _banService.GetBansBatchedAsync(); + return await banService.GetBansBatchedAsync(); } - public override async Task> FetchNewBansAsync() + public override async Task> FetchNewBansAsync() { Logger.LogInformation("Getting new bans for TGMC..."); var recent = await DbContext.Bans @@ -53,13 +48,13 @@ public override async Task> FetchNewBansAsync() while (true) { - var batch = await _banService.GetBansBatchedAsync(page, PAGES_PER_BATCH); + var batch = await banService.GetBansBatchedAsync(page, PagesPerBatch); foundBans.AddRange(batch); - if (!batch.Any() || batch.Any(x => recent.Any(y => y.BannedOn == x.BannedOn && y.CKey == y.CKey))) + if (batch.Count == 0 || batch.Any(x => recent.Any(y => y.BannedOn == x.BannedOn && y.CKey == x.CKey))) { break; } - page += PAGES_PER_BATCH; + page += PagesPerBatch; } return foundBans; diff --git a/CentCom.Server/BanSources/TgBanParser.cs b/CentCom.Server/BanSources/TgBanParser.cs index c7bab4c..34ee709 100644 --- a/CentCom.Server/BanSources/TgBanParser.cs +++ b/CentCom.Server/BanSources/TgBanParser.cs @@ -9,16 +9,10 @@ namespace CentCom.Server.BanSources; -public class TgBanParser : BanParser +public class TgBanParser(DatabaseContext dbContext, TgBanService banService, ILogger logger) + : BanParser(dbContext, logger) { - private readonly TgBanService _banService; - - public TgBanParser(DatabaseContext dbContext, TgBanService banService, ILogger logger) : base(dbContext, logger) - { - _banService = banService; - } - - protected override Dictionary Sources => new Dictionary + protected override Dictionary Sources => new() { { "tgstation", new BanSource { @@ -31,13 +25,13 @@ public TgBanParser(DatabaseContext dbContext, TgBanService banService, ILogger true; protected override string Name => "/tg/station"; - public override async Task> FetchAllBansAsync() + public override async Task> FetchAllBansAsync() { Logger.LogInformation("Fetching all bans for /tg/station..."); - return await _banService.GetBansBatchedAsync(); + return await banService.GetBansBatchedAsync(Sources["tgstation"]); } - public override async Task> FetchNewBansAsync() + public override async Task> FetchNewBansAsync() { Logger.LogInformation("Fetching new bans for /tg/station..."); var recent = await DbContext.Bans @@ -52,7 +46,7 @@ public override async Task> FetchNewBansAsync() var page = 1; while (true) { - var bans = await _banService.GetBansAsync(page); + var bans = await banService.GetBansAsync(page); if (bans.Count == 0) break; @@ -65,6 +59,6 @@ public override async Task> FetchNewBansAsync() page++; } - return foundBans.DistinctBy(x => x.BanID); + return foundBans.DistinctBy(x => x.BanID).ToList(); } } \ No newline at end of file diff --git a/CentCom.Server/BanSources/VgBanParser.cs b/CentCom.Server/BanSources/VgBanParser.cs index 975bc32..31aa1bc 100644 --- a/CentCom.Server/BanSources/VgBanParser.cs +++ b/CentCom.Server/BanSources/VgBanParser.cs @@ -16,7 +16,7 @@ public VgBanParser(DatabaseContext dbContext, VgBanService banService, ILogger Sources => new Dictionary + protected override Dictionary Sources => new() { { "vgstation", new BanSource { @@ -29,13 +29,13 @@ public VgBanParser(DatabaseContext dbContext, VgBanService banService, ILogger false; protected override string Name => "/vg/station"; - public override async Task> FetchAllBansAsync() + public override async Task> FetchAllBansAsync() { Logger.LogInformation("Fetching all bans for /vg/station..."); return await _banService.GetBansAsync(); } - public override async Task> FetchNewBansAsync() + public override async Task> FetchNewBansAsync() { // Note that the /vg/station website only has a single page for bans, so we always do a full refresh Logger.LogInformation("Fetching new bans for /vg/station..."); diff --git a/CentCom.Server/BanSources/YogBanParser.cs b/CentCom.Server/BanSources/YogBanParser.cs index efe9008..f4a0286 100644 --- a/CentCom.Server/BanSources/YogBanParser.cs +++ b/CentCom.Server/BanSources/YogBanParser.cs @@ -9,17 +9,12 @@ namespace CentCom.Server.BanSources; -public class YogBanParser : BanParser +public class YogBanParser(DatabaseContext dbContext, YogBanService banService, ILogger logger) + : BanParser(dbContext, logger) { private const int PagesPerBatch = 12; - private readonly YogBanService _banService; - public YogBanParser(DatabaseContext dbContext, YogBanService banService, ILogger logger) : base(dbContext, logger) - { - _banService = banService; - } - - protected override Dictionary Sources => new Dictionary + protected override Dictionary Sources => new() { { "yogstation", new BanSource { @@ -32,7 +27,7 @@ public YogBanParser(DatabaseContext dbContext, YogBanService banService, ILogger protected override bool SourceSupportsBanIDs => true; protected override string Name => "YogStation"; - public override async Task> FetchNewBansAsync() + public override async Task> FetchNewBansAsync() { Logger.LogInformation("Getting new bans for YogStation..."); var recent = await DbContext.Bans @@ -47,7 +42,7 @@ public override async Task> FetchNewBansAsync() while (true) { - var batch = (await _banService.GetBansBatchedAsync(page, PagesPerBatch)).ToArray(); + var batch = (await banService.GetBansBatchedAsync(page, PagesPerBatch)).ToArray(); foundBans.AddRange(batch); if (!batch.Any() || batch.Any(x => recent.Any(y => y.BanID == x.BanID))) { @@ -59,9 +54,9 @@ public override async Task> FetchNewBansAsync() return foundBans; } - public override async Task> FetchAllBansAsync() + public override async Task> FetchAllBansAsync() { Logger.LogInformation("Getting all bans for YogStation..."); - return await _banService.GetBansBatchedAsync(); + return await banService.GetBansBatchedAsync(); } } \ No newline at end of file diff --git a/CentCom.Server/CentCom.Server.csproj b/CentCom.Server/CentCom.Server.csproj index 08e147f..c12ae17 100644 --- a/CentCom.Server/CentCom.Server.csproj +++ b/CentCom.Server/CentCom.Server.csproj @@ -10,34 +10,34 @@ - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - + + + + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - + + + + + + + + + + diff --git a/CentCom.Server/Data/DatabaseUpdater.cs b/CentCom.Server/Data/DatabaseUpdater.cs index 3593f0c..bf159d2 100644 --- a/CentCom.Server/Data/DatabaseUpdater.cs +++ b/CentCom.Server/Data/DatabaseUpdater.cs @@ -18,7 +18,7 @@ public class DatabaseUpdater : IJob /// /// Types of BanParsers which should not be automatically configured with a refresh schedule /// - private readonly List _autoConfigBypass = new List + private readonly List _autoConfigBypass = new() { typeof(StandardBanParser) }; diff --git a/CentCom.Server/Extensions/AsyncParallelForEach.cs b/CentCom.Server/Extensions/AsyncParallelForEach.cs index 35e4971..482d422 100644 --- a/CentCom.Server/Extensions/AsyncParallelForEach.cs +++ b/CentCom.Server/Extensions/AsyncParallelForEach.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; -namespace Extensions; +namespace CentCom.Server.Extensions; // Taken from https://scatteredcode.net/parallel-foreach-async-in-c/ public static class Extensions diff --git a/CentCom.Server/External/Raw/TgRawBan.cs b/CentCom.Server/External/Raw/TgRawBan.cs index aa4fb4f..464b32b 100644 --- a/CentCom.Server/External/Raw/TgRawBan.cs +++ b/CentCom.Server/External/Raw/TgRawBan.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Text.Json.Serialization; using CentCom.Common.Extensions; using CentCom.Common.Models; diff --git a/CentCom.Server/External/TgApiResponse.cs b/CentCom.Server/External/TgApiResponse.cs index 105ed4a..f29ffce 100644 --- a/CentCom.Server/External/TgApiResponse.cs +++ b/CentCom.Server/External/TgApiResponse.cs @@ -19,7 +19,7 @@ public class TgApiPagination public class TgApiResponse { [JsonPropertyName("data")] - public IEnumerable Data { get; set; } + public List Data { get; set; } [JsonPropertyName("pagination")] public TgApiPagination Pagination { get; set; } diff --git a/CentCom.Server/FlatData/FlatDataFile.cs b/CentCom.Server/FlatData/FlatDataFile.cs index 248199f..fe52900 100644 --- a/CentCom.Server/FlatData/FlatDataFile.cs +++ b/CentCom.Server/FlatData/FlatDataFile.cs @@ -7,7 +7,7 @@ public class FlatDataFile { public uint Version { get; set; } public string Name { get; set; } - public IEnumerable Sources { get; set; } - public IEnumerable JobBans { get; set; } - public IEnumerable ServerBans { get; set; } + public List Sources { get; set; } + public List JobBans { get; set; } + public List ServerBans { get; set; } } \ No newline at end of file diff --git a/CentCom.Server/FlatData/FlatDataImporter.cs b/CentCom.Server/FlatData/FlatDataImporter.cs index 1f631b7..9320a6a 100644 --- a/CentCom.Server/FlatData/FlatDataImporter.cs +++ b/CentCom.Server/FlatData/FlatDataImporter.cs @@ -28,8 +28,8 @@ public async Task RunImports() { foreach (var file in Directory.GetFiles("FlatData/JSON/")) { - var fileData = File.ReadAllText(file); - FlatDataFile deserializedData = null; + var fileData = await File.ReadAllTextAsync(file); + FlatDataFile deserializedData; try { deserializedData = JsonSerializer.Deserialize(fileData, new JsonSerializerOptions @@ -40,7 +40,7 @@ public async Task RunImports() } catch (Exception) { - _logger.LogError($"Failed to deserialize flat data file: '{file}'"); + _logger.LogError("Failed to deserialize flat data file: '{File}'", file); continue; } @@ -52,17 +52,19 @@ public async Task RunImports() // We need to update the data if (lastVersion == null || lastVersion.Version < deserializedData.Version) { - _logger.LogInformation(lastVersion == null ? $"Data from '{deserializedData.Name}' missing from database, adding..." - : $"Out-of-date data found from '{deserializedData.Name} (v{lastVersion.Version}, updating to v{deserializedData.Version}), updating...'"); + if (lastVersion == null) + _logger.LogInformation("Data from '{Name}' missing from database, adding...", deserializedData.Name); + else + _logger.LogInformation("Out-of-date data found from '{Name} (v{LastVersion}, updating to v{NewVersion}), updating...'", deserializedData.Name, lastVersion.Version, deserializedData.Version); try { // Make this an atomic operation to ensure failed imports don't leave holes in the data - using var transaction = await _dbContext.Database.BeginTransactionAsync(); + await using var transaction = await _dbContext.Database.BeginTransactionAsync(); // Update any ban sources if necessary var toUpdate = await _dbContext.BanSources - .Where(x => deserializedData.Sources.Select(x => x.Name).Contains(x.Name)) + .Where(x => deserializedData.Sources.Select(y => y.Name).Contains(x.Name)) .Include(x => x.Bans) .ToListAsync(); foreach (var source in toUpdate) @@ -70,7 +72,7 @@ public async Task RunImports() var match = deserializedData.Sources.First(x => x.Name == source.Name); if (match.RoleplayLevel != source.RoleplayLevel || match.Display != source.Display) { - _logger.LogInformation($"Updating ban source '{source.Name}', found mis-matching metadata with new version of flat dataset"); + _logger.LogInformation("Updating ban source '{Name}', found mis-matching metadata with new version of flat dataset", source.Name); source.Display = match.Display; source.RoleplayLevel = match.RoleplayLevel; } diff --git a/CentCom.Server/Program.cs b/CentCom.Server/Program.cs index e884410..8712e30 100644 --- a/CentCom.Server/Program.cs +++ b/CentCom.Server/Program.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net.Http; using System.Threading.Tasks; using CentCom.Common.Configuration; using CentCom.Common.Data; @@ -14,7 +15,6 @@ using Microsoft.Extensions.Hosting; using Quartz; using Serilog; -using Serilog.Filters; namespace CentCom.Server; @@ -27,7 +27,7 @@ static Task Main(string[] args) .Enrich.FromLogContext() .WriteTo.Logger(lc => { - lc.Filter.ByExcluding(Matching.FromSource("Quartz")); + //lc.Filter.ByExcluding(Matching.FromSource("Quartz")); lc.WriteTo.Console( outputTemplate: "[{Timestamp:HH:mm:ss} {Level:u3}] ({SourceContext}) {Message:lj}{NewLine}{Exception}"); @@ -43,7 +43,7 @@ static Task Main(string[] args) Log.Logger.ForContext() .Information("Starting CentCom Server {Version} ({Commit})", AssemblyInformation.Current.Version, AssemblyInformation.Current.Commit[..7]); - + return CreateHostBuilder(args).RunConsoleAsync(); } @@ -90,16 +90,21 @@ private static IHostBuilder CreateHostBuilder(string[] args) => throw new ArgumentOutOfRangeException(); } - // Add ban services as singletons - services.AddSingleton(); + // Add ban services to contact relevant APIs + services.AddHttpClient(); services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - - // Standard provider is transient as it differs per request - services.AddTransient(); + services.AddHttpClient(); + services.AddHttpClient(); + services.AddHttpClient(); + services.AddHttpClient(); + + // Special consideration for fulp and the SSL woes + var fulpClient = services.AddHttpClient(); + if (config.GetSection("sourceConfig").GetValue("allowFulpExpiredSSL")) + fulpClient.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler() + { + ServerCertificateCustomValidationCallback = (_, _, _, _) => true + }); // Add ban parsers var parsers = AppDomain.CurrentDomain.GetAssemblies().Aggregate(new List(), (curr, next) => @@ -120,17 +125,12 @@ private static IHostBuilder CreateHostBuilder(string[] args) => // Add Quartz services.AddQuartz(q => { - q.UseMicrosoftDependencyInjectionJobFactory(); - q.ScheduleJob(trigger => trigger .StartNow() .WithIdentity("updater"), job => job.WithIdentity("updater")); }); - services.AddQuartzHostedService(o => - { - o.WaitForJobsToComplete = true; - }); + services.AddQuartzHostedService(o => { o.WaitForJobsToComplete = true; }); }); } \ No newline at end of file diff --git a/CentCom.Server/Services/BeeBanService.cs b/CentCom.Server/Services/BeeBanService.cs index bb4e230..b9066f7 100644 --- a/CentCom.Server/Services/BeeBanService.cs +++ b/CentCom.Server/Services/BeeBanService.cs @@ -2,42 +2,29 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Net; +using System.Net.Http; using System.Text.Json; using System.Threading.Tasks; using CentCom.Common.Extensions; using CentCom.Common.Models; -using Extensions; +using CentCom.Server.Extensions; using Microsoft.Extensions.Logging; -using RestSharp; namespace CentCom.Server.Services; -public class BeeBanService : RestBanService +public class BeeBanService(HttpClient client, ILogger logger) : HttpBanService(client, logger) { private const int ParallelRequests = 1; - private static readonly BanSource LrpSource = new BanSource { Name = "bee-lrp" }; - private static readonly BanSource MrpSource = new BanSource { Name = "bee-mrp" }; - - public BeeBanService(ILogger logger) : base(logger) - { - } + private static readonly BanSource LrpSource = new() { Name = "bee-lrp" }; + private static readonly BanSource MrpSource = new() { Name = "bee-mrp" }; protected override string BaseUrl => "https://api.beestation13.com/"; - internal async Task> GetBansAsync(int page = 1) + internal async Task> GetBansAsync(int page = 1) { - var request = - new RestRequest("bans").AddQueryParameter("page", page.ToString()); - var response = await Client.ExecuteAsync(request); - - if (response.StatusCode != HttpStatusCode.OK) - { - FailedRequest(response); - } - var toReturn = new List(); - var content = JsonSerializer.Deserialize(response.Content); + var content = + await GetAsync("bans", new Dictionary() { { "page", page.ToString() } }); foreach (var b in content.GetProperty("data").EnumerateArray()) { var expiryString = b.GetProperty("unbanned_datetime").GetString() ?? @@ -75,7 +62,7 @@ internal async Task> GetBansAsync(int page = 1) return toReturn; } - public async Task> GetBansBatchedAsync(int startpage = 1, int pages = -1) + public async Task> GetBansBatchedAsync(int startpage = 1, int pages = -1) { var maxPages = await GetNumberOfPagesAsync(); var range = Enumerable.Range(startpage, pages != -1 ? pages : maxPages + 8); // pad with 8 pages for safety @@ -88,19 +75,11 @@ await range.AsyncParallelForEach(async page => toReturn.Add(b); } }, ParallelRequests); - return toReturn; + return toReturn.ToList(); } - internal async Task GetNumberOfPagesAsync() - { - var request = new RestRequest("bans"); - var result = await Client.ExecuteAsync(request); - - if (result.StatusCode != HttpStatusCode.OK) - FailedRequest(result); - - return JsonSerializer.Deserialize(result.Content).GetProperty("pages").GetInt32(); - } + internal async Task GetNumberOfPagesAsync() => + (await GetAsync("bans")).GetProperty("pages").GetInt32(); private static BanSource ParseBanSource(string raw) { diff --git a/CentCom.Server/Services/FulpBanService.cs b/CentCom.Server/Services/FulpBanService.cs index 133a52f..dc8d8f6 100644 --- a/CentCom.Server/Services/FulpBanService.cs +++ b/CentCom.Server/Services/FulpBanService.cs @@ -2,51 +2,27 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Net; +using System.Net.Http; using System.Text.Json; using System.Threading.Tasks; using CentCom.Common.Extensions; using CentCom.Common.Models; -using Extensions; -using Microsoft.Extensions.Configuration; +using CentCom.Server.Extensions; using Microsoft.Extensions.Logging; -using RestSharp; namespace CentCom.Server.Services; -public class FulpBanService : RestBanService +public class FulpBanService(HttpClient client, ILogger logger) : HttpBanService(client, logger) { - private readonly bool _allowExpiredSsl; private const int RecordsPerPage = 50; - private static readonly BanSource BanSource = new BanSource { Name = "fulp" }; - - public FulpBanService(ILogger logger, IConfiguration config) : base(logger) - { - _allowExpiredSsl = config.GetSection("sourceConfig").GetValue("allowFulpExpiredSSL"); - } + private static readonly BanSource BanSource = new() { Name = "fulp" }; protected override string BaseUrl => "https://api.fulp.gg/"; - protected override RestClientOptions GenerateClientOptions() - { - var baseOptions = base.GenerateClientOptions(); - if (_allowExpiredSsl) - baseOptions.RemoteCertificateValidationCallback = (sender, certificate, chain, sslPolicyError) => true; - return baseOptions; - } - - public async Task> GetBansAsync(int page = 1) + public async Task> GetBansAsync(int page = 1) { - var request = new RestRequest($"bans/{RecordsPerPage}/{page}"); - var response = await Client.ExecuteAsync(request); - - if (response.StatusCode != HttpStatusCode.OK) - { - FailedRequest(response); - } - + var content = await GetAsync>($"bans/{RecordsPerPage}/{page}"); var toReturn = new List(); - var content = JsonSerializer.Deserialize>(response.Content); foreach (var ban in content["value"].GetProperty("bans").EnumerateArray()) { // Need to get both the expiration as well as the unbanned time as they can differ @@ -84,7 +60,7 @@ public async Task> GetBansAsync(int page = 1) return toReturn; } - public async Task> GetBansBatchedAsync(int startPage = 1, int pages = -1) + public async Task> GetBansBatchedAsync(int startPage = 1, int pages = -1) { var maxPages = await GetNumberOfPagesAsync(); var range = Enumerable.Range(startPage, pages != -1 ? Math.Min(startPage + pages, maxPages) : maxPages); @@ -96,20 +72,12 @@ await range.AsyncParallelForEach(async page => toReturn.Add(b); } }, 6); - return toReturn; + return toReturn.ToList(); } public async Task GetNumberOfPagesAsync() { - var request = new RestRequest($"bans/{RecordsPerPage}/1"); - var result = await Client.ExecuteAsync(request); - - if (result.StatusCode != HttpStatusCode.OK) - { - FailedRequest(result); - } - - var content = JsonSerializer.Deserialize>(result.Content); + var content = await GetAsync>($"bans/{RecordsPerPage}/1"); if (content["value"].TryGetProperty("lastPage", out var lastpage)) { return lastpage.GetInt32(); diff --git a/CentCom.Server/Services/HttpBanService.cs b/CentCom.Server/Services/HttpBanService.cs new file mode 100644 index 0000000..f152ac4 --- /dev/null +++ b/CentCom.Server/Services/HttpBanService.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using CentCom.Server.Exceptions; +using Microsoft.AspNetCore.WebUtilities; +using Microsoft.Extensions.Logging; + +namespace CentCom.Server.Services; + +public abstract class HttpBanService +{ + private readonly ILogger _logger; + private readonly HttpClient _httpClient; + + public virtual JsonSerializerOptions JsonOptions => new() + { + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter() }, + NumberHandling = JsonNumberHandling.AllowReadingFromString + }; + + protected HttpBanService(HttpClient httpClient, ILogger logger) + { + _httpClient = httpClient; + _logger = logger; + ConfigureClient(); + } + + protected abstract string BaseUrl { get; } + + protected void ConfigureClient() + { + if (BaseUrl != null) + _httpClient.BaseAddress = new Uri(BaseUrl); + _httpClient.DefaultRequestHeaders.UserAgent.ParseAdd( + $"Mozilla/5.0 (compatible; CentComBot/{Assembly.GetExecutingAssembly().GetName().Version}; +https://centcom.melonmesa.com/scraper)"); + } + + protected void SetBaseAddress(string address) => _httpClient.BaseAddress = new Uri(address); + + protected async Task GetAsync(string endpoint, Dictionary? queryParams = null, + JsonSerializerOptions? options = null) => + JsonSerializer.Deserialize(await GetAsStringAsync(endpoint, queryParams), options ?? JsonOptions); + + protected async Task GetAsStringAsync(string endpoint, Dictionary? queryParams = null) + { + var url = queryParams is not null ? QueryHelpers.AddQueryString(endpoint, queryParams) : endpoint; + var response = await _httpClient.GetAsync(url); + if (!response.IsSuccessStatusCode) + await FailedRequest(response); + + return await response.Content.ReadAsStringAsync(); + } + + protected async Task FailedRequest(HttpResponseMessage response) + { + var content = await response.Content.ReadAsStringAsync(); + _logger.LogError( + "Source website returned a non-200 HTTP response code.\n\tCode: {ResponseCode}\n\tRequest URL: \"{RequestUrl}\"", + response.StatusCode, response.RequestMessage?.RequestUri); + throw new BanSourceUnavailableException( + $"Source website returned a non-200 HTTP response code.\n\tCode: {response.StatusCode}\n\tRequest URL: \"{response.RequestMessage?.RequestUri}\"", + content); + } +} \ No newline at end of file diff --git a/CentCom.Server/Services/RestBanService.cs b/CentCom.Server/Services/RestBanService.cs deleted file mode 100644 index 47ecad2..0000000 --- a/CentCom.Server/Services/RestBanService.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System.Reflection; -using System.Text; -using CentCom.Server.Exceptions; -using Microsoft.Extensions.Logging; -using RestSharp; - -namespace CentCom.Server.Services; - -public abstract class RestBanService -{ - private readonly ILogger _logger; - - protected RestBanService(ILogger logger) - { - _logger = logger; - InitializeClient(); - } - - protected RestClient Client { get; private set; } - protected abstract string BaseUrl { get; } - - protected void FailedRequest(RestResponse response) - { - // Build error - var url = Client.BuildUri(response.Request); - var messageBuilder = - new StringBuilder( - $"Source website returned a non-200 HTTP response code.\n\tCode: {response.StatusCode}"); - if (url != response.ResponseUri) // Add redirected URL if present - messageBuilder.Append($"\n\tResponse URL: \"{response.ResponseUri}\""); - messageBuilder.Append($"\n\tRequest URL: \"{url}\""); - var message = messageBuilder.ToString(); - - // Log error as appropriate - _logger.LogError(message); - throw new BanSourceUnavailableException(message, response.Content); - } - - - protected void InitializeClient(ConfigureSerialization configureSerialization = null) - { - if (BaseUrl == null) - return; - - Client = new RestClient(GenerateClientOptions(), configureSerialization: configureSerialization); - } - - protected virtual RestClientOptions GenerateClientOptions() => - new(BaseUrl) - { - // Setup user agent - UserAgent = - $"Mozilla/5.0 (compatible; CentComBot/{Assembly.GetExecutingAssembly().GetName().Version}; +https://centcom.melonmesa.com/scraper)" - }; -} \ No newline at end of file diff --git a/CentCom.Server/Services/StandardProviderService.cs b/CentCom.Server/Services/StandardProviderService.cs index 65d6514..e8885a4 100644 --- a/CentCom.Server/Services/StandardProviderService.cs +++ b/CentCom.Server/Services/StandardProviderService.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net; +using System.Net.Http; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading.Tasks; @@ -10,34 +10,30 @@ using CentCom.Common.Models.Rest; using CentCom.Server.Configuration; using Microsoft.Extensions.Logging; -using RestSharp; -using RestSharp.Serializers.Json; namespace CentCom.Server.Services; -public class StandardProviderService : RestBanService +public class StandardProviderService(HttpClient client, ILogger logger) + : HttpBanService(client, logger) { private string _baseUrl; private bool _configured; - public StandardProviderService(ILogger logger) : base(logger) - { - } - public BanSource Source { get; private set; } protected override string BaseUrl => _baseUrl; - private async Task> GetBansAsync(int? cursor = null) + public override JsonSerializerOptions JsonOptions => new JsonSerializerOptions() { - var request = new RestRequest("api/ban"); - if (cursor.HasValue) - request.AddQueryParameter("cursor", cursor.ToString()); - var response = await Client.ExecuteAsync>(request); - - if (response.StatusCode != HttpStatusCode.OK) - FailedRequest(response); + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Converters = { new JsonStringEnumConverter() } + }.AddCentComOptions(); - return response.Data.Select(x => new Ban + private async Task> GetBansAsync(int? cursor = null) + { + var data = await GetAsync>("api/ban", + cursor.HasValue ? new Dictionary() { { "cursor", cursor.ToString() } } : null, JsonOptions); + + return data.Select(x => new Ban { BanID = x.Id.ToString(), BannedBy = x.BannedBy?.CanonicalKey, @@ -53,10 +49,10 @@ private async Task> GetBansAsync(int? cursor = null) }) .ToHashSet(), SourceNavigation = Source - }); + }).ToList(); } - public async Task> GetBansBatchedAsync(int? cursor = null, IEnumerable searchFor = null) + public async Task> GetBansBatchedAsync(int? cursor = null, List searchFor = null) { if (!_configured) throw new Exception("Cannot get bans from an unconfigured external source"); @@ -69,11 +65,11 @@ public async Task> GetBansBatchedAsync(int? cursor = null, IEnu do { lastResponse = (await GetBansAsync(lastRequested)).ToList(); - if (!lastResponse.Any()) + if (lastResponse.Count == 0) break; lastRequested = int.Parse(lastResponse[^1].BanID); result.AddRange(lastResponse); - } while (lastResponse.Any() && + } while (lastResponse.Count != 0 && (searchFor == null || lastResponse.Any(x => searchFor.Contains(int.Parse(x.BanID))))); return result; @@ -85,15 +81,7 @@ public void Configure(StandardProviderConfiguration config) throw new Exception("Cannot re-configure standard exporter provider, already configured"); _configured = true; _baseUrl = config.Url; + SetBaseAddress(_baseUrl); Source = new BanSource { Name = config.Id, Display = config.Display, RoleplayLevel = config.RoleplayLevel }; - - // Setup JSON for client - var options = new JsonSerializerOptions(); - options.AddCentComOptions(); - options.PropertyNamingPolicy = JsonNamingPolicy.CamelCase; - options.Converters.Insert(0, new JsonStringEnumConverter()); - - // Re-initialize client with new url - InitializeClient(o => o.UseSystemTextJson(options)); } } \ No newline at end of file diff --git a/CentCom.Server/Services/TGMCBanService.cs b/CentCom.Server/Services/TGMCBanService.cs index 7bc6d14..ae241a1 100644 --- a/CentCom.Server/Services/TGMCBanService.cs +++ b/CentCom.Server/Services/TGMCBanService.cs @@ -2,14 +2,13 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Net; +using System.Net.Http; using System.Text.Json; using System.Threading.Tasks; using CentCom.Common.Extensions; using CentCom.Common.Models; -using Extensions; +using CentCom.Server.Extensions; using Microsoft.Extensions.Logging; -using RestSharp; namespace CentCom.Server.Services; @@ -22,41 +21,32 @@ namespace CentCom.Server.Services; /// for using the paging must account for the possibility of a job ban /// spanning two seperate pages. /// -public class TGMCBanService : RestBanService +public class TGMCBanService(HttpClient client, ILogger logger) : HttpBanService(client, logger) { private const int RecordsPerPage = 100; - private static readonly BanSource BanSource = new BanSource { Name = "tgmc" }; - - public TGMCBanService(ILogger logger) : base(logger) - { - } + private static readonly BanSource BanSource = new() { Name = "tgmc" }; protected override string BaseUrl => "https://statbus.psykzz.com/api/"; - public async Task> GetBansAsync(int page = 1) + public async Task> GetBansAsync(int page = 1) { - var request = new RestRequest($"bans/{page}").AddQueryParameter("limit", RecordsPerPage.ToString()); - var response = await Client.ExecuteAsync(request); - - if (response.StatusCode != HttpStatusCode.OK) - { - FailedRequest(response); - } - var toReturn = new List(); var dirtyBans = new List(); - var content = JsonSerializer.Deserialize>(response.Content); + var content = await GetAsync>($"bans/{page}", + new Dictionary() { { "limit", RecordsPerPage.ToString() } }); foreach (var bh in content["bans"].EnumerateObject()) { var ban = bh.Value; // Ban expiration could be based on the expiration time field or the existance of the unbanned datetime // field, so we have to check both. - var expiration = ban.GetProperty("unbanned_datetime").GetString() == null ? (DateTime?)null + var expiration = ban.GetProperty("unbanned_datetime").GetString() == null + ? (DateTime?)null : DateTime.Parse(ban.GetProperty("unbanned_datetime").GetString()); if (!expiration.HasValue) { - expiration = ban.GetProperty("expiration_time").GetString() == null ? null + expiration = ban.GetProperty("expiration_time").GetString() == null + ? null : DateTime.Parse(ban.GetProperty("expiration_time").GetString()); } @@ -96,7 +86,7 @@ public async Task> GetBansAsync(int page = 1) return toReturn; } - public async Task> GetBansBatchedAsync(int startPage = 1, int pages = -1) + public async Task> GetBansBatchedAsync(int startPage = 1, int pages = -1) { var maxPages = await GetNumberOfPagesAsync(); var range = Enumerable.Range(startPage, pages != -1 ? Math.Min(startPage + pages, maxPages) : maxPages); @@ -111,7 +101,7 @@ await range.AsyncParallelForEach(async page => if (dirtyBans.IsEmpty) - return Enumerable.Empty(); + return []; // We have to ensure that our jobs are correctly grouped due to possible errors with paging var cleanBans = new List(dirtyBans.Where(x => x.BanType == BanType.Server)); @@ -131,6 +121,7 @@ await range.AsyncParallelForEach(async page => // is not an optimal solution cleanBans.RemoveAt(0); } + if (pages != -1 && startPage + pages < maxPages && cleanBans.LastOrDefault()?.BanType == BanType.Job) { // Discard the last ban if it is a job ban as it may be incomplete. @@ -143,15 +134,8 @@ await range.AsyncParallelForEach(async page => public async Task GetNumberOfPagesAsync() { - var request = new RestRequest("bans/1").AddQueryParameter("limit", RecordsPerPage.ToString()); - var result = await Client.ExecuteAsync(request); - - if (result.StatusCode != HttpStatusCode.OK) - { - FailedRequest(result); - } - - var content = JsonSerializer.Deserialize>(result.Content); + var content = await GetAsync>("bans/1", + new Dictionary() { { "limit", RecordsPerPage.ToString() } }); if (content["page"].TryGetProperty("total", out var lastpage)) { return lastpage.GetInt32(); diff --git a/CentCom.Server/Services/TgBanService.cs b/CentCom.Server/Services/TgBanService.cs index 5f9a9f1..87589a1 100644 --- a/CentCom.Server/Services/TgBanService.cs +++ b/CentCom.Server/Services/TgBanService.cs @@ -1,50 +1,23 @@ using System.Collections.Generic; using System.Linq; -using System.Net; -using System.Text.Json; -using System.Text.Json.Serialization; +using System.Net.Http; using System.Threading.Tasks; using CentCom.Common.Models; using CentCom.Server.External; using CentCom.Server.External.Raw; using Microsoft.Extensions.Logging; -using RestSharp; -using RestSharp.Serializers.Json; namespace CentCom.Server.Services; -public class TgBanService : RestBanService +public class TgBanService(HttpClient client, ILogger logger) : HttpBanService(client, logger) { - private static readonly BanSource BanSource = new BanSource { Name = "tgstation" }; - - public static readonly JsonSerializerOptions JsonOptions = new() - { - PropertyNameCaseInsensitive = true, - Converters = { new JsonStringEnumConverter() }, - NumberHandling = JsonNumberHandling.AllowReadingFromString - }; - - public TgBanService(ILogger logger) : base(logger) - { - // Re-initialize to control JSON serialization behaviour - InitializeClient(o => o.UseSystemTextJson(JsonOptions)); - } - protected override string BaseUrl => "https://statbus.space/"; - public async Task> GetBansAsync(int? page = null) - { - var request = new RestRequest($"bans/public/v1/{page}") - .AddQueryParameter("json", "true"); - var response = await Client.ExecuteAsync(request); - - if (response.StatusCode != HttpStatusCode.OK) - FailedRequest(response); - - return response.Data.Data.ToList(); - } + public async Task> GetBansAsync(int? page = null) => + (await GetAsync($"bans/public/v1/{page}", + new Dictionary() { { "json", "true" } })).Data.ToList(); - public async Task> GetBansBatchedAsync() + public async Task> GetBansBatchedAsync(BanSource source) { var allBans = new List(); var page = 1; @@ -58,6 +31,6 @@ public async Task> GetBansBatchedAsync() page++; } - return allBans.Select(x => x.AsBan(BanSource)).DistinctBy(x => x.BanID); + return allBans.Select(x => x.AsBan(source)).DistinctBy(x => x.BanID).ToList(); } } \ No newline at end of file diff --git a/CentCom.Server/Services/VgBanService.cs b/CentCom.Server/Services/VgBanService.cs index b7af3d8..fcdfa80 100644 --- a/CentCom.Server/Services/VgBanService.cs +++ b/CentCom.Server/Services/VgBanService.cs @@ -11,18 +11,13 @@ namespace CentCom.Server.Services; -public class VgBanService +public class VgBanService(ILogger logger) { - private static readonly BanSource BanSource = new BanSource { Name = "vgstation" }; - private readonly ILogger _logger; - - public VgBanService(ILogger logger) - { - _logger = logger; - } + private static readonly BanSource BanSource = new() { Name = "vgstation" }; + private readonly ILogger _logger = logger; // TODO: cleanup - public async Task> GetBansAsync() + public async Task> GetBansAsync() { var toReturn = new List(); var config = AngleSharp.Configuration.Default.WithDefaultLoader(); @@ -32,7 +27,7 @@ public async Task> GetBansAsync() if (document.StatusCode != HttpStatusCode.OK) { _logger.LogError( - $"Source website returned a non-200 HTTP response code. Url: \"{document.Url}\", code: {document.StatusCode}"); + "Source website returned a non-200 HTTP response code. Url: \"{Url}\", code: {StatusCode}", document.Url, document.StatusCode); throw new BanSourceUnavailableException( $"Source website returned a non-200 HTTP response code. Url: \"{document.Url}\", code: {document.StatusCode}", document.TextContent); } diff --git a/CentCom.Server/Services/YogBanService.cs b/CentCom.Server/Services/YogBanService.cs index 9487d56..bd9d8bc 100644 --- a/CentCom.Server/Services/YogBanService.cs +++ b/CentCom.Server/Services/YogBanService.cs @@ -2,7 +2,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Net; +using System.Net.Http; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; @@ -10,38 +10,24 @@ using CentCom.Common.Extensions; using CentCom.Common.Models; using Microsoft.Extensions.Logging; -using RestSharp; namespace CentCom.Server.Services; -public class YogBanService : RestBanService +public class YogBanService(HttpClient client, ILogger logger) : HttpBanService(client, logger) { private const int ParallelRequests = 12; private const int RequestsPerMinute = 60; - private static readonly BanSource BanSource = new BanSource { Name = "yogstation" }; - private readonly Regex _pagesPattern = new Regex(@"]+>(?[0-9]+)<\/a>", RegexOptions.Compiled | RegexOptions.Multiline); - - public YogBanService(ILogger logger) : base(logger) - { - } + private static readonly BanSource BanSource = new() { Name = "yogstation" }; + private readonly Regex _pagesPattern = new(@"]+>(?[0-9]+)<\/a>", RegexOptions.Compiled | RegexOptions.Multiline); protected override string BaseUrl => "https://yogstation.net/"; - private async Task> GetBansAsync(int page = 1) + private async Task> GetBansAsync(int page = 1) { - var request = new RestRequest("bans") - .AddQueryParameter("json", "1") - .AddQueryParameter("page", page.ToString()) - .AddQueryParameter("amount", "1000"); - var response = await Client.ExecuteAsync(request); - - if (response.StatusCode != HttpStatusCode.OK) - { - FailedRequest(response); - } - var toReturn = new List(); - var content = JsonSerializer.Deserialize>>(response.Content); + var content = await GetAsync>>("bans", + new Dictionary() { { "json", "1" }, { "page", page.ToString() }, { "amount", "1000" } }); + foreach (var b in content) { var expiryString = b["unbanned_datetime"].GetString() ?? b["expiration_time"].GetString(); @@ -71,7 +57,7 @@ private async Task> GetBansAsync(int page = 1) return toReturn; } - public async Task> GetBansBatchedAsync(int startpage = 1, int pages = -1) + public async Task> GetBansBatchedAsync(int startpage = 1, int pages = -1) { var maxPages = await GetNumberOfPagesAsync(); var range = Enumerable.Range(startpage, pages != -1 ? pages : maxPages + 1); // pad with a page for safety @@ -113,20 +99,13 @@ public async Task> GetBansBatchedAsync(int startpage = 1, int p await Task.WhenAll(allTasks); - return toReturn; + return toReturn.ToList(); } private async Task GetNumberOfPagesAsync() { - var request = new RestRequest("bans"); - var result = await Client.ExecuteAsync(request); - - if (result.StatusCode != HttpStatusCode.OK) - { - FailedRequest(result); - } - - var match = _pagesPattern.Matches(result.Content); + var content = await GetAsStringAsync("bans"); + var match = _pagesPattern.Matches(content); if (!match.Any()) { throw new Exception("Failed to find page numbers on yogstation.net bans page"); diff --git a/CentCom.Test/BanServices/BeeBanServiceTests.cs b/CentCom.Test/BanServices/BeeBanServiceTests.cs index 4e8d85c..fb71b0c 100644 --- a/CentCom.Test/BanServices/BeeBanServiceTests.cs +++ b/CentCom.Test/BanServices/BeeBanServiceTests.cs @@ -1,3 +1,4 @@ +using System.Net.Http; using System.Threading.Tasks; using CentCom.Server.Services; using Xunit; @@ -9,7 +10,7 @@ public class BeeBanServiceTests [Fact] public async Task BeeBans_ShouldGetPages() { - var toTest = new BeeBanService(null); + var toTest = new BeeBanService(new HttpClient(), null); var result = await toTest.GetNumberOfPagesAsync(); Assert.NotEqual(0, result); } @@ -17,7 +18,7 @@ public async Task BeeBans_ShouldGetPages() [Fact] public async Task BeeBans_ShouldGetBans() { - var toTest = new BeeBanService(null); + var toTest = new BeeBanService(new HttpClient(), null); var result = await toTest.GetBansAsync(); Assert.NotNull(result); Assert.NotEmpty(result); diff --git a/CentCom.Test/BanServices/TgBanServiceTests.cs b/CentCom.Test/BanServices/TgBanServiceTests.cs deleted file mode 100644 index 1582916..0000000 --- a/CentCom.Test/BanServices/TgBanServiceTests.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.IO; -using System.Linq; -using System.Text.Json; -using System.Threading.Tasks; -using CentCom.Server.External; -using CentCom.Server.External.Raw; -using CentCom.Server.Services; -using Xunit; - -namespace CentCom.Test.BanServices; - -public class TgBanServiceTests -{ - [Fact] - public async Task TgBans_ShouldParseData() - { - var testData = await File.ReadAllTextAsync("BanServices/TgBanSample.json"); - var result = JsonSerializer.Deserialize(testData, TgBanService.JsonOptions); - var banData = result.Data.Select(x => x.AsBan(null)); - Assert.NotNull(result); - Assert.NotEmpty(banData); - } -} \ No newline at end of file diff --git a/CentCom.Test/CentCom.Test.csproj b/CentCom.Test/CentCom.Test.csproj index 7e35165..4ecf0ff 100644 --- a/CentCom.Test/CentCom.Test.csproj +++ b/CentCom.Test/CentCom.Test.csproj @@ -5,17 +5,17 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Directory.Build.props b/Directory.Build.props index 151841d..ce757ee 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,6 +1,6 @@ - net7.0 + net9.0 MelonMesa and CentCom Contributors Copyright © $([System.DateTime]::Now.Year) MelonMesa and CentCom Contributors