diff --git a/MergerCli/Dockerfile b/MergerCli/Dockerfile index 62667a82..0a4cd407 100644 --- a/MergerCli/Dockerfile +++ b/MergerCli/Dockerfile @@ -1,10 +1,11 @@ #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. FROM mcr.microsoft.com/dotnet/runtime:6.0 AS base +ARG VERSION="1.0.0.0" WORKDIR /app FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -ARG VERSION="9.9.9.9" +ARG VERSION="1.0.0.0" WORKDIR /src COPY ["MergerCli/MergerCli.csproj", "MergerCli/"] COPY ["MergerLogic/MergerLogic.csproj", "MergerLogic/"] diff --git a/MergerCli/Process.cs b/MergerCli/Process.cs index 2e778cf5..d63c0d51 100644 --- a/MergerCli/Process.cs +++ b/MergerCli/Process.cs @@ -125,7 +125,8 @@ private void ProcessBatch(TileFormat targetFormat, IData baseData, List ne } } - baseData.UpdateTiles(tiles); + var t = Task.Run(() => baseData.UpdateTiles(tiles)); + t.Wait(); Interlocked.Add(ref tileProgressCount, tiles.Count); this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] Tile Count: {tileProgressCount} / {totalTileCount}"); diff --git a/MergerCli/appsettings.json b/MergerCli/appsettings.json index d3eec2a7..272f6088 100644 --- a/MergerCli/appsettings.json +++ b/MergerCli/appsettings.json @@ -1,6 +1,7 @@ { "GENERAL": { "validate": false, + "servicePointManagerDefaultConnectionLimit": 100, "resumeOutputFolder": "./", "uploadOnly": false, "parallel": { diff --git a/MergerLogic/Clients/S3Client.cs b/MergerLogic/Clients/S3Client.cs index c7df9d96..a1202bf5 100644 --- a/MergerLogic/Clients/S3Client.cs +++ b/MergerLogic/Clients/S3Client.cs @@ -31,12 +31,10 @@ public S3Client(IAmazonS3 client, IPathUtils pathUtils, IGeoUtils geoUtils, IIma string methodName = MethodBase.GetCurrentMethod().Name; try { - this._logger.LogDebug($"[{methodName}] start, key {key}"); + this._logger.LogDebug($"[{methodName}] start, BucketName: {this._bucket}, Key: {key}"); var request = new GetObjectRequest() { BucketName = this._bucket, Key = key }; - this._logger.LogDebug($"[{methodName}] start GetObjectAsync, BucketName: {request.BucketName}, Key: {request.Key}"); var getObjectTask = this._client.GetObjectAsync(request); GetObjectResponse res = getObjectTask.Result; - this._logger.LogDebug($"[{methodName}] requested key {key} recieved"); byte[] image; using (MemoryStream ms = new MemoryStream()) @@ -48,12 +46,12 @@ public S3Client(IAmazonS3 client, IPathUtils pathUtils, IGeoUtils geoUtils, IIma image = ms.ToArray(); } - this._logger.LogDebug($"[{methodName}] end, key {key}"); + this._logger.LogDebug($"[{methodName}] end, BucketName: {this._bucket}, Key: {key}"); return image; } catch (AggregateException e) { - this._logger.LogDebug($"[{methodName}] exception, Message: {e.Message}"); + this._logger.LogError($"[{methodName}] exception, Message: {e.Message}, returned null"); return null; } } @@ -65,6 +63,7 @@ public S3Client(IAmazonS3 client, IPathUtils pathUtils, IGeoUtils geoUtils, IIma var key = this.GetTileKey(z, x, y); if (key == null) { + this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y} - no tile was found"); return null; } @@ -99,7 +98,7 @@ public override bool TileExists(int z, int x, int y) public void UpdateTile(Tile tile) { string methodName = MethodBase.GetCurrentMethod().Name; - this._logger.LogDebug($"[{methodName}] start {tile.ToString()}"); + this._logger.LogDebug($"[{methodName}] start {tile.ToString()} to BucketName: {this._bucket}"); string key = this._pathUtils.GetTilePath(this.path, tile, true); var request = new PutObjectRequest() @@ -111,23 +110,21 @@ public void UpdateTile(Tile tile) using (var ms = new MemoryStream(buffer)) { request.InputStream = ms; - this._logger.LogDebug($"[{methodName}] start PutObjectAsync BucketName: {request.BucketName}, Key: {request.Key}"); var task = this._client.PutObjectAsync(request); var res = task.Result; } - this._logger.LogDebug($"[{methodName}] end {tile.ToString()}"); + this._logger.LogDebug($"[{methodName}] end {tile.ToString()} to BucketName: {this._bucket}"); } private string? GetTileKey(int z, int x, int y) { string methodName = MethodBase.GetCurrentMethod().Name; - this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y}"); + this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y} from BucketName: {this._bucket}"); string keyPrefix = this._pathUtils.GetTilePathWithoutExtension(this.path, z, x, y, true); var listRequests = new ListObjectsV2Request { BucketName = this._bucket, Prefix = keyPrefix, MaxKeys = 1 }; - this._logger.LogDebug($"[{methodName}] start ListObjectsV2Async BucketName: {listRequests.BucketName}, Prefix: {listRequests.Prefix}"); var listObjectsTask = this._client.ListObjectsV2Async(listRequests); string? result = listObjectsTask.Result.S3Objects.FirstOrDefault()?.Key; - this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y}"); + this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y} from BucketName: {this._bucket}"); return result; } } diff --git a/MergerLogic/DataTypes/Data.cs b/MergerLogic/DataTypes/Data.cs index ac589231..c4197829 100644 --- a/MergerLogic/DataTypes/Data.cs +++ b/MergerLogic/DataTypes/Data.cs @@ -79,7 +79,7 @@ protected Data(IServiceProvider container, DataType type, string path, int batch this._container = container; var loggerFactory = container.GetRequiredService(); this._logger = loggerFactory.CreateLogger(this.GetType()); - this._logger.LogInformation($"[{methodName}] Ctor started"); + this._logger.LogInformation($"[{methodName}] Data Ctor started"); this.Type = type; this.Path = path; this.BatchSize = batchSize; @@ -139,6 +139,7 @@ protected Data(IServiceProvider container, DataType type, string path, int batch this._logger.LogInformation($"[{methodName}] Initialize Started"); this.Initialize(); + this._logger.LogInformation($"[{methodName}] Initialize ended"); this._logger.LogInformation($"[{methodName}] Checking if exists, {this.Type}: {this.Path}"); bool exists = this.Exists(); @@ -152,9 +153,8 @@ protected Data(IServiceProvider container, DataType type, string path, int batch throw new Exception($"{this.Type} source {path} does not exist."); } } - this._logger.LogInformation($"[{methodName}] Validate Started"); this.Validate(); - this._logger.LogInformation($"[{methodName}] Ctor Ended"); + this._logger.LogInformation($"[{methodName}] Data Ctor Ended"); } protected virtual void Initialize() @@ -256,18 +256,18 @@ public bool TileExists(Coord coord) public Tile? GetCorrespondingTile(Coord coords, bool upscale) { - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start for coord: {coords.ToString()}, upscale: {upscale}"); + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start for coord: z: {coords.Z}, x: {coords.X}, y: {coords.Y}, upscale: {upscale}"); Tile? correspondingTile = this.GetTile(coords.Z, coords.X, coords.Y); if (upscale && correspondingTile == null) { correspondingTile = this.GetLastExistingTile(coords); } - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end for coord: {coords.ToString()}, upscale: {upscale}"); + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end for coord: z: {coords.Z}, x: {coords.X}, y: {coords.Y}, upscale: {upscale}"); return correspondingTile; } - public void UpdateTiles(IEnumerable tiles) + public async Task UpdateTiles(IEnumerable tiles) { this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] update tiles started"); var targetTiles = tiles.Select(tile => @@ -276,11 +276,11 @@ public void UpdateTiles(IEnumerable tiles) Tile? targetTile = this.FromCurrentGridTile(convertedTile); return targetTile; }).Where(tile => tile is not null); - this.InternalUpdateTiles(targetTiles); + await this.InternalUpdateTilesAsync(targetTiles); this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] update tiles ended"); } - protected abstract void InternalUpdateTiles(IEnumerable targetTiles); + protected abstract Task InternalUpdateTilesAsync(IEnumerable targetTiles); public virtual void Wrapup() { diff --git a/MergerLogic/DataTypes/Fs.cs b/MergerLogic/DataTypes/Fs.cs index d0fc512c..c1aa5c82 100644 --- a/MergerLogic/DataTypes/Fs.cs +++ b/MergerLogic/DataTypes/Fs.cs @@ -174,7 +174,7 @@ public override long TileCount() return count; } - protected override void InternalUpdateTiles(IEnumerable targetTiles) + protected override Task InternalUpdateTilesAsync(IEnumerable targetTiles) { foreach (Tile tile in targetTiles) { @@ -190,6 +190,7 @@ protected override void InternalUpdateTiles(IEnumerable targetTiles) } } } + return Task.CompletedTask; } } } diff --git a/MergerLogic/DataTypes/Gpkg.cs b/MergerLogic/DataTypes/Gpkg.cs index 2923e455..3cbf81ec 100644 --- a/MergerLogic/DataTypes/Gpkg.cs +++ b/MergerLogic/DataTypes/Gpkg.cs @@ -189,11 +189,12 @@ public override long TileCount() return result; } - protected override void InternalUpdateTiles(IEnumerable targetTiles) + protected override Task InternalUpdateTilesAsync(IEnumerable targetTiles) { this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] started"); this.Utils.InsertTiles(targetTiles); this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] ended"); + return Task.CompletedTask; } } } diff --git a/MergerLogic/DataTypes/HttpDataSource.cs b/MergerLogic/DataTypes/HttpDataSource.cs index 17a1ce80..061d0974 100644 --- a/MergerLogic/DataTypes/HttpDataSource.cs +++ b/MergerLogic/DataTypes/HttpDataSource.cs @@ -84,7 +84,7 @@ protected IEnumerable GetTiles() } } - protected override void InternalUpdateTiles(IEnumerable targetTiles) + protected override Task InternalUpdateTilesAsync(IEnumerable targetTiles) { throw new NotImplementedException(); } diff --git a/MergerLogic/DataTypes/IData.cs b/MergerLogic/DataTypes/IData.cs index c8208823..fbc36d81 100644 --- a/MergerLogic/DataTypes/IData.cs +++ b/MergerLogic/DataTypes/IData.cs @@ -16,7 +16,7 @@ public interface IData long TileCount(); bool TileExists(Coord coord); bool TileExists(Tile tile); - void UpdateTiles(IEnumerable tiles); + Task UpdateTiles(IEnumerable tiles); void Wrapup(); } } diff --git a/MergerLogic/DataTypes/S3.cs b/MergerLogic/DataTypes/S3.cs index 18cbefa9..9a3540cb 100644 --- a/MergerLogic/DataTypes/S3.cs +++ b/MergerLogic/DataTypes/S3.cs @@ -67,17 +67,17 @@ public override void Reset() private List GetZoomLevels() { - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start"); - List zoomLevels = new List(); + List zoomLevels = new List(Data.MaxZoomRead); for (int zoomLevel = 0; zoomLevel < Data.MaxZoomRead; zoomLevel++) { + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] Check if zoom {zoomLevel} exists"); + // TODO: need to check if can be updated dynamically if (this.FolderExists($"{zoomLevel}/")) { zoomLevels.Add(zoomLevel); } } - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] ended"); return zoomLevels; } @@ -158,15 +158,16 @@ private bool FolderExists(string directory) }; var task = this._client.ListObjectsV2Async(listRequests); var response = task.Result; - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end"); - return response.KeyCount > 0; + bool isExists = response.KeyCount > 0; + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] directory: {directory}, isExists={isExists}"); + return isExists; } public override bool Exists() { - this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] bucket: {this._bucket}, path: {this.Path}"); + this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] Checking if exists, bucket: {this._bucket}, path: {this.Path}"); bool exists = FolderExists(""); - this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] ended"); + this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] exists={exists} ended"); return exists; } @@ -196,14 +197,14 @@ public override long TileCount() return tileCount; } - protected override void InternalUpdateTiles(IEnumerable targetTiles) + protected override async Task InternalUpdateTilesAsync(IEnumerable targetTiles) { - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start"); - foreach (var tile in targetTiles) + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start upload tiles to S3"); + await Parallel.ForEachAsync(targetTiles, new ParallelOptions { MaxDegreeOfParallelism = -1 }, async (tile, cancellationToken) => { - this.Utils.UpdateTile(tile); - } - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end"); + await Task.Run(() => this.Utils.UpdateTile(tile), cancellationToken); + }); + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end upload tiles to S3"); } } } diff --git a/MergerLogic/Monitoring/OpenTelemetryFormattedConsoleExporter.cs b/MergerLogic/Monitoring/OpenTelemetryFormattedConsoleExporter.cs index 08b62fe7..81522e83 100644 --- a/MergerLogic/Monitoring/OpenTelemetryFormattedConsoleExporter.cs +++ b/MergerLogic/Monitoring/OpenTelemetryFormattedConsoleExporter.cs @@ -1,6 +1,8 @@ using OpenTelemetry; using OpenTelemetry.Exporter; using OpenTelemetry.Logs; +using System; +using System.Net; namespace MergerLogic.Monitoring { @@ -8,6 +10,7 @@ public class OpenTelemetryFormattedConsoleExporter : ConsoleExporter { private const string SERVICE_NAME_ATTRIBUTE = "service.name"; private const string SERVICE_VERSION_ATTRIBUTE = "service.version"; + private const string SERVICE_HOST_NAME_ATTRIBUTE = "service.host.name"; public OpenTelemetryFormattedConsoleExporter(ConsoleExporterOptions options) : base(options) @@ -29,9 +32,13 @@ private string MCTextFormat(LogRecord record) var resource = this.ParseResource(); var serviceName = this.GetResourceAttribute(resource, SERVICE_NAME_ATTRIBUTE, "unknown_service"); var serviceVersion = this.GetResourceAttribute(resource, SERVICE_VERSION_ATTRIBUTE, "unknown_version"); + if (!resource.ContainsKey(SERVICE_HOST_NAME_ATTRIBUTE)) + { + resource.Add(SERVICE_HOST_NAME_ATTRIBUTE, Dns.GetHostName()); + } + var serviceHostName = this.GetResourceAttribute(resource, SERVICE_HOST_NAME_ATTRIBUTE, "unknown_host_name"); var exception = record.Exception != null ? $" [{record.Exception}]" : string.Empty; - - return $"[{this.FormatTime(record.Timestamp)}] [{record.LogLevel}] [{serviceName}] [{serviceVersion}] [{record.CategoryName}] {record.State}{exception}"; + return $"[{this.FormatTime(record.Timestamp)}] [{record.LogLevel}] [{serviceName}] [{serviceHostName}] [{serviceVersion}] [{Environment.CurrentManagedThreadId}] [{record.CategoryName}] {record.State}{exception}"; } private string FormatTime(DateTime time) diff --git a/MergerLogicUnitTests/DataTypes/FSTest.cs b/MergerLogicUnitTests/DataTypes/FSTest.cs index 82481e96..e4d41afa 100644 --- a/MergerLogicUnitTests/DataTypes/FSTest.cs +++ b/MergerLogicUnitTests/DataTypes/FSTest.cs @@ -11,6 +11,7 @@ using System.IO; using System.IO.Abstractions; using System.Linq; +using System.Threading.Tasks; namespace MergerLogicUnitTests.DataTypes { @@ -481,7 +482,8 @@ public void UpdateTiles(bool isBase, bool isOneXOne, GridOrigin origin) fileMock.InSequence(seq).Setup(f => f.OpenWrite()).Returns(fileStream); } - fsSource.UpdateTiles(testTiles); + var t = Task.Run(() => fsSource.UpdateTiles(testTiles)); + t.Wait(); CollectionAssert.AreEqual(new byte[] { 0 }, fileStreams[0].ToArray()); if (isOneXOne) diff --git a/MergerLogicUnitTests/DataTypes/GpkgTest.cs b/MergerLogicUnitTests/DataTypes/GpkgTest.cs index af06c58a..c200d6e2 100644 --- a/MergerLogicUnitTests/DataTypes/GpkgTest.cs +++ b/MergerLogicUnitTests/DataTypes/GpkgTest.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; namespace MergerLogicUnitTests.DataTypes { @@ -455,7 +456,8 @@ public void UpdateTiles(bool isBase, bool isOneXOne, GridOrigin origin) new Tile(1, 2, 3, new byte[] { }), new Tile(7, 7, 7, new byte[] { }), new Tile(2, 2, 3, new byte[] { }) }; - gpkg.UpdateTiles(testTiles); + var t = Task.Run(() => gpkg.UpdateTiles(testTiles)); + t.Wait(); var expectedTiles = isOneXOne ? new Tile[] { testTiles[0], testTiles[2] } : testTiles; var tileComparer = EqualityComparerFactory.Create((tile1, tile2) => diff --git a/MergerLogicUnitTests/DataTypes/S3Test.cs b/MergerLogicUnitTests/DataTypes/S3Test.cs index 1a6bcd0c..81714e45 100644 --- a/MergerLogicUnitTests/DataTypes/S3Test.cs +++ b/MergerLogicUnitTests/DataTypes/S3Test.cs @@ -5,13 +5,13 @@ using MergerLogic.DataTypes; using MergerLogic.Utils; using MergerLogicUnitTests.testUtils; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; using System.Collections.Generic; using System.Threading; +using System.Threading.Tasks; namespace MergerLogicUnitTests.DataTypes { @@ -438,7 +438,8 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) { var testTiles = new Tile[] { - new Tile(1, 2, 3, new byte[] { }), new Tile(7, 7, 7, new byte[] { }), + new Tile(1, 2, 3, new byte[] { }), + new Tile(7, 7, 7, new byte[] { }), new Tile(2, 2, 3, new byte[] { }) }; var seq = new MockSequence(); @@ -449,7 +450,6 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) if (origin == GridOrigin.UPPER_LEFT) { this._geoUtilsMock - .InSequence(seq) .Setup(utils => utils.FlipY(It.IsAny())) .Returns(t => t.Y); } @@ -457,7 +457,6 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) if (isOneXOne) { this._oneXOneConvertorMock - .InSequence(seq) .Setup(converter => converter.TryFromTwoXOne(It.IsAny())) .Returns(tile => tile.Z != 7 ? tile : null); } @@ -465,7 +464,6 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) if (!isOneXOne || tile.Z != 7) { this._s3UtilsMock - .InSequence(seq) .Setup(utils => utils.UpdateTile(It.IsAny())); } } @@ -473,7 +471,8 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) Grid grid = isOneXOne ? Grid.OneXOne : Grid.TwoXOne; var s3Source = new S3(this._pathUtilsMock.Object, this._serviceProviderMock.Object, "test", 10, grid, origin, false); - s3Source.UpdateTiles(testTiles); + var t = Task.Run(() => s3Source.UpdateTiles(testTiles)); + t.Wait(); var expectedTiles = isOneXOne ? new Tile[] { testTiles[0], testTiles[2] } : testTiles; Func tileEqualFunc = (tile1, tile2) => tile1?.Z == tile2?.Z && tile1?.X == tile2?.X && tile1?.Y == tile2?.Y; diff --git a/MergerLogicUnitTests/DataTypes/TMSTest.cs b/MergerLogicUnitTests/DataTypes/TMSTest.cs index 2f727ad1..62916fce 100644 --- a/MergerLogicUnitTests/DataTypes/TMSTest.cs +++ b/MergerLogicUnitTests/DataTypes/TMSTest.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; namespace MergerLogicUnitTests.DataTypes { @@ -425,7 +426,7 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) new Tile(2, 2, 3, new byte[] { 2}) }; - Assert.ThrowsException(() => tmsSource.UpdateTiles(testTiles)); + Assert.ThrowsExceptionAsync(async () => await tmsSource.UpdateTiles(testTiles)); this.VerifyAll(); } diff --git a/MergerLogicUnitTests/DataTypes/WMTSTest.cs b/MergerLogicUnitTests/DataTypes/WMTSTest.cs index cc538b10..43d167f0 100644 --- a/MergerLogicUnitTests/DataTypes/WMTSTest.cs +++ b/MergerLogicUnitTests/DataTypes/WMTSTest.cs @@ -426,7 +426,7 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) new Tile(2, 2, 3, new byte[] { 2}) }; - Assert.ThrowsException(() => wmtsSource.UpdateTiles(testTiles)); + Assert.ThrowsExceptionAsync(async () => await wmtsSource.UpdateTiles(testTiles)); this.VerifyAll(); } diff --git a/MergerLogicUnitTests/DataTypes/XYZTest.cs b/MergerLogicUnitTests/DataTypes/XYZTest.cs index 0c87f6cc..cb4f892d 100644 --- a/MergerLogicUnitTests/DataTypes/XYZTest.cs +++ b/MergerLogicUnitTests/DataTypes/XYZTest.cs @@ -426,7 +426,7 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) new Tile(2, 2, 3, new byte[] { 2}) }; - Assert.ThrowsException(() => xyzSource.UpdateTiles(testTiles)); + Assert.ThrowsExceptionAsync(async () => await xyzSource.UpdateTiles(testTiles)); this.VerifyAll(); } diff --git a/MergerService/Dockerfile b/MergerService/Dockerfile index 30c571ac..2f2894a9 100644 --- a/MergerService/Dockerfile +++ b/MergerService/Dockerfile @@ -1,11 +1,12 @@ #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base +ARG VERSION="1.0.0.0" WORKDIR /app EXPOSE 80 FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -ARG VERSION="9.9.9.9" +ARG VERSION="1.0.0.0" WORKDIR /src COPY ["MergerService/MergerService.csproj", "MergerService/"] COPY ["MergerLogic/MergerLogic.csproj", "MergerLogic/"] diff --git a/MergerService/appsettings.Development.json b/MergerService/appsettings.Development.json index 59b46ad1..f29d1b3c 100644 --- a/MergerService/appsettings.Development.json +++ b/MergerService/appsettings.Development.json @@ -1,6 +1,7 @@ { "GENERAL": { "validate": false, + "servicePointManagerDefaultConnectionLimit": 100, "inputPath": "input", "gpkgPath": "gpkg", "filePath": "tiles", diff --git a/MergerService/appsettings.json b/MergerService/appsettings.json index 4b5a21ae..49fb1814 100644 --- a/MergerService/appsettings.json +++ b/MergerService/appsettings.json @@ -1,6 +1,7 @@ { "GENERAL": { "validate": false, + "servicePointManagerDefaultConnectionLimit": 100, "inputPath": "input", "gpkgPath": "gpkg", "filePath": "tiles", diff --git a/MergerService/src/run.cs b/MergerService/src/run.cs index d1a0c623..13247312 100644 --- a/MergerService/src/run.cs +++ b/MergerService/src/run.cs @@ -91,7 +91,7 @@ private List BuildDataList(Source[] paths, int batchSize) { using (this._activitySource.StartActivity("sources parsing")) { - List sources = new List(); + List sources = new List(paths.Length); if (paths.Length != 0) { @@ -106,7 +106,6 @@ private List BuildDataList(Source[] paths, int batchSize) source.Grid, source.Origin)); } } - return sources; } } @@ -132,6 +131,7 @@ public void Start() string methodName = MethodBase.GetCurrentMethod().Name; this._logger.LogDebug($"[{methodName}] Start App"); var pollingTime = this._configurationManager.GetConfiguration("TASK", "pollingTime"); + ServicePointManager.DefaultConnectionLimit = this._configurationManager.GetConfiguration("GENERAL", "servicePointManagerDefaultConnectionLimit"); var taskTypes = BuildTypeList(); if (taskTypes.Count == 0) @@ -229,7 +229,7 @@ public void Start() } } - private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallbackUrl) + private async void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallbackUrl) { string methodName = MethodBase.GetCurrentMethod().Name; this._logger.LogDebug($"[{methodName}] start {task.ToString()}"); @@ -255,11 +255,12 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba TimeSpan ts; bool shouldUpscale = !metadata.IsNewTarget; - Func getTileByCoord = metadata.IsNewTarget + bool isNewTarget = metadata.IsNewTarget; + Func getTileByCoord = isNewTarget ? (_, _) => null : (source, coord) => { - this._logger.LogDebug($"[{methodName}] GetCorrespondingTile start for coord {coord.ToString()}, shouldUpscale: {shouldUpscale}"); + this._logger.LogDebug($"[{methodName}] GetCorrespondingTile start for coord z: {coord.Z}, x: {coord.X}, y: {coord.Y}, shouldUpscale: {shouldUpscale}"); Tile? resultTile = source.GetCorrespondingTile(coord, shouldUpscale); this._logger.LogDebug($"[{methodName}] GetCorrespondingTile finished resultTile={resultTile}"); return resultTile; @@ -302,11 +303,10 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba { continue; } - - this._logger.LogDebug($"[{methodName}] BuildDataList"); + this._logger.LogDebug($"[{methodName}] BuildDataLists"); List sources = this.BuildDataList(metadata.Sources, this._batchSize); IData target = sources[0]; - target.IsNew = metadata.IsNewTarget; + target.IsNew = isNewTarget; // TODO: fix to use inner batch size (add iteration inside loop below) List tiles = new List((int)singleTileBatchCount); @@ -324,7 +324,7 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba Coord coord = new Coord(bounds.Zoom, x, y); // Create tile builder list for current coord for all sources - List correspondingTileBuilders = new List(); + List correspondingTileBuilders = new List(sources.Count); // Add target tile correspondingTileBuilders.Add(() => getTileByCoord(sources[0], coord)); // Add all sources tiles @@ -349,9 +349,8 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba // Show progress every batchSize if (overallTileProgressCount % this._batchSize == 0) { - this._logger.LogDebug( - $"[{methodName}] Job: {task.JobId}, Task: {task.Id}, Tile Count: {overallTileProgressCount} / {totalTileCount}"); - UpdateRelativeProgress(task, overallTileProgressCount, totalTileCount, taskUtils); + this._logger.LogInformation($"[{methodName}] Job: {task.JobId}, Task: {task.Id}, Tile Count: {overallTileProgressCount} / {totalTileCount}"); + UpdateRelativeProgress(task, overallTileProgressCount, totalTileCount, taskUtils); } } } @@ -359,8 +358,8 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba using (this._activitySource.StartActivity("saving tiles")) { - this._logger.LogDebug($"[{methodName}] target UpdateTiles"); - target.UpdateTiles(tiles); + this._logger.LogInformation($"[{methodName}] Total amount of tiles to upload to target source: {tiles.Count}"); + await target.UpdateTiles(tiles); this._logger.LogDebug($"[{methodName}] UpdateRelativeProgress"); UpdateRelativeProgress(task, overallTileProgressCount, totalTileCount, taskUtils); } diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 8a789109..546605ba 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -10,6 +10,7 @@ metadata: name: {{ .Release.Name }}-configmap data: GENERAL__validate: {{ .Values.env.general.validate | quote }} + GENERAL__servicePointManagerDefaultConnectionLimit: {{ .Values.env.general.servicePointManagerDefaultConnectionLimit | quote }} TASK__jobManagerUrl: {{ .Values.rasterCommon.serviceUrls.jobManager | quote }} {{- range $i, $e := .Values.env.TASK.types }} TASK__types__{{ $i }}__jobType: {{ $e.jobType }} diff --git a/helm/values.yaml b/helm/values.yaml index 4295c000..39368777 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -76,6 +76,7 @@ rasterCommon: env: general: validate: true + servicePointManagerDefaultConnectionLimit: 100 batchSize: 1000 TASK: types: @@ -98,7 +99,7 @@ env: url: http://localhost:4317/v1/trace ratio: 0.01 metrics: - enabled: true + enabled: false url: http://localhost:4317/v1/metrics interval: 5000 resources: