From f4e544911b4011ed80bd177bfda588257ba48afd Mon Sep 17 00:00:00 2001 From: asafMasa Date: Sun, 30 Jul 2023 18:26:41 +0300 Subject: [PATCH 1/7] feat: handle async upload --- MergerCli/Dockerfile | 4 +++- MergerLogic/DataTypes/S3.cs | 13 +++++++++---- MergerService/Dockerfile | 4 +++- MergerService/Program.cs | 2 ++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/MergerCli/Dockerfile b/MergerCli/Dockerfile index 62667a82..110bde71 100644 --- a/MergerCli/Dockerfile +++ b/MergerCli/Dockerfile @@ -1,16 +1,18 @@ #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. FROM mcr.microsoft.com/dotnet/runtime:6.0 AS base +ARG VERSION="9.9.9.9" WORKDIR /app FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -ARG VERSION="9.9.9.9" +ARG VERSION WORKDIR /src COPY ["MergerCli/MergerCli.csproj", "MergerCli/"] COPY ["MergerLogic/MergerLogic.csproj", "MergerLogic/"] RUN dotnet restore "MergerCli/MergerCli.csproj" COPY . . WORKDIR "/src/MergerCli" +RUN echo version is $VERSION RUN dotnet build "MergerCli.csproj" -c Release -o /app/build /p:Version=$VERSION FROM build AS publish diff --git a/MergerLogic/DataTypes/S3.cs b/MergerLogic/DataTypes/S3.cs index 18cbefa9..34585812 100644 --- a/MergerLogic/DataTypes/S3.cs +++ b/MergerLogic/DataTypes/S3.cs @@ -198,12 +198,17 @@ public override long TileCount() protected override void InternalUpdateTiles(IEnumerable targetTiles) { - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start"); - foreach (var tile in targetTiles) + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start upload tiles to S3"); + this.ParallelUpdateTiles(targetTiles); + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end upload tiles to S3"); + } + + private async void ParallelUpdateTiles(IEnumerable targetTiles) + { + await Parallel.ForEachAsync(targetTiles, new ParallelOptions { MaxDegreeOfParallelism = 50 }, async (tile, cancellationToken) => { this.Utils.UpdateTile(tile); - } - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end"); + }); } } } diff --git a/MergerService/Dockerfile b/MergerService/Dockerfile index 30c571ac..8053f992 100644 --- a/MergerService/Dockerfile +++ b/MergerService/Dockerfile @@ -1,17 +1,19 @@ #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base +ARG VERSION="9.9.9.9" WORKDIR /app EXPOSE 80 FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -ARG VERSION="9.9.9.9" +ARG VERSION WORKDIR /src COPY ["MergerService/MergerService.csproj", "MergerService/"] COPY ["MergerLogic/MergerLogic.csproj", "MergerLogic/"] RUN dotnet restore "MergerService/MergerService.csproj" COPY . . WORKDIR "/src/MergerService" +RUN echo version is $VERSION RUN dotnet build "MergerService.csproj" -c Release -o /app/build /p:Version=$VERSION FROM build AS publish diff --git a/MergerService/Program.cs b/MergerService/Program.cs index 0a0b4da3..02a76af9 100644 --- a/MergerService/Program.cs +++ b/MergerService/Program.cs @@ -2,6 +2,7 @@ using MergerLogic.Clients; using MergerService.Src; using MergerService.Utils; +using System.Net; var builder = WebApplication.CreateBuilder(args); @@ -24,4 +25,5 @@ app.Run(); }); +ServicePointManager.DefaultConnectionLimit = 100; app.Services.GetRequiredService().Start(); From fa93da974e793835e39d2d238c7968d1fd4db9c2 Mon Sep 17 00:00:00 2001 From: asafMasa Date: Wed, 2 Aug 2023 12:44:05 +0300 Subject: [PATCH 2/7] chore: update logs --- MergerCli/Dockerfile | 1 - MergerCli/appsettings.json | 1 + MergerLogic/DataTypes/Data.cs | 9 +++++---- MergerLogic/DataTypes/S3.cs | 9 ++++++--- MergerService/Dockerfile | 1 - MergerService/Program.cs | 2 -- MergerService/appsettings.Development.json | 1 + MergerService/appsettings.json | 1 + MergerService/src/run.cs | 17 ++++++++++------- helm/templates/configmap.yaml | 1 + helm/values.yaml | 1 + 11 files changed, 26 insertions(+), 18 deletions(-) diff --git a/MergerCli/Dockerfile b/MergerCli/Dockerfile index 110bde71..2408e059 100644 --- a/MergerCli/Dockerfile +++ b/MergerCli/Dockerfile @@ -12,7 +12,6 @@ COPY ["MergerLogic/MergerLogic.csproj", "MergerLogic/"] RUN dotnet restore "MergerCli/MergerCli.csproj" COPY . . WORKDIR "/src/MergerCli" -RUN echo version is $VERSION RUN dotnet build "MergerCli.csproj" -c Release -o /app/build /p:Version=$VERSION FROM build AS publish diff --git a/MergerCli/appsettings.json b/MergerCli/appsettings.json index d3eec2a7..9d7b57bf 100644 --- a/MergerCli/appsettings.json +++ b/MergerCli/appsettings.json @@ -1,6 +1,7 @@ { "GENERAL": { "validate": false, + "defaultConnectionLimit": 100, "resumeOutputFolder": "./", "uploadOnly": false, "parallel": { diff --git a/MergerLogic/DataTypes/Data.cs b/MergerLogic/DataTypes/Data.cs index ac589231..89ef1205 100644 --- a/MergerLogic/DataTypes/Data.cs +++ b/MergerLogic/DataTypes/Data.cs @@ -79,7 +79,7 @@ protected Data(IServiceProvider container, DataType type, string path, int batch this._container = container; var loggerFactory = container.GetRequiredService(); this._logger = loggerFactory.CreateLogger(this.GetType()); - this._logger.LogInformation($"[{methodName}] Ctor started"); + this._logger.LogInformation($"[{methodName}] Data Ctor started"); this.Type = type; this.Path = path; this.BatchSize = batchSize; @@ -139,6 +139,7 @@ protected Data(IServiceProvider container, DataType type, string path, int batch this._logger.LogInformation($"[{methodName}] Initialize Started"); this.Initialize(); + this._logger.LogInformation($"[{methodName}] Initialize ended"); this._logger.LogInformation($"[{methodName}] Checking if exists, {this.Type}: {this.Path}"); bool exists = this.Exists(); @@ -154,7 +155,7 @@ protected Data(IServiceProvider container, DataType type, string path, int batch } this._logger.LogInformation($"[{methodName}] Validate Started"); this.Validate(); - this._logger.LogInformation($"[{methodName}] Ctor Ended"); + this._logger.LogInformation($"[{methodName}] Data Ctor Ended"); } protected virtual void Initialize() @@ -256,14 +257,14 @@ public bool TileExists(Coord coord) public Tile? GetCorrespondingTile(Coord coords, bool upscale) { - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start for coord: {coords.ToString()}, upscale: {upscale}"); + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start for coord: z: {coords.Z}, x: {coords.X}, y: {coords.Y}, upscale: {upscale}"); Tile? correspondingTile = this.GetTile(coords.Z, coords.X, coords.Y); if (upscale && correspondingTile == null) { correspondingTile = this.GetLastExistingTile(coords); } - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end for coord: {coords.ToString()}, upscale: {upscale}"); + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end for coord: z: {coords.Z}, x: {coords.X}, y: {coords.Y}, upscale: {upscale}"); return correspondingTile; } diff --git a/MergerLogic/DataTypes/S3.cs b/MergerLogic/DataTypes/S3.cs index 34585812..05ff960c 100644 --- a/MergerLogic/DataTypes/S3.cs +++ b/MergerLogic/DataTypes/S3.cs @@ -68,10 +68,12 @@ public override void Reset() private List GetZoomLevels() { this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start"); - List zoomLevels = new List(); + List zoomLevels = new List(Data.MaxZoomRead); for (int zoomLevel = 0; zoomLevel < Data.MaxZoomRead; zoomLevel++) { + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] Check if zoom {zoomLevel} exists"); + // TODO: need to check if can be updated dynamically if (this.FolderExists($"{zoomLevel}/")) { zoomLevels.Add(zoomLevel); @@ -158,8 +160,9 @@ private bool FolderExists(string directory) }; var task = this._client.ListObjectsV2Async(listRequests); var response = task.Result; - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end"); - return response.KeyCount > 0; + bool isExists = response.KeyCount > 0; + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] directory: {directory}, isExists={isExists}"); + return isExists; } public override bool Exists() diff --git a/MergerService/Dockerfile b/MergerService/Dockerfile index 8053f992..ee672042 100644 --- a/MergerService/Dockerfile +++ b/MergerService/Dockerfile @@ -13,7 +13,6 @@ COPY ["MergerLogic/MergerLogic.csproj", "MergerLogic/"] RUN dotnet restore "MergerService/MergerService.csproj" COPY . . WORKDIR "/src/MergerService" -RUN echo version is $VERSION RUN dotnet build "MergerService.csproj" -c Release -o /app/build /p:Version=$VERSION FROM build AS publish diff --git a/MergerService/Program.cs b/MergerService/Program.cs index 02a76af9..b5034c63 100644 --- a/MergerService/Program.cs +++ b/MergerService/Program.cs @@ -24,6 +24,4 @@ { app.Run(); }); - -ServicePointManager.DefaultConnectionLimit = 100; app.Services.GetRequiredService().Start(); diff --git a/MergerService/appsettings.Development.json b/MergerService/appsettings.Development.json index 59b46ad1..da245e51 100644 --- a/MergerService/appsettings.Development.json +++ b/MergerService/appsettings.Development.json @@ -1,6 +1,7 @@ { "GENERAL": { "validate": false, + "defaultConnectionLimit": 100, "inputPath": "input", "gpkgPath": "gpkg", "filePath": "tiles", diff --git a/MergerService/appsettings.json b/MergerService/appsettings.json index 4b5a21ae..286ace4f 100644 --- a/MergerService/appsettings.json +++ b/MergerService/appsettings.json @@ -1,6 +1,7 @@ { "GENERAL": { "validate": false, + "defaultConnectionLimit": 100, "inputPath": "input", "gpkgPath": "gpkg", "filePath": "tiles", diff --git a/MergerService/src/run.cs b/MergerService/src/run.cs index d1a0c623..740b97e3 100644 --- a/MergerService/src/run.cs +++ b/MergerService/src/run.cs @@ -89,9 +89,11 @@ private string BuildPath(Source source, bool isTarget) private List BuildDataList(Source[] paths, int batchSize) { + string methodName = MethodBase.GetCurrentMethod().Name; using (this._activitySource.StartActivity("sources parsing")) { - List sources = new List(); + this._logger.LogDebug($"[{methodName}] start"); + List sources = new List(paths.Length); if (paths.Length != 0) { @@ -106,7 +108,7 @@ private List BuildDataList(Source[] paths, int batchSize) source.Grid, source.Origin)); } } - + this._logger.LogDebug($"[{methodName}] end"); return sources; } } @@ -132,6 +134,7 @@ public void Start() string methodName = MethodBase.GetCurrentMethod().Name; this._logger.LogDebug($"[{methodName}] Start App"); var pollingTime = this._configurationManager.GetConfiguration("TASK", "pollingTime"); + ServicePointManager.DefaultConnectionLimit = this._configurationManager.GetConfiguration("GENERAL", "defaultConnectionLimit"); var taskTypes = BuildTypeList(); if (taskTypes.Count == 0) @@ -255,11 +258,12 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba TimeSpan ts; bool shouldUpscale = !metadata.IsNewTarget; - Func getTileByCoord = metadata.IsNewTarget + bool isNewTarget = metadata.IsNewTarget; + Func getTileByCoord = isNewTarget ? (_, _) => null : (source, coord) => { - this._logger.LogDebug($"[{methodName}] GetCorrespondingTile start for coord {coord.ToString()}, shouldUpscale: {shouldUpscale}"); + this._logger.LogDebug($"[{methodName}] GetCorrespondingTile start for coord z: {coord.Z}, x: {coord.X}, y: {coord.Y}, shouldUpscale: {shouldUpscale}"); Tile? resultTile = source.GetCorrespondingTile(coord, shouldUpscale); this._logger.LogDebug($"[{methodName}] GetCorrespondingTile finished resultTile={resultTile}"); return resultTile; @@ -303,10 +307,9 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba continue; } - this._logger.LogDebug($"[{methodName}] BuildDataList"); List sources = this.BuildDataList(metadata.Sources, this._batchSize); IData target = sources[0]; - target.IsNew = metadata.IsNewTarget; + target.IsNew = isNewTarget; // TODO: fix to use inner batch size (add iteration inside loop below) List tiles = new List((int)singleTileBatchCount); @@ -324,7 +327,7 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba Coord coord = new Coord(bounds.Zoom, x, y); // Create tile builder list for current coord for all sources - List correspondingTileBuilders = new List(); + List correspondingTileBuilders = new List(sources.Count); // Add target tile correspondingTileBuilders.Add(() => getTileByCoord(sources[0], coord)); // Add all sources tiles diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 8a789109..b0dd6aff 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -10,6 +10,7 @@ metadata: name: {{ .Release.Name }}-configmap data: GENERAL__validate: {{ .Values.env.general.validate | quote }} + GENERAL__defaultConnectionLimit: {{ .Values.env.general.defaultConnectionLimit | quote }} TASK__jobManagerUrl: {{ .Values.rasterCommon.serviceUrls.jobManager | quote }} {{- range $i, $e := .Values.env.TASK.types }} TASK__types__{{ $i }}__jobType: {{ $e.jobType }} diff --git a/helm/values.yaml b/helm/values.yaml index 38b6645e..b592999b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -76,6 +76,7 @@ rasterCommon: env: general: validate: true + defaultConnectionLimit: 100 batchSize: 1000 TASK: types: From 8993426544bcc8aae7f8e93410c48c1781c1c3f9 Mon Sep 17 00:00:00 2001 From: asafMasa Date: Wed, 2 Aug 2023 12:46:47 +0300 Subject: [PATCH 3/7] chore: update logs --- MergerService/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MergerService/Program.cs b/MergerService/Program.cs index b5034c63..0a0b4da3 100644 --- a/MergerService/Program.cs +++ b/MergerService/Program.cs @@ -2,7 +2,6 @@ using MergerLogic.Clients; using MergerService.Src; using MergerService.Utils; -using System.Net; var builder = WebApplication.CreateBuilder(args); @@ -24,4 +23,5 @@ { app.Run(); }); + app.Services.GetRequiredService().Start(); From e73aecdbdd7799db70874dbe99d982c817737172 Mon Sep 17 00:00:00 2001 From: asafMasa Date: Wed, 2 Aug 2023 13:06:22 +0300 Subject: [PATCH 4/7] chore: update logs --- MergerCli/Dockerfile | 2 +- MergerService/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MergerCli/Dockerfile b/MergerCli/Dockerfile index 2408e059..15883955 100644 --- a/MergerCli/Dockerfile +++ b/MergerCli/Dockerfile @@ -5,7 +5,7 @@ ARG VERSION="9.9.9.9" WORKDIR /app FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -ARG VERSION +ARG VERSION="9.9.9.9" WORKDIR /src COPY ["MergerCli/MergerCli.csproj", "MergerCli/"] COPY ["MergerLogic/MergerLogic.csproj", "MergerLogic/"] diff --git a/MergerService/Dockerfile b/MergerService/Dockerfile index ee672042..84acaeeb 100644 --- a/MergerService/Dockerfile +++ b/MergerService/Dockerfile @@ -6,7 +6,7 @@ WORKDIR /app EXPOSE 80 FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -ARG VERSION +ARG VERSION="9.9.9.9" WORKDIR /src COPY ["MergerService/MergerService.csproj", "MergerService/"] COPY ["MergerLogic/MergerLogic.csproj", "MergerLogic/"] From 39925812d0bb52359a1a892b8ed47864d448acd2 Mon Sep 17 00:00:00 2001 From: asafMasa Date: Wed, 2 Aug 2023 17:33:41 +0300 Subject: [PATCH 5/7] chore: update logs --- MergerLogic/DataTypes/Data.cs | 1 - MergerLogic/DataTypes/S3.cs | 4 ++-- MergerService/src/run.cs | 7 +++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/MergerLogic/DataTypes/Data.cs b/MergerLogic/DataTypes/Data.cs index 89ef1205..d67c4585 100644 --- a/MergerLogic/DataTypes/Data.cs +++ b/MergerLogic/DataTypes/Data.cs @@ -153,7 +153,6 @@ protected Data(IServiceProvider container, DataType type, string path, int batch throw new Exception($"{this.Type} source {path} does not exist."); } } - this._logger.LogInformation($"[{methodName}] Validate Started"); this.Validate(); this._logger.LogInformation($"[{methodName}] Data Ctor Ended"); } diff --git a/MergerLogic/DataTypes/S3.cs b/MergerLogic/DataTypes/S3.cs index 05ff960c..54ed6f42 100644 --- a/MergerLogic/DataTypes/S3.cs +++ b/MergerLogic/DataTypes/S3.cs @@ -167,9 +167,9 @@ private bool FolderExists(string directory) public override bool Exists() { - this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] bucket: {this._bucket}, path: {this.Path}"); + this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] Checking if exists, bucket: {this._bucket}, path: {this.Path}"); bool exists = FolderExists(""); - this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] ended"); + this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] exists={exists} ended"); return exists; } diff --git a/MergerService/src/run.cs b/MergerService/src/run.cs index 740b97e3..d20691a3 100644 --- a/MergerService/src/run.cs +++ b/MergerService/src/run.cs @@ -352,9 +352,8 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba // Show progress every batchSize if (overallTileProgressCount % this._batchSize == 0) { - this._logger.LogDebug( - $"[{methodName}] Job: {task.JobId}, Task: {task.Id}, Tile Count: {overallTileProgressCount} / {totalTileCount}"); - UpdateRelativeProgress(task, overallTileProgressCount, totalTileCount, taskUtils); + this._logger.LogInformation($"[{methodName}] Job: {task.JobId}, Task: {task.Id}, Tile Count: {overallTileProgressCount} / {totalTileCount}"); + UpdateRelativeProgress(task, overallTileProgressCount, totalTileCount, taskUtils); } } } @@ -362,7 +361,7 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba using (this._activitySource.StartActivity("saving tiles")) { - this._logger.LogDebug($"[{methodName}] target UpdateTiles"); + this._logger.LogInformation($"[{methodName}] Total amount of tiles to upload to target source: {tiles.Count}"); target.UpdateTiles(tiles); this._logger.LogDebug($"[{methodName}] UpdateRelativeProgress"); UpdateRelativeProgress(task, overallTileProgressCount, totalTileCount, taskUtils); From 4e2adb819ae6cfdbbabdd33722ca462245b4d23e Mon Sep 17 00:00:00 2001 From: Asaf Masa Date: Sun, 6 Aug 2023 17:22:32 +0300 Subject: [PATCH 6/7] chore: fix code review comments --- MergerCli/Dockerfile | 4 ++-- MergerCli/appsettings.json | 2 +- MergerLogic/Clients/S3Client.cs | 19 ++++++++----------- MergerLogic/DataTypes/S3.cs | 4 +--- .../OpenTelemetryFormattedConsoleExporter.cs | 9 ++++++++- MergerService/Dockerfile | 4 ++-- MergerService/appsettings.Development.json | 2 +- MergerService/appsettings.json | 2 +- MergerService/src/run.cs | 7 ++----- helm/templates/configmap.yaml | 2 +- helm/values.yaml | 2 +- 11 files changed, 28 insertions(+), 29 deletions(-) diff --git a/MergerCli/Dockerfile b/MergerCli/Dockerfile index 15883955..0a4cd407 100644 --- a/MergerCli/Dockerfile +++ b/MergerCli/Dockerfile @@ -1,11 +1,11 @@ #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. FROM mcr.microsoft.com/dotnet/runtime:6.0 AS base -ARG VERSION="9.9.9.9" +ARG VERSION="1.0.0.0" WORKDIR /app FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -ARG VERSION="9.9.9.9" +ARG VERSION="1.0.0.0" WORKDIR /src COPY ["MergerCli/MergerCli.csproj", "MergerCli/"] COPY ["MergerLogic/MergerLogic.csproj", "MergerLogic/"] diff --git a/MergerCli/appsettings.json b/MergerCli/appsettings.json index 9d7b57bf..272f6088 100644 --- a/MergerCli/appsettings.json +++ b/MergerCli/appsettings.json @@ -1,7 +1,7 @@ { "GENERAL": { "validate": false, - "defaultConnectionLimit": 100, + "servicePointManagerDefaultConnectionLimit": 100, "resumeOutputFolder": "./", "uploadOnly": false, "parallel": { diff --git a/MergerLogic/Clients/S3Client.cs b/MergerLogic/Clients/S3Client.cs index c7df9d96..a1202bf5 100644 --- a/MergerLogic/Clients/S3Client.cs +++ b/MergerLogic/Clients/S3Client.cs @@ -31,12 +31,10 @@ public S3Client(IAmazonS3 client, IPathUtils pathUtils, IGeoUtils geoUtils, IIma string methodName = MethodBase.GetCurrentMethod().Name; try { - this._logger.LogDebug($"[{methodName}] start, key {key}"); + this._logger.LogDebug($"[{methodName}] start, BucketName: {this._bucket}, Key: {key}"); var request = new GetObjectRequest() { BucketName = this._bucket, Key = key }; - this._logger.LogDebug($"[{methodName}] start GetObjectAsync, BucketName: {request.BucketName}, Key: {request.Key}"); var getObjectTask = this._client.GetObjectAsync(request); GetObjectResponse res = getObjectTask.Result; - this._logger.LogDebug($"[{methodName}] requested key {key} recieved"); byte[] image; using (MemoryStream ms = new MemoryStream()) @@ -48,12 +46,12 @@ public S3Client(IAmazonS3 client, IPathUtils pathUtils, IGeoUtils geoUtils, IIma image = ms.ToArray(); } - this._logger.LogDebug($"[{methodName}] end, key {key}"); + this._logger.LogDebug($"[{methodName}] end, BucketName: {this._bucket}, Key: {key}"); return image; } catch (AggregateException e) { - this._logger.LogDebug($"[{methodName}] exception, Message: {e.Message}"); + this._logger.LogError($"[{methodName}] exception, Message: {e.Message}, returned null"); return null; } } @@ -65,6 +63,7 @@ public S3Client(IAmazonS3 client, IPathUtils pathUtils, IGeoUtils geoUtils, IIma var key = this.GetTileKey(z, x, y); if (key == null) { + this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y} - no tile was found"); return null; } @@ -99,7 +98,7 @@ public override bool TileExists(int z, int x, int y) public void UpdateTile(Tile tile) { string methodName = MethodBase.GetCurrentMethod().Name; - this._logger.LogDebug($"[{methodName}] start {tile.ToString()}"); + this._logger.LogDebug($"[{methodName}] start {tile.ToString()} to BucketName: {this._bucket}"); string key = this._pathUtils.GetTilePath(this.path, tile, true); var request = new PutObjectRequest() @@ -111,23 +110,21 @@ public void UpdateTile(Tile tile) using (var ms = new MemoryStream(buffer)) { request.InputStream = ms; - this._logger.LogDebug($"[{methodName}] start PutObjectAsync BucketName: {request.BucketName}, Key: {request.Key}"); var task = this._client.PutObjectAsync(request); var res = task.Result; } - this._logger.LogDebug($"[{methodName}] end {tile.ToString()}"); + this._logger.LogDebug($"[{methodName}] end {tile.ToString()} to BucketName: {this._bucket}"); } private string? GetTileKey(int z, int x, int y) { string methodName = MethodBase.GetCurrentMethod().Name; - this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y}"); + this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y} from BucketName: {this._bucket}"); string keyPrefix = this._pathUtils.GetTilePathWithoutExtension(this.path, z, x, y, true); var listRequests = new ListObjectsV2Request { BucketName = this._bucket, Prefix = keyPrefix, MaxKeys = 1 }; - this._logger.LogDebug($"[{methodName}] start ListObjectsV2Async BucketName: {listRequests.BucketName}, Prefix: {listRequests.Prefix}"); var listObjectsTask = this._client.ListObjectsV2Async(listRequests); string? result = listObjectsTask.Result.S3Objects.FirstOrDefault()?.Key; - this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y}"); + this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y} from BucketName: {this._bucket}"); return result; } } diff --git a/MergerLogic/DataTypes/S3.cs b/MergerLogic/DataTypes/S3.cs index 54ed6f42..05f3e164 100644 --- a/MergerLogic/DataTypes/S3.cs +++ b/MergerLogic/DataTypes/S3.cs @@ -67,7 +67,6 @@ public override void Reset() private List GetZoomLevels() { - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start"); List zoomLevels = new List(Data.MaxZoomRead); for (int zoomLevel = 0; zoomLevel < Data.MaxZoomRead; zoomLevel++) @@ -79,7 +78,6 @@ private List GetZoomLevels() zoomLevels.Add(zoomLevel); } } - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] ended"); return zoomLevels; } @@ -208,7 +206,7 @@ protected override void InternalUpdateTiles(IEnumerable targetTiles) private async void ParallelUpdateTiles(IEnumerable targetTiles) { - await Parallel.ForEachAsync(targetTiles, new ParallelOptions { MaxDegreeOfParallelism = 50 }, async (tile, cancellationToken) => + await Parallel.ForEachAsync(targetTiles, new ParallelOptions { MaxDegreeOfParallelism = -1 }, async (tile, cancellationToken) => { this.Utils.UpdateTile(tile); }); diff --git a/MergerLogic/Monitoring/OpenTelemetryFormattedConsoleExporter.cs b/MergerLogic/Monitoring/OpenTelemetryFormattedConsoleExporter.cs index 2a0e71e4..e2f34274 100644 --- a/MergerLogic/Monitoring/OpenTelemetryFormattedConsoleExporter.cs +++ b/MergerLogic/Monitoring/OpenTelemetryFormattedConsoleExporter.cs @@ -1,6 +1,7 @@ using OpenTelemetry; using OpenTelemetry.Exporter; using OpenTelemetry.Logs; +using System.Net; namespace MergerLogic.Monitoring { @@ -8,6 +9,7 @@ public class OpenTelemetryFormattedConsoleExporter : ConsoleExporter { private const string SERVICE_NAME_ATTRIBUTE = "service.name"; private const string SERVICE_VERSION_ATTRIBUTE = "service.version"; + private const string SERVICE_HOST_NAME_ATTRIBUTE = "service.host.name"; public OpenTelemetryFormattedConsoleExporter(ConsoleExporterOptions options) : base(options) @@ -29,8 +31,13 @@ private string MCTextFormat(LogRecord record) var resource = this.ParseResource(); var serviceName = this.GetResourceAttribute(resource, SERVICE_NAME_ATTRIBUTE, "unknown_service"); var serviceVersion = this.GetResourceAttribute(resource, SERVICE_VERSION_ATTRIBUTE, "unknown_version"); + if (!resource.ContainsKey(SERVICE_HOST_NAME_ATTRIBUTE)) + { + resource.Add(SERVICE_HOST_NAME_ATTRIBUTE, Dns.GetHostName()); + } + var serviceHostName = this.GetResourceAttribute(resource, SERVICE_HOST_NAME_ATTRIBUTE, "unknown_host_name"); - return $"[{this.FormatTime(record.Timestamp)}] [{record.LogLevel}] [{serviceName}] [{serviceVersion}] [{record.CategoryName}] {record.State}"; + return $"[{this.FormatTime(record.Timestamp)}] [{record.LogLevel}] [{serviceHostName}] [{serviceName}] [{serviceVersion}] [{Environment.CurrentManagedThreadId}] [{record.CategoryName}] {record.State}"; } private string FormatTime(DateTime time) diff --git a/MergerService/Dockerfile b/MergerService/Dockerfile index 84acaeeb..2f2894a9 100644 --- a/MergerService/Dockerfile +++ b/MergerService/Dockerfile @@ -1,12 +1,12 @@ #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base -ARG VERSION="9.9.9.9" +ARG VERSION="1.0.0.0" WORKDIR /app EXPOSE 80 FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -ARG VERSION="9.9.9.9" +ARG VERSION="1.0.0.0" WORKDIR /src COPY ["MergerService/MergerService.csproj", "MergerService/"] COPY ["MergerLogic/MergerLogic.csproj", "MergerLogic/"] diff --git a/MergerService/appsettings.Development.json b/MergerService/appsettings.Development.json index da245e51..f29d1b3c 100644 --- a/MergerService/appsettings.Development.json +++ b/MergerService/appsettings.Development.json @@ -1,7 +1,7 @@ { "GENERAL": { "validate": false, - "defaultConnectionLimit": 100, + "servicePointManagerDefaultConnectionLimit": 100, "inputPath": "input", "gpkgPath": "gpkg", "filePath": "tiles", diff --git a/MergerService/appsettings.json b/MergerService/appsettings.json index 286ace4f..49fb1814 100644 --- a/MergerService/appsettings.json +++ b/MergerService/appsettings.json @@ -1,7 +1,7 @@ { "GENERAL": { "validate": false, - "defaultConnectionLimit": 100, + "servicePointManagerDefaultConnectionLimit": 100, "inputPath": "input", "gpkgPath": "gpkg", "filePath": "tiles", diff --git a/MergerService/src/run.cs b/MergerService/src/run.cs index d20691a3..61d6d2ed 100644 --- a/MergerService/src/run.cs +++ b/MergerService/src/run.cs @@ -89,10 +89,8 @@ private string BuildPath(Source source, bool isTarget) private List BuildDataList(Source[] paths, int batchSize) { - string methodName = MethodBase.GetCurrentMethod().Name; using (this._activitySource.StartActivity("sources parsing")) { - this._logger.LogDebug($"[{methodName}] start"); List sources = new List(paths.Length); if (paths.Length != 0) @@ -108,7 +106,6 @@ private List BuildDataList(Source[] paths, int batchSize) source.Grid, source.Origin)); } } - this._logger.LogDebug($"[{methodName}] end"); return sources; } } @@ -134,7 +131,7 @@ public void Start() string methodName = MethodBase.GetCurrentMethod().Name; this._logger.LogDebug($"[{methodName}] Start App"); var pollingTime = this._configurationManager.GetConfiguration("TASK", "pollingTime"); - ServicePointManager.DefaultConnectionLimit = this._configurationManager.GetConfiguration("GENERAL", "defaultConnectionLimit"); + ServicePointManager.DefaultConnectionLimit = this._configurationManager.GetConfiguration("GENERAL", "servicePointManagerDefaultConnectionLimit"); var taskTypes = BuildTypeList(); if (taskTypes.Count == 0) @@ -306,7 +303,7 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba { continue; } - + this._logger.LogDebug($"[{methodName}] BuildDataLists"); List sources = this.BuildDataList(metadata.Sources, this._batchSize); IData target = sources[0]; target.IsNew = isNewTarget; diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index b0dd6aff..546605ba 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -10,7 +10,7 @@ metadata: name: {{ .Release.Name }}-configmap data: GENERAL__validate: {{ .Values.env.general.validate | quote }} - GENERAL__defaultConnectionLimit: {{ .Values.env.general.defaultConnectionLimit | quote }} + GENERAL__servicePointManagerDefaultConnectionLimit: {{ .Values.env.general.servicePointManagerDefaultConnectionLimit | quote }} TASK__jobManagerUrl: {{ .Values.rasterCommon.serviceUrls.jobManager | quote }} {{- range $i, $e := .Values.env.TASK.types }} TASK__types__{{ $i }}__jobType: {{ $e.jobType }} diff --git a/helm/values.yaml b/helm/values.yaml index b592999b..39368777 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -76,7 +76,7 @@ rasterCommon: env: general: validate: true - defaultConnectionLimit: 100 + servicePointManagerDefaultConnectionLimit: 100 batchSize: 1000 TASK: types: From af05f4b086d72faf561b17b4d9b1e8154659a55d Mon Sep 17 00:00:00 2001 From: Asaf Masa Date: Mon, 7 Aug 2023 17:42:13 +0300 Subject: [PATCH 7/7] chore: fix tests for async --- MergerCli/Process.cs | 3 ++- MergerLogic/DataTypes/Data.cs | 6 +++--- MergerLogic/DataTypes/Fs.cs | 3 ++- MergerLogic/DataTypes/Gpkg.cs | 3 ++- MergerLogic/DataTypes/HttpDataSource.cs | 2 +- MergerLogic/DataTypes/IData.cs | 2 +- MergerLogic/DataTypes/S3.cs | 11 +++-------- MergerLogicUnitTests/DataTypes/FSTest.cs | 4 +++- MergerLogicUnitTests/DataTypes/GpkgTest.cs | 4 +++- MergerLogicUnitTests/DataTypes/S3Test.cs | 11 +++++------ MergerLogicUnitTests/DataTypes/TMSTest.cs | 3 ++- MergerLogicUnitTests/DataTypes/WMTSTest.cs | 2 +- MergerLogicUnitTests/DataTypes/XYZTest.cs | 2 +- MergerService/src/run.cs | 4 ++-- 14 files changed, 31 insertions(+), 29 deletions(-) diff --git a/MergerCli/Process.cs b/MergerCli/Process.cs index 2e778cf5..d63c0d51 100644 --- a/MergerCli/Process.cs +++ b/MergerCli/Process.cs @@ -125,7 +125,8 @@ private void ProcessBatch(TileFormat targetFormat, IData baseData, List ne } } - baseData.UpdateTiles(tiles); + var t = Task.Run(() => baseData.UpdateTiles(tiles)); + t.Wait(); Interlocked.Add(ref tileProgressCount, tiles.Count); this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] Tile Count: {tileProgressCount} / {totalTileCount}"); diff --git a/MergerLogic/DataTypes/Data.cs b/MergerLogic/DataTypes/Data.cs index d67c4585..c4197829 100644 --- a/MergerLogic/DataTypes/Data.cs +++ b/MergerLogic/DataTypes/Data.cs @@ -267,7 +267,7 @@ public bool TileExists(Coord coord) return correspondingTile; } - public void UpdateTiles(IEnumerable tiles) + public async Task UpdateTiles(IEnumerable tiles) { this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] update tiles started"); var targetTiles = tiles.Select(tile => @@ -276,11 +276,11 @@ public void UpdateTiles(IEnumerable tiles) Tile? targetTile = this.FromCurrentGridTile(convertedTile); return targetTile; }).Where(tile => tile is not null); - this.InternalUpdateTiles(targetTiles); + await this.InternalUpdateTilesAsync(targetTiles); this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] update tiles ended"); } - protected abstract void InternalUpdateTiles(IEnumerable targetTiles); + protected abstract Task InternalUpdateTilesAsync(IEnumerable targetTiles); public virtual void Wrapup() { diff --git a/MergerLogic/DataTypes/Fs.cs b/MergerLogic/DataTypes/Fs.cs index d0fc512c..c1aa5c82 100644 --- a/MergerLogic/DataTypes/Fs.cs +++ b/MergerLogic/DataTypes/Fs.cs @@ -174,7 +174,7 @@ public override long TileCount() return count; } - protected override void InternalUpdateTiles(IEnumerable targetTiles) + protected override Task InternalUpdateTilesAsync(IEnumerable targetTiles) { foreach (Tile tile in targetTiles) { @@ -190,6 +190,7 @@ protected override void InternalUpdateTiles(IEnumerable targetTiles) } } } + return Task.CompletedTask; } } } diff --git a/MergerLogic/DataTypes/Gpkg.cs b/MergerLogic/DataTypes/Gpkg.cs index 2923e455..3cbf81ec 100644 --- a/MergerLogic/DataTypes/Gpkg.cs +++ b/MergerLogic/DataTypes/Gpkg.cs @@ -189,11 +189,12 @@ public override long TileCount() return result; } - protected override void InternalUpdateTiles(IEnumerable targetTiles) + protected override Task InternalUpdateTilesAsync(IEnumerable targetTiles) { this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] started"); this.Utils.InsertTiles(targetTiles); this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] ended"); + return Task.CompletedTask; } } } diff --git a/MergerLogic/DataTypes/HttpDataSource.cs b/MergerLogic/DataTypes/HttpDataSource.cs index 17a1ce80..061d0974 100644 --- a/MergerLogic/DataTypes/HttpDataSource.cs +++ b/MergerLogic/DataTypes/HttpDataSource.cs @@ -84,7 +84,7 @@ protected IEnumerable GetTiles() } } - protected override void InternalUpdateTiles(IEnumerable targetTiles) + protected override Task InternalUpdateTilesAsync(IEnumerable targetTiles) { throw new NotImplementedException(); } diff --git a/MergerLogic/DataTypes/IData.cs b/MergerLogic/DataTypes/IData.cs index c8208823..fbc36d81 100644 --- a/MergerLogic/DataTypes/IData.cs +++ b/MergerLogic/DataTypes/IData.cs @@ -16,7 +16,7 @@ public interface IData long TileCount(); bool TileExists(Coord coord); bool TileExists(Tile tile); - void UpdateTiles(IEnumerable tiles); + Task UpdateTiles(IEnumerable tiles); void Wrapup(); } } diff --git a/MergerLogic/DataTypes/S3.cs b/MergerLogic/DataTypes/S3.cs index 05f3e164..9a3540cb 100644 --- a/MergerLogic/DataTypes/S3.cs +++ b/MergerLogic/DataTypes/S3.cs @@ -197,19 +197,14 @@ public override long TileCount() return tileCount; } - protected override void InternalUpdateTiles(IEnumerable targetTiles) + protected override async Task InternalUpdateTilesAsync(IEnumerable targetTiles) { this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] start upload tiles to S3"); - this.ParallelUpdateTiles(targetTiles); - this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end upload tiles to S3"); - } - - private async void ParallelUpdateTiles(IEnumerable targetTiles) - { await Parallel.ForEachAsync(targetTiles, new ParallelOptions { MaxDegreeOfParallelism = -1 }, async (tile, cancellationToken) => { - this.Utils.UpdateTile(tile); + await Task.Run(() => this.Utils.UpdateTile(tile), cancellationToken); }); + this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] end upload tiles to S3"); } } } diff --git a/MergerLogicUnitTests/DataTypes/FSTest.cs b/MergerLogicUnitTests/DataTypes/FSTest.cs index 82481e96..e4d41afa 100644 --- a/MergerLogicUnitTests/DataTypes/FSTest.cs +++ b/MergerLogicUnitTests/DataTypes/FSTest.cs @@ -11,6 +11,7 @@ using System.IO; using System.IO.Abstractions; using System.Linq; +using System.Threading.Tasks; namespace MergerLogicUnitTests.DataTypes { @@ -481,7 +482,8 @@ public void UpdateTiles(bool isBase, bool isOneXOne, GridOrigin origin) fileMock.InSequence(seq).Setup(f => f.OpenWrite()).Returns(fileStream); } - fsSource.UpdateTiles(testTiles); + var t = Task.Run(() => fsSource.UpdateTiles(testTiles)); + t.Wait(); CollectionAssert.AreEqual(new byte[] { 0 }, fileStreams[0].ToArray()); if (isOneXOne) diff --git a/MergerLogicUnitTests/DataTypes/GpkgTest.cs b/MergerLogicUnitTests/DataTypes/GpkgTest.cs index af06c58a..c200d6e2 100644 --- a/MergerLogicUnitTests/DataTypes/GpkgTest.cs +++ b/MergerLogicUnitTests/DataTypes/GpkgTest.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; namespace MergerLogicUnitTests.DataTypes { @@ -455,7 +456,8 @@ public void UpdateTiles(bool isBase, bool isOneXOne, GridOrigin origin) new Tile(1, 2, 3, new byte[] { }), new Tile(7, 7, 7, new byte[] { }), new Tile(2, 2, 3, new byte[] { }) }; - gpkg.UpdateTiles(testTiles); + var t = Task.Run(() => gpkg.UpdateTiles(testTiles)); + t.Wait(); var expectedTiles = isOneXOne ? new Tile[] { testTiles[0], testTiles[2] } : testTiles; var tileComparer = EqualityComparerFactory.Create((tile1, tile2) => diff --git a/MergerLogicUnitTests/DataTypes/S3Test.cs b/MergerLogicUnitTests/DataTypes/S3Test.cs index 1a6bcd0c..81714e45 100644 --- a/MergerLogicUnitTests/DataTypes/S3Test.cs +++ b/MergerLogicUnitTests/DataTypes/S3Test.cs @@ -5,13 +5,13 @@ using MergerLogic.DataTypes; using MergerLogic.Utils; using MergerLogicUnitTests.testUtils; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; using System.Collections.Generic; using System.Threading; +using System.Threading.Tasks; namespace MergerLogicUnitTests.DataTypes { @@ -438,7 +438,8 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) { var testTiles = new Tile[] { - new Tile(1, 2, 3, new byte[] { }), new Tile(7, 7, 7, new byte[] { }), + new Tile(1, 2, 3, new byte[] { }), + new Tile(7, 7, 7, new byte[] { }), new Tile(2, 2, 3, new byte[] { }) }; var seq = new MockSequence(); @@ -449,7 +450,6 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) if (origin == GridOrigin.UPPER_LEFT) { this._geoUtilsMock - .InSequence(seq) .Setup(utils => utils.FlipY(It.IsAny())) .Returns(t => t.Y); } @@ -457,7 +457,6 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) if (isOneXOne) { this._oneXOneConvertorMock - .InSequence(seq) .Setup(converter => converter.TryFromTwoXOne(It.IsAny())) .Returns(tile => tile.Z != 7 ? tile : null); } @@ -465,7 +464,6 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) if (!isOneXOne || tile.Z != 7) { this._s3UtilsMock - .InSequence(seq) .Setup(utils => utils.UpdateTile(It.IsAny())); } } @@ -473,7 +471,8 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) Grid grid = isOneXOne ? Grid.OneXOne : Grid.TwoXOne; var s3Source = new S3(this._pathUtilsMock.Object, this._serviceProviderMock.Object, "test", 10, grid, origin, false); - s3Source.UpdateTiles(testTiles); + var t = Task.Run(() => s3Source.UpdateTiles(testTiles)); + t.Wait(); var expectedTiles = isOneXOne ? new Tile[] { testTiles[0], testTiles[2] } : testTiles; Func tileEqualFunc = (tile1, tile2) => tile1?.Z == tile2?.Z && tile1?.X == tile2?.X && tile1?.Y == tile2?.Y; diff --git a/MergerLogicUnitTests/DataTypes/TMSTest.cs b/MergerLogicUnitTests/DataTypes/TMSTest.cs index 2f727ad1..62916fce 100644 --- a/MergerLogicUnitTests/DataTypes/TMSTest.cs +++ b/MergerLogicUnitTests/DataTypes/TMSTest.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; namespace MergerLogicUnitTests.DataTypes { @@ -425,7 +426,7 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) new Tile(2, 2, 3, new byte[] { 2}) }; - Assert.ThrowsException(() => tmsSource.UpdateTiles(testTiles)); + Assert.ThrowsExceptionAsync(async () => await tmsSource.UpdateTiles(testTiles)); this.VerifyAll(); } diff --git a/MergerLogicUnitTests/DataTypes/WMTSTest.cs b/MergerLogicUnitTests/DataTypes/WMTSTest.cs index cc538b10..43d167f0 100644 --- a/MergerLogicUnitTests/DataTypes/WMTSTest.cs +++ b/MergerLogicUnitTests/DataTypes/WMTSTest.cs @@ -426,7 +426,7 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) new Tile(2, 2, 3, new byte[] { 2}) }; - Assert.ThrowsException(() => wmtsSource.UpdateTiles(testTiles)); + Assert.ThrowsExceptionAsync(async () => await wmtsSource.UpdateTiles(testTiles)); this.VerifyAll(); } diff --git a/MergerLogicUnitTests/DataTypes/XYZTest.cs b/MergerLogicUnitTests/DataTypes/XYZTest.cs index 0c87f6cc..cb4f892d 100644 --- a/MergerLogicUnitTests/DataTypes/XYZTest.cs +++ b/MergerLogicUnitTests/DataTypes/XYZTest.cs @@ -426,7 +426,7 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin) new Tile(2, 2, 3, new byte[] { 2}) }; - Assert.ThrowsException(() => xyzSource.UpdateTiles(testTiles)); + Assert.ThrowsExceptionAsync(async () => await xyzSource.UpdateTiles(testTiles)); this.VerifyAll(); } diff --git a/MergerService/src/run.cs b/MergerService/src/run.cs index 61d6d2ed..13247312 100644 --- a/MergerService/src/run.cs +++ b/MergerService/src/run.cs @@ -229,7 +229,7 @@ public void Start() } } - private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallbackUrl) + private async void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallbackUrl) { string methodName = MethodBase.GetCurrentMethod().Name; this._logger.LogDebug($"[{methodName}] start {task.ToString()}"); @@ -359,7 +359,7 @@ private void RunTask(MergeTask task, ITaskUtils taskUtils, string? managerCallba using (this._activitySource.StartActivity("saving tiles")) { this._logger.LogInformation($"[{methodName}] Total amount of tiles to upload to target source: {tiles.Count}"); - target.UpdateTiles(tiles); + await target.UpdateTiles(tiles); this._logger.LogDebug($"[{methodName}] UpdateRelativeProgress"); UpdateRelativeProgress(task, overallTileProgressCount, totalTileCount, taskUtils); }