MapColonies · nitzanmorr · Oct 12, 2025 · Oct 12, 2025 · Oct 12, 2025 · Oct 16, 2025
diff --git a/MergerCli/Process.cs b/MergerCli/Process.cs
@@ -54,10 +54,10 @@ public void Start(IData baseData, IData newData, BatchStatusManager batchStatusM
             }
 
             this._logger.LogInformation($"[{MethodBase.GetCurrentMethod()?.Name}] Total amount of tiles to merge: {totalTileCount - tileProgressCount}");
-            
+
             ParallelRun(baseData, newData, batchStatusManager,
                 tileProgressCount, totalTileCount, resumeBatchIdentifier, resumeMode, pollForBatch);
-            
+
             batchStatusManager.CompleteLayer(newData.Path);
             newData.Reset();
             // base data wrap up is in program as the same base data object is used in multiple calls 
@@ -79,7 +79,7 @@ public void Start(IData baseData, IData newData, BatchStatusManager batchStatusM
                             newData.setBatchIdentifier(resumeBatchIdentifier);
                         }
                     }
-                    
+
                     var incompleteBatch = batchStatusManager.GetFirstIncompleteBatch(newData.Path);
                     if (incompleteBatch is not null)
                     {
@@ -88,21 +88,21 @@ public void Start(IData baseData, IData newData, BatchStatusManager batchStatusM
                 }
 
                 List<Tile> newTiles = newData.GetNextBatch(out string? currentBatchIdentifier, out string? nextBatchIdentifier, totalTileCount);
-                
+
                 if (!resumeMode && newTiles.Count != 0)
                 {
                     batchStatusManager.AssignBatch(newData.Path, currentBatchIdentifier);
                     batchStatusManager.SetCurrentBatch(newData.Path, nextBatchIdentifier);
                 }
-                
+
                 return (newTiles, currentBatchIdentifier);
             }
         }
-        
-        private void ProcessBatch(IData baseData, List<Tile> newTiles, ref long tileProgressCount, long totalTileCount,ref bool pollForBatch)
+
+        private void ProcessBatch(IData baseData, List<Tile> newTiles, ref long tileProgressCount, long totalTileCount, ref bool pollForBatch)
         {
             ConcurrentBag<Tile> tiles = new ConcurrentBag<Tile>();
-            
+
             if (newTiles.Count == 0)
             {
                 pollForBatch = false;
@@ -118,7 +118,7 @@ private void ProcessBatch(IData baseData, List<Tile> newTiles, ref long tileProg
                 var targetCoords = newTile.GetCoord();
                 List<CorrespondingTileBuilder> correspondingTileBuilders = new List<CorrespondingTileBuilder>()
                 {
-                    () => baseData.GetCorrespondingTile(targetCoords, shouldUpscale),
+                    () => baseData.GetCorrespondingTile(targetCoords, null, shouldUpscale),
                     () => newTile
                 };
 
@@ -137,7 +137,7 @@ private void ProcessBatch(IData baseData, List<Tile> newTiles, ref long tileProg
         }
 
         private void ParallelRun(IData baseData, IData newData,
-            BatchStatusManager batchStatusManager, long tileProgressCount, long totalTileCount, string? resumeBatchIdentifier, bool resumeMode,bool pollForBatch)
+            BatchStatusManager batchStatusManager, long tileProgressCount, long totalTileCount, string? resumeBatchIdentifier, bool resumeMode, bool pollForBatch)
         {
             var numOfThreads = this._configManager.GetConfiguration<int>("GENERAL", "parallel", "numOfThreads");
             Parallel.For(0, numOfThreads, new ParallelOptions { MaxDegreeOfParallelism = -1 }, _ =>
@@ -151,7 +151,7 @@ private void ParallelRun(IData baseData, IData newData,
                 }
             });
         }
-        
+
         public void Validate(IData baseData, IData newData, string? incompleteBatchIdentifier)
         {
             List<Tile> newTiles;
@@ -170,7 +170,7 @@ public void Validate(IData baseData, IData newData, string? incompleteBatchIdent
                 for (int i = 0; i < newTiles.Count; i++)
                 {
                     Tile newTile = newTiles[i];
-                    bool baseTileExists = baseData.TileExists(newTile.GetCoord());
+                    bool baseTileExists = baseData.TileExists(newTile.GetCoord(), null);
 
                     if (baseTileExists)
                     {

diff --git a/MergerLogic/Clients/FileClient.cs b/MergerLogic/Clients/FileClient.cs
@@ -1,22 +1,28 @@
 using MergerLogic.Batching;
 using System.IO.Abstractions;
 using MergerLogic.Utils;
+using MergerLogic.ImageProcessing;
 
 namespace MergerLogic.Clients;
 
 public class FileClient : DataUtils, IFileClient
 {
     private readonly IFileSystem _fileSystem;
 
-    public FileClient(string path, IGeoUtils geoUtils, IFileSystem fileSystem) 
+    public FileClient(string path, IGeoUtils geoUtils, IFileSystem fileSystem)
         : base(path, geoUtils)
     {
         this._fileSystem = fileSystem;
     }
 
-    public override Tile? GetTile(int z, int x, int y)
+    public override Tile? GetTile(int z, int x, int y, TileFormat? format)
     {
-        var tilePath = this.GetTilePath(z, x, y);
+        if (format is null)
+        {
+            throw new ArgumentNullException(nameof(format));
+        }
+
+        var tilePath = this.GetTilePath(z, x, y, format.Value);
 
         if (tilePath != null)
         {
@@ -29,23 +35,23 @@ public FileClient(string path, IGeoUtils geoUtils, IFileSystem fileSystem)
         }
     }
 
-    public override bool TileExists(int z, int x, int y)
+    public override bool TileExists(int z, int x, int y, TileFormat? format)
     {
-        return this.GetTilePath(z,x,y) != null;
+        if (format is null)
+        {
+            throw new ArgumentNullException(nameof(format));
+        }
+        return this.GetTilePath(z, x, y, format.Value) != null;
     }
 
-    private string? GetTilePath(int z, int x, int y)
+    private string? GetTilePath(int z, int x, int y, TileFormat format)
     {
         var tilePath = this._fileSystem.Path.Join(z.ToString(), x.ToString(), y.ToString());
-        try
-        {
-            //this may or may not be faster then checking specific files of every supported type depending on the used file system
-            return this._fileSystem.Directory
-                .EnumerateFiles(this.path, $"{tilePath}.*", SearchOption.TopDirectoryOnly).FirstOrDefault();
-        }
-        catch (DirectoryNotFoundException)
+        string fullPath = this._fileSystem.Path.Join(this.path, tilePath, ".", format.ToString().ToLower());
+        if (this._fileSystem.File.Exists(fullPath))
         {
-            return null;
+            return fullPath;
         }
+        return null;
     }
 }
diff --git a/MergerLogic/Clients/GpkgClient.cs b/MergerLogic/Clients/GpkgClient.cs
@@ -1,5 +1,6 @@
 using MergerLogic.Batching;
 using MergerLogic.Extensions;
+using MergerLogic.ImageProcessing;
 using MergerLogic.Utils;
 using Microsoft.Extensions.Logging;
 using System.Data.SQLite;
@@ -124,7 +125,7 @@ public void UpdateExtent(Extent extent)
             }
         }
 
-        public override Tile? GetTile(int z, int x, int y)
+        public override Tile? GetTile(int z, int x, int y, TileFormat? format = null)
         {
             byte[]? blob = null;
 
@@ -146,7 +147,7 @@ public void UpdateExtent(Extent extent)
             return this.CreateTile(z, x, y, blob);
         }
 
-        public override bool TileExists(int z, int x, int y)
+        public override bool TileExists(int z, int x, int y, TileFormat? format = null)
         {
             using (var connection = new SQLiteConnection($"Data Source={this.path}"))
             {

diff --git a/MergerLogic/Clients/HttpSourceClient.cs b/MergerLogic/Clients/HttpSourceClient.cs
@@ -1,4 +1,5 @@
 using MergerLogic.Batching;
+using MergerLogic.ImageProcessing;
 using MergerLogic.Utils;
 
 namespace MergerLogic.Clients
@@ -15,14 +16,14 @@ public HttpSourceClient(string path, IHttpRequestUtils httpClient, IPathPatternU
             this._pathPatternUtils = pathPatternUtils;
         }
 
-        public override Tile? GetTile(int z, int x, int y)
+        public override Tile? GetTile(int z, int x, int y, TileFormat? format = null)
         {
             string url = this._pathPatternUtils.RenderUrlTemplate(x, y, z);
             byte[]? data = this._httpClient.GetData(url, true);
             return this.CreateTile(z, x, y, data);
         }
 
-        public override bool TileExists(int z, int x, int y)
+        public override bool TileExists(int z, int x, int y, TileFormat? format = null)
         {
             return this.GetTile(z, x, y) is not null;
         }

diff --git a/MergerLogic/Clients/S3Client.cs b/MergerLogic/Clients/S3Client.cs
@@ -39,7 +39,7 @@
             {
                 return en.ErrorCode == "NoSuchKey";
             }
-            
+
             return false;
         }
 
@@ -79,21 +79,20 @@
             }
         }
 
-        public override Tile? GetTile(int z, int x, int y)
+        public override Tile? GetTile(int z, int x, int y, TileFormat? format)
         {
             string methodName = MethodBase.GetCurrentMethod().Name;
             this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y}");
-            string keyPrefix = this._pathUtils.GetTilePath(this.path, z, x, y, TileFormat.Jpeg, true);
+            if (format is null)
+            {
+                throw new ArgumentNullException(nameof(format));
+            }
+            string keyPrefix = this._pathUtils.GetTilePath(this.path, z, x, y, format.Value, true);
 
             byte[]? imageBytes = this.GetImageBytes(keyPrefix);
             if (imageBytes == null)
             {
-                keyPrefix = this._pathUtils.GetTilePath(this.path, z, x, y, TileFormat.Png, true);
-                imageBytes = this.GetImageBytes(keyPrefix);
-                if (imageBytes == null)
-                {
-                    return null;
-                }
+                return null;
             }
 
             this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y}");
@@ -102,22 +101,22 @@

        public Tile? GetTile(string key)
        {
            string methodName = MethodBase.GetCurrentMethod().Name;
            this._logger.LogDebug($"[{methodName}] start key: {key}");
            byte[]? imageBytes = this.GetImageBytes(key);
            if (imageBytes == null)
             {
                 return null;
             }
-            
+
             this._logger.LogDebug($"[{methodName}] end key: {key}");
             Coord coords = this._pathUtils.FromPath(key, true);
             return this.CreateTile(coords, imageBytes);
         }
 
-        public override bool TileExists(int z, int x, int y)
+        public override bool TileExists(int z, int x, int y, TileFormat? format = null)
         {
             string methodName = MethodBase.GetCurrentMethod().Name;
             this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y}");
            bool exists = this.GetTileKey(z, x, y) != null;
            this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y}");
@@ -126,7 +125,7 @@

        public void UpdateTile(Tile tile)
        {
            string methodName = MethodBase.GetCurrentMethod().Name;
            this._logger.LogDebug($"[{methodName}] start {tile.ToString()}");
            string key = this._pathUtils.GetTilePath(this.path, tile, true);

@@ -151,7 +150,7 @@

        private string? GetTileKey(int z, int x, int y)
        {
            string methodName = MethodBase.GetCurrentMethod().Name;
            string keyPrefix = this._pathUtils.GetTilePathWithoutExtension(this.path, z, x, y, true);

            try

diff --git a/MergerLogic/DataTypes/Data.cs b/MergerLogic/DataTypes/Data.cs
@@ -1,4 +1,6 @@
+using Amazon.S3.Model;
 using MergerLogic.Batching;
+using MergerLogic.ImageProcessing;
 using MergerLogic.Monitoring.Metrics;
 using MergerLogic.Utils;
 using Microsoft.Extensions.DependencyInjection;
@@ -41,9 +43,9 @@ public abstract class Data<TUtilsType> : IData where TUtilsType : IDataUtils
         public const int MaxZoomRead = 25;
 
         protected delegate int ValFromCoordFunction(Coord coord);
-        protected delegate Tile? GetTileFromXYZFunction(int z, int x, int y);
+        protected delegate Tile? GetTileFromXYZFunction(int z, int x, int y, TileFormat? format);
         protected delegate Coord? GetCoordFromCoordFunction(Coord coord);
-        protected delegate Tile? GetTileFromCoordFunction(Coord coord);
+        protected delegate Tile? GetTileFromCoordFunction(Coord coord, TileFormat? format);
         protected delegate Tile TileConvertorFunction(Tile tile);
         protected delegate Tile? NullableTileConvertorFunction(Tile tile);
         protected IServiceProvider _container;
@@ -125,10 +127,10 @@ protected Data(IServiceProvider container, DataType type, string path, int batch
                 {
                     return this.GeoUtils.FlipY(coord);
                 };
-                this.GetTile = (z, x, y) =>
+                this.GetTile = (z, x, y, format) =>
                 {
                     int newY = this.GeoUtils.FlipY(z, y);
-                    Tile? tile = fixedGridGetTileFunction(z, x, newY);
+                    Tile? tile = fixedGridGetTileFunction(z, x, newY, format);
                     //set cords to current origin
                     tile?.SetCoords(z, x, y);
                     return tile;
@@ -197,7 +199,7 @@ protected virtual Extent GetExtent()
 
         public abstract void Reset();
 
-        protected virtual Tile? InternalGetLastExistingTile(Coord coords)
+        protected virtual Tile? InternalGetLastExistingTile(Coord coords, TileFormat? format)
         {
             this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] started for coord: z:{coords.Z}, x:{coords.X}, y:{coords.Y}");
             // get tiles coordinates
@@ -224,7 +226,7 @@ await Parallel.ForEachAsync(coordsArray, async (coord, cancellationToken) =>
                 {
                     await Task.Run(() =>
                     {
-                        Tile? tile = this.Utils.GetTile(coord.Z, coord.X, coord.Y);
+                        Tile? tile = this.Utils.GetTile(coord.Z, coord.X, coord.Y, format);
                         if (tile != null)
                         {
                             zOrderToTileDictionary.TryAdd(coord.Z, tile);
@@ -247,12 +249,12 @@ await Task.Run(() =>
             return lastTile;
         }
 
-        public bool TileExists(Tile tile)
+        public bool TileExists(Tile tile, TileFormat? format)
         {
-            return this.TileExists(tile.GetCoord());
+            return this.TileExists(tile.GetCoord(), format);
         }
 
-        public bool TileExists(Coord coord)
+        public bool TileExists(Coord coord, TileFormat? format)
         {
             coord.Y = this.ConvertOriginCoord(coord);
             Coord? newCoord = this.FromCurrentGridCoord(coord);
@@ -262,43 +264,43 @@ public bool TileExists(Coord coord)
                 return false;
             }
 
-            return this.Utils.TileExists(newCoord.Z, newCoord.X, newCoord.Y);
+            return this.Utils.TileExists(newCoord.Z, newCoord.X, newCoord.Y, format);
         }
 
         //TODO: move to util after IOC
-        protected Tile? GetLastOneXOneExistingTile(Coord coords)
+        protected Tile? GetLastOneXOneExistingTile(Coord coords, TileFormat? format)
         {
             Coord? newCoords = this.FromCurrentGridCoord(coords);
             if (newCoords is null)
             {
                 return null;
             }
-            Tile? tile = this.InternalGetLastExistingTile(newCoords);
+            Tile? tile = this.InternalGetLastExistingTile(newCoords, format);
             return tile != null ? this.OneXOneConvertor?.ToTwoXOne(tile) : null;
         }
 
-        protected virtual Tile? GetOneXOneTile(int z, int x, int y)
+        protected virtual Tile? GetOneXOneTile(int z, int x, int y, TileFormat? format)
         {
             Coord? oneXoneBaseCoords = this.OneXOneConvertor?.TryFromTwoXOne(z, x, y);
             if (oneXoneBaseCoords == null)
             {
                 return null;
             }
-            Tile? tile = this.Utils.GetTile(oneXoneBaseCoords);
+            Tile? tile = this.Utils.GetTile(oneXoneBaseCoords, format);
             return tile != null ? this.OneXOneConvertor?.ToTwoXOne(tile) : null;
         }
 
         public abstract List<Tile> GetNextBatch(out string batchIdentifier, out string? nextBatchIdentifier, long? totalTilesCount);
 
-        public Tile? GetCorrespondingTile(Coord coords, bool upscale)
+        public Tile? GetCorrespondingTile(Coord coords, TileFormat? format, bool upscale)
         {
             this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] start for coord: z:{coords.Z}, x:{coords.X}, y:{coords.Y}, upscale: {upscale}");
             Stopwatch stopwatch = Stopwatch.StartNew();
-            Tile? correspondingTile = this.GetTile(coords.Z, coords.X, coords.Y);
+            Tile? correspondingTile = this.GetTile(coords.Z, coords.X, coords.Y, format);
 
             if (upscale && correspondingTile == null)
             {
-                correspondingTile = this.GetLastExistingTile(coords);
+                correspondingTile = this.GetLastExistingTile(coords, format);
             }
             stopwatch.Stop();
             this._metricsProvider.TotalFetchTimePerTileHistogram(stopwatch.Elapsed.TotalMilliseconds);

diff --git a/MergerLogic/DataTypes/Fs.cs b/MergerLogic/DataTypes/Fs.cs
@@ -94,7 +94,7 @@ private IEnumerator<Tile> GetTiles()
                             .Where(file => this._supportedFileExtensions.Any(x => file.EndsWith(x, System.StringComparison.OrdinalIgnoreCase))))
                 {
                     Coord coord = this._pathUtils.FromPath(filePath);
-                    Tile? tile = this.Utils.GetTile(coord);
+                    Tile? tile = this.Utils.GetTile(coord, null);
                     if (tile is null)
                     {
                         continue;

diff --git a/MergerLogic/DataTypes/Gpkg.cs b/MergerLogic/DataTypes/Gpkg.cs
@@ -1,5 +1,6 @@
 using MergerLogic.Batching;
 using MergerLogic.Clients;
+using MergerLogic.ImageProcessing;
 using MergerLogic.Utils;
 using Microsoft.Extensions.Logging;
 using System.Reflection;
@@ -126,7 +127,7 @@ public override void setBatchIdentifier(string batchIdentifier)
             }
         }
 
-        protected override Tile? InternalGetLastExistingTile(Coord baseCoords)
+        protected override Tile? InternalGetLastExistingTile(Coord baseCoords, TileFormat? format = null)
         {
             this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] started for coord: z:{baseCoords.Z}, x:{baseCoords.X}, y:{baseCoords.Y}");
             int cordsLength = baseCoords.Z << 1;