From 60267b913caf52480e3bfb236211d88f9eae0062 Mon Sep 17 00:00:00 2001 From: Batu Akan Date: Mon, 20 Oct 2025 16:28:30 +0200 Subject: [PATCH 1/8] Added support to cache online tile maps, also added custom http headers to pass to the remote tile server --- src/index.ts | 172 ++++++++++++++++++++++++++++++++++++++++++--------- src/types.ts | 5 ++ 2 files changed, 148 insertions(+), 29 deletions(-) diff --git a/src/index.ts b/src/index.ts index 5dcc1ff..dd442c5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,7 @@ import * as bluebird from 'bluebird' import path from 'path' import fs from 'fs' +import os from "os" import * as _ from 'lodash' import { findCharts } from './charts' import { apiRoutePrefix } from './constants' @@ -20,8 +21,8 @@ interface Config { interface ChartProviderApp extends ServerAPI, - ResourceProviderRegistry, - Application { + ResourceProviderRegistry, + Application { config: { ssl: boolean configPath: string @@ -122,6 +123,24 @@ module.exports = (app: ChartProviderApp): Plugin => { description: 'Map URL (for tilelayer include {z}, {x} and {y} parameters, e.g. "http://example.org/{z}/{x}/{y}.png")' }, + cached: { + type: 'boolean', + title: 'Cache tiles locally', + description: + 'If enabled, tiles fetched from the server will be cached locally to reduce bandwidth usage.', + default: false + }, + headers: { + type: 'array', + title: 'Headers', + description: + 'List of http headers to be sent to the server when requesting map tiles.', + items: { + title: 'Header Name and Value', + description: 'Name and Value of the HTTP header', + type: 'string' + } + }, style: { type: 'string', title: 'Vector Map Style', @@ -165,6 +184,27 @@ module.exports = (app: ChartProviderApp): Plugin => { app.debug(`** loaded config: ${config}`) props = { ...config } + const getLocalIPAddress = (): string | null => { + const interfaces = os.networkInterfaces(); + + for (const name of Object.keys(interfaces)) { + const iface = interfaces[name]; + if (!iface) continue; + + for (const alias of iface) { + // Skip internal (i.e., 127.0.0.1) and non-IPv4 addresses + if (alias.family === "IPv4" && !alias.internal) { + return alias.address; + } + } + } + + return null; // not found + } + const urlBase = `${app.config.ssl ? 'https' : 'http'}://${getLocalIPAddress()}:${'getExternalPort' in app.config ? app.config.getExternalPort() : 3000 + }` + app.debug(`**urlBase** ${urlBase}`) + const chartPaths = _.isEmpty(props.chartPaths) ? [defaultChartsPath] : resolveUniqueChartPaths(props.chartPaths, configBasePath) @@ -172,7 +212,7 @@ module.exports = (app: ChartProviderApp): Plugin => { const onlineProviders = _.reduce( props.onlineChartProviders, (result: { [key: string]: object }, data) => { - const provider = convertOnlineProviderConfig(data) + const provider = convertOnlineProviderConfig(data, urlBase) result[provider.identifier] = provider return result }, @@ -233,28 +273,34 @@ module.exports = (app: ChartProviderApp): Plugin => { if (!provider) { return res.sendStatus(404) } - switch (provider._fileFormat) { - case 'directory': - return serveTileFromFilesystem( - res, - provider, - parseInt(z), - parseInt(x), - parseInt(y) - ) - case 'mbtiles': - return serveTileFromMbtiles( - res, - provider, - parseInt(z), - parseInt(x), - parseInt(y) - ) - default: - console.log( - `Unknown chart provider fileformat ${provider._fileFormat}` - ) - res.status(500).send() + if (provider.cached === true) { + return serveTileFromCache(res, provider, parseInt(z), parseInt(x), parseInt(y)) + } + else + { + switch (provider._fileFormat) { + case 'directory': + return serveTileFromFilesystem( + res, + provider, + parseInt(z), + parseInt(x), + parseInt(y) + ) + case 'mbtiles': + return serveTileFromMbtiles( + res, + provider, + parseInt(z), + parseInt(x), + parseInt(y) + ) + default: + console.log( + `Unknown chart provider fileformat ${provider._fileFormat}` + ) + res.status(500).send() + } } } ) @@ -322,6 +368,55 @@ module.exports = (app: ChartProviderApp): Plugin => { } } + const serveTileFromCache = async ( + res: Response, + provider: ChartProvider, + z: number, + x: number, + y: number + ) => { + const tilePath = path.join(defaultChartsPath, `${provider.name}`, `${z}`, `${x}`, `${y}.${provider.format}`) + if (fs.existsSync(tilePath)) { + res.set('Content-Type', `image/${provider.format}`) + res.set('Cache-Control', responseHttpOptions.headers['Cache-Control']) + const stream = fs.createReadStream(tilePath) + stream.on('error', (err) => { + console.error(`Error reading cached tile ${tilePath}:`, err) + if (!res.headersSent) { + res.sendStatus(500) + } + }) + stream.pipe(res) + return + } + if (!provider.remoteUrl) { + console.error(`No remote URL defined for cached provider ${provider.name}`) + res.sendStatus(500) + return + } + const url = provider.remoteUrl + .replace("{z}", z.toString()) + .replace("{x}", x.toString()) + .replace("{y}", y.toString()) + .replace("{-y}", (Math.pow(2, z) - 1 - y).toString()); + const response = await fetch(url, { + headers: provider.headers + }) + if (!response.ok) { + console.error(`Error fetching tile ${provider.name}/${z}/${x}/${y}:`) + res.sendStatus(500) + } + const arrayBuffer = await response.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + if (!fs.existsSync(path.dirname(tilePath))) { + fs.mkdirSync(path.dirname(tilePath), { recursive: true }) + } + fs.writeFileSync(tilePath, buffer) + res.set('Content-Type', `image/${provider.format}`) + res.send(buffer) + + } + return plugin } @@ -341,8 +436,24 @@ const resolveUniqueChartPaths = ( return _.uniq(paths) } -const convertOnlineProviderConfig = (provider: OnlineChartProvider) => { +const convertOnlineProviderConfig = (provider: OnlineChartProvider, urlBase: string) => { const id = _.kebabCase(_.deburr(provider.name)) + + const parseHeaders = (arr?: string[] | null): { [key: string]: string } => { + const result: { [key: string]: string } = {} + if (!arr) return result + for (const entry of arr) { + if (typeof entry !== 'string') continue + const idx = entry.indexOf(':') + if (idx === -1) continue + const key = entry.slice(0, idx).trim() + const value = entry.slice(idx + 1).trim() + if (key) result[key] = value + } + return result + } + + const data = { identifier: id, name: provider.name, @@ -355,13 +466,16 @@ const convertOnlineProviderConfig = (provider: OnlineChartProvider) => { type: provider.serverType ? provider.serverType : 'tilelayer', style: provider.style ? provider.style : null, v1: { - tilemapUrl: provider.url, + tilemapUrl: provider.cached ? `${urlBase}${chartTilesPath}/${id}/{z}/{x}/{y}` : provider.url, chartLayers: provider.layers ? provider.layers : null }, v2: { - url: provider.url, + url: provider.cached ? `${urlBase}${chartTilesPath}/${id}/{z}/{x}/{y}` : provider.url, layers: provider.layers ? provider.layers : null - } + }, + cached: provider.cached ? provider.cached : false, + remoteUrl: provider.cached ? provider.url : null, + headers: parseHeaders(provider.headers), } return data } diff --git a/src/types.ts b/src/types.ts index dad6bba..dc2d503 100644 --- a/src/types.ts +++ b/src/types.ts @@ -31,6 +31,9 @@ export interface ChartProvider { format?: string style?: string layers?: string[] + cached?: boolean + remoteUrl?: string + headers?: { [key: string]: string } } export interface OnlineChartProvider { @@ -41,6 +44,8 @@ export interface OnlineChartProvider { serverType: MapSourceType format: 'png' | 'jpg' url: string + cached: boolean + headers?: string[] style: string layers: string[] } From c2408a2ec45be10108f1efb7f8a33546dd65f7e0 Mon Sep 17 00:00:00 2001 From: Batu Akan Date: Sun, 26 Oct 2025 17:32:47 +0100 Subject: [PATCH 2/8] Added chart downloader web addin for charts-addin --- package.json | 3 + public/index.html | 324 +++++++++++++++++++++++++++++++++++++++++ src/chartDownloader.ts | 293 +++++++++++++++++++++++++++++++++++++ src/index.ts | 141 +++++++++--------- 4 files changed, 694 insertions(+), 67 deletions(-) create mode 100644 public/index.html create mode 100644 src/chartDownloader.ts diff --git a/package.json b/package.json index 99325a7..c982250 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "keywords": [ "signalk", "signalk-node-server-plugin", + "signalk-webapp", "nautic", "chart", "mbtiles", @@ -31,6 +32,8 @@ "dependencies": { "@mapbox/mbtiles": "^0.12.1", "@signalk/server-api": "^2.0.0", + "@turf/boolean-intersects": "^7.2.0", + "@turf/helpers": "^7.2.0", "bluebird": "3.5.1", "lodash": "^4.17.11", "xml2js": "0.6.2" diff --git a/public/index.html b/public/index.html new file mode 100644 index 0000000..04306b6 --- /dev/null +++ b/public/index.html @@ -0,0 +1,324 @@ + + + + + Signalk charts-addin + + + + +
+ Seed charts for offline usage + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + +
#ChartRegionTotal tilesDownloadedCachedFailedProgressStatusActions
Loading progress…
+ + +
+ + + + + \ No newline at end of file diff --git a/src/chartDownloader.ts b/src/chartDownloader.ts new file mode 100644 index 0000000..d43673d --- /dev/null +++ b/src/chartDownloader.ts @@ -0,0 +1,293 @@ +import fs from "fs"; +import path from "path"; +import pLimit from "p-limit" +import booleanIntersects from '@turf/boolean-intersects' +import type { BBox, FeatureCollection, Polygon } from 'geojson' +import { polygon } from '@turf/helpers' + +interface Tile { + x: number + y: number + z: number +} + +export enum DownloadStatus { + NotStarted, + Preparing, + Downloading, + Completed, + Cancelled +} + +export class ChartDownloader { + public static ActiveDownloads: { [key: number]: ChartDownloader } = {}; + private static nextJobId = 1; + + private downloadStatus: DownloadStatus = DownloadStatus.NotStarted; + private progress: number = 0; + private totalTiles: number = 0; + private downloadedTiles: number = 0; + private failedTiles: number = 0; + private cachedTiles: number = 0; + + private concurrentDownloadsLimit = 20; + private regionName: string = ""; + private cancelRequested: boolean = false; + + + constructor(private urlBase: string, private chartsPath: string, private provider: any) {} + + public static createAndRegister(urlBase: string, chartsPath: string, provider: any): { jobId: number, downloader: ChartDownloader } { + const downloader = new ChartDownloader(urlBase, chartsPath, provider); + const jobId = this.nextJobId++; + this.ActiveDownloads[jobId] = downloader; + return { jobId, downloader }; + } + + + /** + * Download map tiles for a specific region. + * @param region + * @param maxZoom Maximum zoom level to download + */ + async downloadTiles(regionGUID: string, maxZoom: number): Promise { + this.downloadStatus = DownloadStatus.Preparing; + const region = await this.getRegion(regionGUID); + const geojson = this.convertRegionToGeoJSON(region); + const tiles = this.getTilesForGeoJSON(geojson, this.provider.minzoom, maxZoom); + const tileToDownload = await this.filterCachedTiles(tiles); + + this.totalTiles = tiles.length; + this.downloadedTiles = 0; + this.cachedTiles = this.totalTiles - tileToDownload.length; + this.regionName = region.name || ""; + + this.downloadStatus = DownloadStatus.Downloading; + const limit = pLimit(this.concurrentDownloadsLimit); // concurrent download limit + const promises: Promise[] = []; + for (const tile of tileToDownload) { + if (this.cancelRequested) break; + promises.push(limit(async () => { + if (this.cancelRequested) return; + const buffer = await ChartDownloader.fetchTile(this.chartsPath, this.provider, tile); + if (this.cancelRequested) return; + if (buffer === null) { + this.failedTiles += 1; + } else { + this.downloadedTiles += 1; + } + })); + } + try { + await Promise.all(promises); + + } catch (err) { + // silent failure, caller can log if needed + console.error(`Error downloading tiles:`, err); + } + if (this.cancelRequested) { + this.downloadStatus = DownloadStatus.Cancelled; + return; + } + this.downloadStatus = DownloadStatus.Completed; + } + + async deleteTiles(regionGUID: string): Promise { + const region = await this.getRegion(regionGUID); + const geojson = this.convertRegionToGeoJSON(region); + const tiles = this.getTilesForGeoJSON(geojson, this.provider.minzoom, this.provider.maxzoom); + for (const tile of tiles) { + if (this.cancelRequested) break; + const tilePath = path.join(this.chartsPath, `${this.provider.name}`, `${tile.z}`, `${tile.x}`, `${tile.y}.${this.provider.format}`); + if (fs.existsSync(tilePath)) { + try { + await fs.promises.unlink(tilePath); + } catch (err) { + console.error(`Error deleting cached tile ${tilePath}:`, err); + } + } + } + this.downloadStatus = DownloadStatus.Completed; + } + + public cancelDownload() { + this.cancelRequested = true; + } + + async filterCachedTiles(allTiles: Tile[]): Promise { + const uncachedTiles: Tile[] = []; + for (const tile of allTiles) { + const tilePath = path.join(this.chartsPath, `${this.provider.name}`, `${tile.z}`, `${tile.x}`, `${tile.y}.${this.provider.format}`); + if (!fs.existsSync(tilePath)) { + uncachedTiles.push(tile); + } + } + return uncachedTiles; + } + + public progressInfo(){ + return { + chartName: this.provider.name, + regionName: this.regionName, + totalTiles: this.totalTiles, + downloadedTiles: this.downloadedTiles, + cachedTiles: this.cachedTiles, + failedTiles: this.failedTiles, + progress: this.totalTiles > 0 ? (this.downloadedTiles + this.cachedTiles) / this.totalTiles : 0, + status: this.downloadStatus + }; + } + + static async fetchTile(chartsPath: string, provider: any, tile: Tile): Promise { + const tilePath = path.join(chartsPath, `${provider.name}`, `${tile.z}`, `${tile.x}`, `${tile.y}.${provider.format}`); + if (fs.existsSync(tilePath)) { + try { + const data = await fs.promises.readFile(tilePath); + return data; + } catch (err) { + console.error(`Error reading cached tile ${tilePath}:`, err); + } + } + if (!provider.remoteUrl) { + console.error(`No remote URL defined for cached provider ${provider.name}`); + return null; + } + const url = provider.remoteUrl + .replace("{z}", tile.z.toString()) + .replace("{x}", tile.x.toString()) + .replace("{y}", tile.y.toString()) + .replace("{-y}", (Math.pow(2, tile.z) - 1 - tile.y).toString()); + const response = await fetch(url, { + headers: provider.headers + }); + if (!response.ok) { + return null; + } + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + if (!fs.existsSync(path.dirname(tilePath))) { + fs.mkdirSync(path.dirname(tilePath), { recursive: true }); + } + await fs.promises.writeFile(tilePath, buffer); + return buffer; + } + + + getTilesForGeoJSON( + geojson: FeatureCollection, + zoomMin = 1, + zoomMax = 14 + ): Tile[] { + const tiles: Tile[] = []; + + for (const feature of geojson.features) { + if (feature.geometry.type !== "Polygon" && feature.geometry.type !== "MultiPolygon") { + console.warn("Skipping non-polygon feature"); + continue; + } + + const bbox = this.getBBox(feature.geometry as Polygon); + for (let z = zoomMin; z <= zoomMax; z++) { + const [minX, minY] = this.lonLatToTileXY(bbox[0], bbox[3], z); // top-left + const [maxX, maxY] = this.lonLatToTileXY(bbox[2], bbox[1], z); // bottom-right + + for (let x = minX; x <= maxX; x++) { + for (let y = minY; y <= maxY; y++) { + const tileBbox = this.tileToBBox(x, y, z); + const tilePoly = this.bboxPolygon(tileBbox); + + if (booleanIntersects(feature as any, tilePoly)) { + tiles.push({ x, y, z }); + } + } + } + } + } + + return tiles; + } + + async getRegion(regionGUID: string): Promise> { + let regionData: any + const resp = await fetch(`${this.urlBase}/signalk/v2/api/resources/regions/${regionGUID}`) + if (!resp.ok) { + const body = await resp.text().catch(() => '') + console.error(`Failed to fetch region ${regionGUID}: ${resp.status} ${resp.statusText} ${body}`) + return {}; + } + regionData = await resp.json() + return regionData; + } + + + convertRegionToGeoJSON(region: Record): FeatureCollection { + const feature = region.feature; + if (!feature || feature.type !== "Feature" || !feature.geometry) { + throw new Error("Invalid region: missing feature or geometry"); + } + + const geoFeature = { + type: "Feature" as const, + id: feature.id || undefined, + geometry: feature.geometry, + properties: { + name: region.name || "", + description: region.description || "", + timestamp: region.timestamp || "", + source: region.$source || "", + ...feature.properties, + }, + }; + + return { + type: "FeatureCollection" as const, + features: [geoFeature], + }; + } + + private getBBox(geometry: Polygon): BBox { + let minLon = 180, minLat = 90, maxLon = -180, maxLat = -90; + for (const ring of geometry.coordinates) { + for (const [lon, lat] of ring) { + minLon = Math.min(minLon, lon); + minLat = Math.min(minLat, lat); + maxLon = Math.max(maxLon, lon); + maxLat = Math.max(maxLat, lat); + } + } + return [minLon, minLat, maxLon, maxLat]; + } + + private lonLatToTileXY(lon: number, lat: number, zoom: number): [number, number] { + const n = 2 ** zoom; + const x = Math.floor(((lon + 180) / 360) * n); + const y = Math.floor( + ((1 - + Math.log(Math.tan((lat * Math.PI) / 180) + 1 / Math.cos((lat * Math.PI) / 180)) / + Math.PI) / + 2) * + n + ); + return [x, y]; + } + + private tileToBBox(x: number, y: number, z: number): BBox { + const n = 2 ** z; + const lon1 = (x / n) * 360 - 180; + const lat1 = (Math.atan(Math.sinh(Math.PI * (1 - (2 * y) / n))) * 180) / Math.PI; + const lon2 = ((x + 1) / n) * 360 - 180; + const lat2 = (Math.atan(Math.sinh(Math.PI * (1 - (2 * (y + 1)) / n))) * 180) / Math.PI; + return [lon1, lat2, lon2, lat1]; + } + + private bboxPolygon(boundingBox: BBox) { + const [minLon, minLat, maxLon, maxLat] = boundingBox; + return polygon([[ + [minLon, minLat], + [maxLon, minLat], + [maxLon, maxLat], + [minLon, maxLat], + [minLon, minLat] + ]]); + } +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index dd442c5..e39c21f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import * as _ from 'lodash' import { findCharts } from './charts' import { apiRoutePrefix } from './constants' import { ChartProvider, OnlineChartProvider } from './types' +import { ChartDownloader } from './chartDownloader' import { Request, Response, Application } from 'express' import { OutgoingHttpHeaders } from 'http' import { @@ -42,6 +43,8 @@ module.exports = (app: ChartProviderApp): Plugin => { chartPaths: [], onlineChartProviders: [] } + + let urlBase: string = '' const configBasePath = app.config.configPath const defaultChartsPath = path.join(configBasePath, '/charts') const serverMajorVersion = app.config.version @@ -49,6 +52,8 @@ module.exports = (app: ChartProviderApp): Plugin => { : '1' ensureDirectoryExists(defaultChartsPath) + + // ******** REQUIRED PLUGIN DEFINITION ******* const CONFIG_SCHEMA = { title: 'Signal K Charts', @@ -123,21 +128,21 @@ module.exports = (app: ChartProviderApp): Plugin => { description: 'Map URL (for tilelayer include {z}, {x} and {y} parameters, e.g. "http://example.org/{z}/{x}/{y}.png")' }, - cached: { + proxy: { type: 'boolean', - title: 'Cache tiles locally', + title: 'Proxy through signalk server', description: - 'If enabled, tiles fetched from the server will be cached locally to reduce bandwidth usage.', + 'Create a proxy to serve remote tiles and cache fetched tiles from the remote server, to serve them locally on subsequent requests.', default: false }, headers: { type: 'array', title: 'Headers', description: - 'List of http headers to be sent to the server when requesting map tiles.', + 'List of http headers to be sent to the remote server when requesting map tiles through proxy.', items: { - title: 'Header Name and Value', - description: 'Name and Value of the HTTP header', + title: 'Header Name: Value', + description: 'Name and Value of the HTTP header separated by colon', type: 'string' } }, @@ -184,25 +189,7 @@ module.exports = (app: ChartProviderApp): Plugin => { app.debug(`** loaded config: ${config}`) props = { ...config } - const getLocalIPAddress = (): string | null => { - const interfaces = os.networkInterfaces(); - - for (const name of Object.keys(interfaces)) { - const iface = interfaces[name]; - if (!iface) continue; - - for (const alias of iface) { - // Skip internal (i.e., 127.0.0.1) and non-IPv4 addresses - if (alias.family === "IPv4" && !alias.internal) { - return alias.address; - } - } - } - - return null; // not found - } - const urlBase = `${app.config.ssl ? 'https' : 'http'}://${getLocalIPAddress()}:${'getExternalPort' in app.config ? app.config.getExternalPort() : 3000 - }` + urlBase = `${app.config.ssl ? 'https' : 'http'}://localhost:${'getExternalPort' in app.config ? app.config.getExternalPort() : 3000}` app.debug(`**urlBase** ${urlBase}`) const chartPaths = _.isEmpty(props.chartPaths) @@ -269,12 +256,15 @@ module.exports = (app: ChartProviderApp): Plugin => { `${chartTilesPath}/:identifier/:z([0-9]*)/:x([0-9]*)/:y([0-9]*)`, async (req: Request, res: Response) => { const { identifier, z, x, y } = req.params + const ix = parseInt(x) + const iy = parseInt(y) + const iz = parseInt(z) const provider = chartProviders[identifier] if (!provider) { return res.sendStatus(404) } if (provider.cached === true) { - return serveTileFromCache(res, provider, parseInt(z), parseInt(x), parseInt(y)) + return serveTileFromCache(res, provider, iz, ix, iy) } else { @@ -283,17 +273,17 @@ module.exports = (app: ChartProviderApp): Plugin => { return serveTileFromFilesystem( res, provider, - parseInt(z), - parseInt(x), - parseInt(y) + iz, + ix, + iy ) case 'mbtiles': return serveTileFromMbtiles( res, provider, - parseInt(z), - parseInt(x), - parseInt(y) + iz, + ix, + iy ) default: console.log( @@ -305,6 +295,56 @@ module.exports = (app: ChartProviderApp): Plugin => { } ) + app.post(`${chartTilesPath}/cache/:identifier/:regionGUID/:maxZoom`, async (req: Request, res: Response) => { + const { identifier, regionGUID, maxZoom } = req.params + const provider = chartProviders[identifier] + if (!provider) { + return res.sendStatus(500) + } + const maxZoomParsed = maxZoom ? parseInt(maxZoom) : 17 + + const { jobId, downloader } = ChartDownloader.createAndRegister(urlBase, defaultChartsPath, provider) + ;(async () => { + await downloader.downloadTiles(regionGUID, maxZoomParsed) + })() + return res.status(200).json({ progress: `${chartTilesPath}/progress/${jobId}` }) + }) + + app.delete(`${chartTilesPath}/cache/:identifier/:regionGUID`, async (req: Request, res: Response) => { + const { identifier, regionGUID } = req.params + const provider = chartProviders[identifier] + if (!provider) { + return res.sendStatus(500) + } + + const { jobId, downloader } = ChartDownloader.createAndRegister(urlBase, defaultChartsPath, provider) + // Long going process, create an endpoint to monitor progress + ;(async () => { + await downloader.deleteTiles(regionGUID) + })() + return res.status(200).json({ progress: `${chartTilesPath}/cache/progress/${jobId}` }) + }) + + app.get(`${chartTilesPath}/cache/progress`, (req: Request, res: Response) => { + const progresses = Object.entries(ChartDownloader.ActiveDownloads).map(([id, job]) => { + return job.progressInfo() + }) + return res.status(200).json(progresses) + + }) + + app.put(`${chartTilesPath}/cache/progress/:id/cancel`, (req: Request, res: Response) => { + const { id } = req.params + const parsedId = parseInt(id) + const job = ChartDownloader.ActiveDownloads[parsedId] + if (job) { + job.cancelDownload() + return res.status(200).send(`Cancelled job ${parsedId}`) + } else { + return res.status(404).send(`Job ${parsedId} not found`) + } + }) + app.debug('** Registering v1 API paths **') app.get( @@ -375,46 +415,13 @@ module.exports = (app: ChartProviderApp): Plugin => { x: number, y: number ) => { - const tilePath = path.join(defaultChartsPath, `${provider.name}`, `${z}`, `${x}`, `${y}.${provider.format}`) - if (fs.existsSync(tilePath)) { - res.set('Content-Type', `image/${provider.format}`) - res.set('Cache-Control', responseHttpOptions.headers['Cache-Control']) - const stream = fs.createReadStream(tilePath) - stream.on('error', (err) => { - console.error(`Error reading cached tile ${tilePath}:`, err) - if (!res.headersSent) { - res.sendStatus(500) - } - }) - stream.pipe(res) - return - } - if (!provider.remoteUrl) { - console.error(`No remote URL defined for cached provider ${provider.name}`) + const buffer = await ChartDownloader.fetchTile(defaultChartsPath, provider, { x, y, z }) + if (!buffer) { res.sendStatus(500) return } - const url = provider.remoteUrl - .replace("{z}", z.toString()) - .replace("{x}", x.toString()) - .replace("{y}", y.toString()) - .replace("{-y}", (Math.pow(2, z) - 1 - y).toString()); - const response = await fetch(url, { - headers: provider.headers - }) - if (!response.ok) { - console.error(`Error fetching tile ${provider.name}/${z}/${x}/${y}:`) - res.sendStatus(500) - } - const arrayBuffer = await response.arrayBuffer() - const buffer = Buffer.from(arrayBuffer) - if (!fs.existsSync(path.dirname(tilePath))) { - fs.mkdirSync(path.dirname(tilePath), { recursive: true }) - } - fs.writeFileSync(tilePath, buffer) res.set('Content-Type', `image/${provider.format}`) res.send(buffer) - } return plugin From b677618551734c0229642b74e132cacfc8748035 Mon Sep 17 00:00:00 2001 From: Batu Akan Date: Sun, 26 Oct 2025 17:44:21 +0100 Subject: [PATCH 3/8] Updated readme and changed config parameter name from cached to proxy --- README.md | 1 + public/index.html | 2 +- src/index.ts | 10 +++++----- src/types.ts | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 92cf917..f67f388 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ For WMS & WMTS sources you can specify the layers you wish to display. Online chart provider layers +A proxy for online charts can be created using the "Proxy through signalk server" option. If enabled tiles will be fetched from the remote server and cached by the signalk server making it possible to store the tiles for offline usage. Additional http headers can be passed to the remote server by adding colon separated headers, e.g. User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64). User-Agent is the header name and Mozilla... will be the value. ### Supported chart formats diff --git a/public/index.html b/public/index.html index 04306b6..92dfc8d 100644 --- a/public/index.html +++ b/public/index.html @@ -270,7 +270,7 @@ }) .forEach(([id, info]) => { - if (info.cached === true) { + if (info.proxy === true) { const opt = document.createElement('option'); opt.value = id; opt.textContent = (info && info.name) ? info.name : id; diff --git a/src/index.ts b/src/index.ts index e39c21f..5ac34d2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -263,7 +263,7 @@ module.exports = (app: ChartProviderApp): Plugin => { if (!provider) { return res.sendStatus(404) } - if (provider.cached === true) { + if (provider.proxy === true) { return serveTileFromCache(res, provider, iz, ix, iy) } else @@ -473,15 +473,15 @@ const convertOnlineProviderConfig = (provider: OnlineChartProvider, urlBase: str type: provider.serverType ? provider.serverType : 'tilelayer', style: provider.style ? provider.style : null, v1: { - tilemapUrl: provider.cached ? `${urlBase}${chartTilesPath}/${id}/{z}/{x}/{y}` : provider.url, + tilemapUrl: provider.proxy ? `${urlBase}${chartTilesPath}/${id}/{z}/{x}/{y}` : provider.url, chartLayers: provider.layers ? provider.layers : null }, v2: { - url: provider.cached ? `${urlBase}${chartTilesPath}/${id}/{z}/{x}/{y}` : provider.url, + url: provider.proxy ? `${urlBase}${chartTilesPath}/${id}/{z}/{x}/{y}` : provider.url, layers: provider.layers ? provider.layers : null }, - cached: provider.cached ? provider.cached : false, - remoteUrl: provider.cached ? provider.url : null, + proxy: provider.proxy ? provider.proxy : false, + remoteUrl: provider.proxy ? provider.url : null, headers: parseHeaders(provider.headers), } return data diff --git a/src/types.ts b/src/types.ts index dc2d503..339e516 100644 --- a/src/types.ts +++ b/src/types.ts @@ -31,7 +31,7 @@ export interface ChartProvider { format?: string style?: string layers?: string[] - cached?: boolean + proxy?: boolean remoteUrl?: string headers?: { [key: string]: string } } @@ -44,7 +44,7 @@ export interface OnlineChartProvider { serverType: MapSourceType format: 'png' | 'jpg' url: string - cached: boolean + proxy: boolean headers?: string[] style: string layers: string[] From 1e1095870b22098a70e7cbf3bab7bb6c53cdeb79 Mon Sep 17 00:00:00 2001 From: Batu Akan Date: Sun, 26 Oct 2025 21:25:58 +0100 Subject: [PATCH 4/8] Fixed failing test --- test/plugin-test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/plugin-test.js b/test/plugin-test.js index 2651889..34e3bef 100644 --- a/test/plugin-test.js +++ b/test/plugin-test.js @@ -75,10 +75,13 @@ describe('GET /resources/charts', () => { expect(result.body['test-name']).to.deep.equal({ bounds: [-180, -90, 180, 90], format: 'jpg', + headers: {}, identifier: 'test-name', maxzoom: 15, minzoom: 2, name: 'Test Name', + proxy: false, + remoteUrl: null, scale: 250000, "style": null, tilemapUrl: 'https://example.com', From 8915b1ed39abc5c152b5564beabdb80f9c55c746 Mon Sep 17 00:00:00 2001 From: Batu Akan Date: Mon, 27 Oct 2025 12:36:11 +0100 Subject: [PATCH 5/8] Added ~tilePath~ instead of full url --- src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 5ac34d2..5e6ea50 100644 --- a/src/index.ts +++ b/src/index.ts @@ -473,11 +473,11 @@ const convertOnlineProviderConfig = (provider: OnlineChartProvider, urlBase: str type: provider.serverType ? provider.serverType : 'tilelayer', style: provider.style ? provider.style : null, v1: { - tilemapUrl: provider.proxy ? `${urlBase}${chartTilesPath}/${id}/{z}/{x}/{y}` : provider.url, + tilemapUrl: provider.proxy ? `~tilePath~/${id}/{z}/{x}/{y}` : provider.url, chartLayers: provider.layers ? provider.layers : null }, v2: { - url: provider.proxy ? `${urlBase}${chartTilesPath}/${id}/{z}/{x}/{y}` : provider.url, + url: provider.proxy ? `~tilePath~/${id}/{z}/{x}/{y}` : provider.url, layers: provider.layers ? provider.layers : null }, proxy: provider.proxy ? provider.proxy : false, From 67ea7ca22a764f668063ef7bbd13d0b191e93a4f Mon Sep 17 00:00:00 2001 From: Batu Akan Date: Fri, 31 Oct 2025 20:06:34 +0100 Subject: [PATCH 6/8] PR fixes - antimeridian check - Added support to seed regions, bboz or tile and subtiles - Updated webui - Added safeguards for low diskspace and huge downloads - Code restructuring - Changed signalk typo to SignalK --- README.md | 2 +- package.json | 3 + public/index.html | 346 +++++------------------ public/index.js | 303 ++++++++++++++++++++ public/style.css | 116 ++++++++ src/@types/geojson-antimeridian-cut.d.ts | 6 + src/chartDownloader.ts | 305 ++++++++++++++------ src/index.ts | 109 +++---- 8 files changed, 771 insertions(+), 419 deletions(-) create mode 100644 public/index.js create mode 100644 public/style.css create mode 100644 src/@types/geojson-antimeridian-cut.d.ts diff --git a/README.md b/README.md index f67f388..b5a3bcf 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ For WMS & WMTS sources you can specify the layers you wish to display. Online chart provider layers -A proxy for online charts can be created using the "Proxy through signalk server" option. If enabled tiles will be fetched from the remote server and cached by the signalk server making it possible to store the tiles for offline usage. Additional http headers can be passed to the remote server by adding colon separated headers, e.g. User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64). User-Agent is the header name and Mozilla... will be the value. +A proxy for online charts can be created using the "Proxy through SignalK server" option. If enabled tiles will be fetched from the remote server and cached by the SignalK server making it possible to store the tiles for offline usage. Additional http headers can be passed to the remote server by adding colon separated headers, e.g. User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64). User-Agent is the header name and Mozilla... will be the value. ### Supported chart formats diff --git a/package.json b/package.json index c982250..290907a 100644 --- a/package.json +++ b/package.json @@ -32,9 +32,12 @@ "dependencies": { "@mapbox/mbtiles": "^0.12.1", "@signalk/server-api": "^2.0.0", + "@turf/bbox": "^7.2.0", "@turf/boolean-intersects": "^7.2.0", "@turf/helpers": "^7.2.0", "bluebird": "3.5.1", + "check-disk-space": "^3.4.0", + "geojson-antimeridian-cut": "^0.1.0", "lodash": "^4.17.11", "xml2js": "0.6.2" }, diff --git a/public/index.html b/public/index.html index 92dfc8d..afda67e 100644 --- a/public/index.html +++ b/public/index.html @@ -2,6 +2,7 @@ + Signalk charts-addin @@ -9,21 +10,49 @@
Seed charts for offline usage - + Select chart provider:
- - + Region + + + +
+ + + +
+ + + + +
+ + + + Select max zoom:
- - + +
+
- - - - - - - - - - - - - - - - - - - - -
#ChartRegionTotal tilesDownloadedCachedFailedProgressStatusActions
Loading progress…
- - + + + + + + + + + + + + + + + + + + + + +
#ChartRegionTotal tilesDownloadedCachedFailedProgressStatusActions
Loading progress…
- - \ No newline at end of file diff --git a/public/index.js b/public/index.js new file mode 100644 index 0000000..e2ac110 --- /dev/null +++ b/public/index.js @@ -0,0 +1,303 @@ + + + +(function () { + + +})(); + +document.addEventListener('DOMContentLoaded', async () => { + const regionSelect = document.getElementById("region"); + const bboxInputs = document.querySelectorAll("#bboxMinLon, #bboxMinLat, #bboxMaxLon, #bboxMaxLat"); + const tileInputs = document.querySelectorAll("#tileX, #tileY, #tileZ"); + + function updateInputs() { + const type = document.querySelector('input[name="inputType"]:checked').value; + + regionSelect.disabled = (type !== "region"); + bboxInputs.forEach(i => i.disabled = (type !== "bbox")); + tileInputs.forEach(i => i.disabled = (type !== "tile")); + } + + // Attach listeners + document.querySelectorAll('input[name="inputType"]').forEach(r => { + r.addEventListener("change", updateInputs); + }); + + updateInputs(); // initial load + + if (!regionSelect) return; + + try { + const resp = await fetch('/signalk/v2/api/resources/regions'); + if (!resp.ok) throw new Error(`Failed to fetch regions: ${resp.status}`); + const data = await resp.json(); + // clear existing options + regionSelect.innerHTML = ''; + + const entries = Object.entries(data || {}); + entries + .sort(([, a], [, b]) => { + const na = (a && a.name) ? String(a.name) : ''; + const nb = (b && b.name) ? String(b.name) : ''; + return na.localeCompare(nb); + }) + .forEach(([id, info]) => { + const opt = document.createElement('option'); + opt.value = id; + opt.textContent = (info && info.name) ? info.name : id; + regionSelect.appendChild(opt); + + }); + + // if no regions returned, add a fallback option + if (!regionSelect.options.length) { + const opt = document.createElement('option'); + opt.value = ''; + opt.textContent = '-- No regions available --'; + regionSelect.appendChild(opt); + } + } catch (err) { + console.error('Error loading regions:', err); + } + + //Fetch available maps and populate the chart select box + const chartSelect = document.getElementById('chart'); + if (!chartSelect) return; + + try { + const resp = await fetch('/signalk/v2/api/resources/charts'); + if (!resp.ok) throw new Error(`Failed to fetch charts: ${resp.status}`); + const data = await resp.json(); + // clear existing options + chartSelect.innerHTML = ''; + + const entries = Object.entries(data || {}); + entries + .sort(([, a], [, b]) => { + const na = (a && a.name) ? String(a.name) : ''; + const nb = (b && b.name) ? String(b.name) : ''; + return na.localeCompare(nb); + }) + .forEach(([id, info]) => { + + if (info.proxy === true) { + const opt = document.createElement('option'); + opt.value = id; + opt.textContent = (info && info.name) ? info.name : id; + chartSelect.appendChild(opt); + } + }); + + // if no maps returned, add a fallback option + if (!chartSelect.options.length) { + const opt = document.createElement('option'); + opt.value = ''; + opt.textContent = '-- No maps available --'; + chartSelect.appendChild(opt); + } + } catch (err) { + console.error('Error loading maps:', err); + } + + document.getElementById('createJobButton').addEventListener('click', async () => { + try { + const chart = document.getElementById('chart').value; + const regionGUID = document.getElementById('region').value; + const maxZoom = document.getElementById('maxZoom').value; + const bbox = { + minLon: parseFloat(document.getElementById('bboxMinLon').value), + minLat: parseFloat(document.getElementById('bboxMinLat').value), + maxLon: parseFloat(document.getElementById('bboxMaxLon').value), + maxLat: parseFloat(document.getElementById('bboxMaxLat').value) + }; + // const tile = { + // z: parseInt(document.getElementById('tileZ').value), + // x: parseInt(document.getElementById('tileX').value), + // y: parseInt(document.getElementById('tileY').value), + + // }; + // console.log(tile); + const type = document.querySelector('input[name="inputType"]:checked').value; + const body = {}; + if (type === 'region') { + body.regionGUID = regionGUID; + } else if (type === 'bbox') { + body.bbox = bbox; + } + // else if (type === 'tile') { + // body.tile = tile; + // } + body.maxZoom = maxZoom; + const resp = await fetch(`/signalk/chart-tiles/cache/${chart}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(body) + }); + if (!resp.ok) throw new Error(`Failed to seed charts: ${resp.status}`); + const data = await resp.json(); + } catch (err) { + console.error('Error seeding charts:', err); + } + }); + + const fmt = n => (typeof n === 'number' ? n.toLocaleString() : '-'); + const statusText = s => { + switch (s) { + case 0: return 'Stopped'; + case 1: return 'Running'; + default: return String(s); + } + }; + + function renderJobs(items) { + const tbody = document.querySelector('#activeJobsTable tbody'); + if (!tbody) return; + tbody.innerHTML = ''; + if (!Array.isArray(items) || items.length === 0) { + const tr = document.createElement('tr'); + tr.innerHTML = 'No active jobs'; + tbody.appendChild(tr); + return; + } + + items.forEach((it, idx) => { + const tr = document.createElement('tr'); + + const pct = (typeof it.progress === 'number' && isFinite(it.progress)) ? Math.max(0, Math.min(100, it.progress * 100)) : 0; + const pctText = `${pct.toFixed(1)}%`; + tr.innerHTML = [ + `${idx + 1}`, + `${it.chartName || '-'}`, + `${it.regionName || '-'}`, + `${fmt(it.totalTiles)}`, + `${fmt(it.downloadedTiles)}`, + `${fmt(it.cachedTiles)}`, + `${fmt(it.failedTiles)}`, + ` + + ${pctText} + `, + `${statusText(it.status)}`, + ` + + + + + + ` + ].join(''); + + tr.querySelectorAll('.startstop-btn').forEach(button => { + button.addEventListener('click', () => { + const playIcon = button.querySelector('.icon-play'); + const pauseIcon = button.querySelector('.icon-pause'); + const jobId = button.getAttribute('data-id'); + const totalTiles = button.getAttribute('data-totalTiles'); + const isRunning = pauseIcon.classList.contains('hidden'); + console.log(isRunning) + if (isRunning) { + // Switch to stop + if (totalTiles > 100000 && !confirm(`This job has more than ${totalTiles} tiles to download. Starting it may take a long time and put high load on the server. Are you sure you want to start it?`)) { + return; + } + takeAction(jobId, 'start'); + button.title = 'Stop'; + } else { + // Switch to start + takeAction(jobId, 'stop'); + button.title = 'Start'; + } + }); + }); + + tr.querySelectorAll('.delete-btn').forEach(button => { + button.addEventListener('click', () => { + if (confirm("This will delete all cached tiles for this job. Are you sure?")) { + const jobId = button.getAttribute('data-id'); + if (jobId) { + takeAction(jobId, 'delete'); + } + } + }); + }); + + tr.querySelectorAll('.remove-btn').forEach(button => { + button.addEventListener('click', () => { + const jobId = button.getAttribute('data-id'); + if (jobId) { + takeAction(jobId, 'remove'); + } + }); + }); + + tbody.appendChild(tr); + }); + + + + function takeAction(jobId, action) { + fetch(`/signalk/chart-tiles/cache/jobs/${jobId}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + action: action, + }) + }) + .then(response => { + if (!response.ok) { + throw new Error(`Failed to ${action} job ${jobId}: ${response.status}`); + } + fetchActiveJobs(); + }) + .catch(error => { + console.error(`Error ${action} job:`, error); + }); + } + } + + + + async function fetchActiveJobs() { + const tbody = document.querySelector('#activeJobsTable tbody'); + try { + const resp = await fetch('/signalk/chart-tiles/cache/jobs'); + if (!resp.ok) throw new Error('HTTP ' + resp.status); + const data = await resp.json(); + renderJobs(data); + } catch (err) { + console.error('Error fetching active jobs:', err); + tbody.innerHTML = 'Error loading active jobs'; + } + } + + + + fetchActiveJobs(); + // refresh every 2 seconds + setInterval(fetchActiveJobs, 2000); + + +}); + diff --git a/public/style.css b/public/style.css new file mode 100644 index 0000000..5bd8509 --- /dev/null +++ b/public/style.css @@ -0,0 +1,116 @@ +#activeJobsTable { + width: 100%; + border-collapse: collapse; + font-family: Arial, sans-serif; + margin-top: 8px; +} + +#activeJobsTable th, +#progressTable td { + border: 1px solid #ddd; + padding: 6px 8px; + text-align: left; + vertical-align: middle; + font-size: 13px; +} + +#activeJobsTable th { + background: #f4f4f4; + font-weight: 600; +} + +.progress-bar { + width: 160px; + height: 18px; + background: #eee; + border-radius: 6px; + overflow: hidden; + display: inline-block; + vertical-align: middle; + margin-right: 8px; +} + +.progress-fill { + display: inline-block; + width: 0%; + height: 100%; + box-sizing: border-box; + background: linear-gradient(90deg, #4caf50, #2e7d32); + transition: width 300ms ease; +} + +.percent { + font-size: 12px; + color: #333; + vertical-align: middle; +} + +.muted { + color: #666; + font-size: 12px; +} + +.action-buttons { + display: flex; + gap: 8px; + justify-content: center; + align-items: center; +} + +.btn { + display: flex; + justify-content: center; + align-items: center; + width: 36px; + height: 36px; + border: none; + border-radius: 8px; + cursor: pointer; + background-color: #444; + transition: transform 0.15s ease, background-color 0.2s ease; + padding: 0; + line-height: 0; +} + +.btn:hover { + transform: scale(1.1); +} + +.icon { + width: 20px; + height: 20px; + fill: white; + display: block; + /* ensures true centering */ + vertical-align: middle; + stroke-linecap: round; + stroke-linejoin: round; +} + +.startstop-btn { + background-color: #27ae60; +} + +.startstop-btn.running { + background-color: #c0392b; +} + +.delete-btn { + background-color: #e74c3c; +} + +.delete-btn:hover { + background-color: #c0392b; +} + +.remove-btn { + background-color: #7f8c8d; +} + +.remove-btn:hover { + background-color: #95a5a6; +} + +.hidden { + display: none; +} \ No newline at end of file diff --git a/src/@types/geojson-antimeridian-cut.d.ts b/src/@types/geojson-antimeridian-cut.d.ts new file mode 100644 index 0000000..9c61ed5 --- /dev/null +++ b/src/@types/geojson-antimeridian-cut.d.ts @@ -0,0 +1,6 @@ +declare module 'geojson-antimeridian-cut' { + import { Feature, FeatureCollection, Geometry } from 'geojson'; + export default function splitGeoJSON | FeatureCollection>( + geojson: T + ): T; +} \ No newline at end of file diff --git a/src/chartDownloader.ts b/src/chartDownloader.ts index d43673d..54d8c2e 100644 --- a/src/chartDownloader.ts +++ b/src/chartDownloader.ts @@ -1,120 +1,185 @@ import fs from "fs"; import path from "path"; import pLimit from "p-limit" +import type { BBox, FeatureCollection, Polygon, MultiPolygon, Feature, Position } from 'geojson' +import splitGeoJSON from 'geojson-antimeridian-cut'; import booleanIntersects from '@turf/boolean-intersects' -import type { BBox, FeatureCollection, Polygon } from 'geojson' +import { bbox } from '@turf/bbox' import { polygon } from '@turf/helpers' +import checkDiskSpace from "check-disk-space"; +import { ChartProvider } from "./types"; -interface Tile { +export interface Tile { x: number y: number z: number } -export enum DownloadStatus { - NotStarted, - Preparing, - Downloading, - Completed, - Cancelled +export enum Status { + Stopped, + Running, + +} + +export class ChartSeedingManager { + // Placeholder for future cache management methods + public static ActiveJobs: { [key: number]: ChartDownloader } = {}; + + public static async createJob(urlBase: string, chartsPath: string, provider: any, maxZoom: number, regionGUI: string | undefined = undefined, bbox: BBox | undefined = undefined, tile: Tile | undefined = undefined): Promise { + const downloader = new ChartDownloader(urlBase, chartsPath, provider); + if (regionGUI) + downloader.initalizeJobFromRegion(regionGUI, maxZoom); + else if (bbox) + downloader.initializeJobFromBBox(bbox, maxZoom); + else if (tile) { + downloader.initializeJobFromTile(tile, maxZoom); + } + this.ActiveJobs[downloader.ID] = downloader; + return downloader; + } + + public static registerRoutes(app: any){ + + } } export class ChartDownloader { - public static ActiveDownloads: { [key: number]: ChartDownloader } = {}; + private static DISK_USAGE_LIMIT = 1024 * 1024 * 1024; // 1 GB private static nextJobId = 1; - private downloadStatus: DownloadStatus = DownloadStatus.NotStarted; - private progress: number = 0; + private id : number = ChartDownloader.nextJobId++; + private maxZoom : number = 15; + private status: Status = Status.Stopped; private totalTiles: number = 0; private downloadedTiles: number = 0; private failedTiles: number = 0; private cachedTiles: number = 0; private concurrentDownloadsLimit = 20; - private regionName: string = ""; + private areaDescription: string = ""; private cancelRequested: boolean = false; + private tiles: Tile[] = []; + private tilesToDownload: Tile[] = []; - constructor(private urlBase: string, private chartsPath: string, private provider: any) {} - public static createAndRegister(urlBase: string, chartsPath: string, provider: any): { jobId: number, downloader: ChartDownloader } { - const downloader = new ChartDownloader(urlBase, chartsPath, provider); - const jobId = this.nextJobId++; - this.ActiveDownloads[jobId] = downloader; - return { jobId, downloader }; + constructor(private urlBase: string, private chartsPath: string, private provider: any) { + } + get ID(): number { + return this.id; + } - /** - * Download map tiles for a specific region. - * @param region - * @param maxZoom Maximum zoom level to download - */ - async downloadTiles(regionGUID: string, maxZoom: number): Promise { - this.downloadStatus = DownloadStatus.Preparing; + + public async initalizeJobFromRegion(regionGUID: string, maxZoom: number): Promise { const region = await this.getRegion(regionGUID); const geojson = this.convertRegionToGeoJSON(region); - const tiles = this.getTilesForGeoJSON(geojson, this.provider.minzoom, maxZoom); - const tileToDownload = await this.filterCachedTiles(tiles); - - this.totalTiles = tiles.length; - this.downloadedTiles = 0; - this.cachedTiles = this.totalTiles - tileToDownload.length; - this.regionName = region.name || ""; + this.tiles = this.getTilesForGeoJSON(geojson, this.provider.minzoom, maxZoom); + this.tilesToDownload = this.filterCachedTiles(this.tiles); + + this.status = Status.Stopped; + this.totalTiles = this.tiles.length; + this.cachedTiles = this.totalTiles - this.tilesToDownload.length; + this.areaDescription = `Region: ${region.name || ""}`; + this.maxZoom = maxZoom; + } + + public async initializeJobFromBBox(bbox: BBox, maxZoom: number): Promise { + this.tiles = this.getTilesForBBox(bbox, maxZoom); + this.tilesToDownload = this.filterCachedTiles(this.tiles); + + this.status = Status.Stopped; + this.totalTiles = this.tiles.length; + this.cachedTiles = this.totalTiles - this.tilesToDownload.length; + this.areaDescription = `BBox: [${bbox.join(", ")}]`; + this.maxZoom = maxZoom; + } - this.downloadStatus = DownloadStatus.Downloading; + public async initializeJobFromTile(tile: Tile, maxZoom: number): Promise { + this.tiles = this.getSubTiles(tile, maxZoom); + this.tilesToDownload = this.filterCachedTiles(this.tiles); + + this.status = Status.Stopped; + this.totalTiles = this.tiles.length; + this.cachedTiles = this.totalTiles - this.tilesToDownload.length; + this.areaDescription = `Tile: [${tile.x}, ${tile.y}, ${tile.z}]`; + this.maxZoom = maxZoom; + } + + /** + * Download map tiles for a specific area. + * + */ + async seedCache(): Promise { + this.cancelRequested = false; + this.status = Status.Running; + this.tilesToDownload = await this.filterCachedTiles(this.tiles); + this.downloadedTiles = 0; + this.cachedTiles = this.totalTiles - this.tilesToDownload.length; const limit = pLimit(this.concurrentDownloadsLimit); // concurrent download limit const promises: Promise[] = []; - for (const tile of tileToDownload) { + let tileCounter = 0 + this.tilesToDownload = await this.filterCachedTiles(this.tiles); + for (const tile of this.tilesToDownload) { if (this.cancelRequested) break; + if (tileCounter % 1000 === 0 && tileCounter > 0) { + try { + const { free } = await checkDiskSpace(this.chartsPath) + if (free < ChartDownloader.DISK_USAGE_LIMIT) { + console.warn(`Low disk space. Stopping download.`); + break; + } + } catch (err) { + console.error(`Error checking disk space:`, err); + break; + } + } promises.push(limit(async () => { - if (this.cancelRequested) return; - const buffer = await ChartDownloader.fetchTile(this.chartsPath, this.provider, tile); - if (this.cancelRequested) return; + if (this.cancelRequested) + return; + const buffer = await ChartDownloader.getTileFromCacheOrRemote(this.chartsPath, this.provider, tile); if (buffer === null) { this.failedTiles += 1; } else { this.downloadedTiles += 1; } })); + tileCounter++ } try { await Promise.all(promises); - + } catch (err) { // silent failure, caller can log if needed console.error(`Error downloading tiles:`, err); } - if (this.cancelRequested) { - this.downloadStatus = DownloadStatus.Cancelled; - return; - } - this.downloadStatus = DownloadStatus.Completed; + this.status = Status.Stopped; } - async deleteTiles(regionGUID: string): Promise { - const region = await this.getRegion(regionGUID); - const geojson = this.convertRegionToGeoJSON(region); - const tiles = this.getTilesForGeoJSON(geojson, this.provider.minzoom, this.provider.maxzoom); - for (const tile of tiles) { + async deleteCache(): Promise { + this.status = Status.Running; + for (const tile of this.tiles) { if (this.cancelRequested) break; const tilePath = path.join(this.chartsPath, `${this.provider.name}`, `${tile.z}`, `${tile.x}`, `${tile.y}.${this.provider.format}`); if (fs.existsSync(tilePath)) { try { - await fs.promises.unlink(tilePath); + fs.promises.unlink(tilePath); + this.cachedTiles -= 1; + this.cachedTiles = Math.max(this.cachedTiles, 0); } catch (err) { console.error(`Error deleting cached tile ${tilePath}:`, err); } } } - this.downloadStatus = DownloadStatus.Completed; + this.status = Status.Stopped; } - public cancelDownload() { + public cancelJob() { this.cancelRequested = true; } - async filterCachedTiles(allTiles: Tile[]): Promise { + private filterCachedTiles(allTiles: Tile[]): Tile[] { const uncachedTiles: Tile[] = []; for (const tile of allTiles) { const tilePath = path.join(this.chartsPath, `${this.provider.name}`, `${tile.z}`, `${tile.x}`, `${tile.y}.${this.provider.format}`); @@ -125,20 +190,21 @@ export class ChartDownloader { return uncachedTiles; } - public progressInfo(){ + public info() { return { + id: this.id, chartName: this.provider.name, - regionName: this.regionName, + regionName: this.areaDescription, totalTiles: this.totalTiles, downloadedTiles: this.downloadedTiles, cachedTiles: this.cachedTiles, failedTiles: this.failedTiles, - progress: this.totalTiles > 0 ? (this.downloadedTiles + this.cachedTiles) / this.totalTiles : 0, - status: this.downloadStatus + progress: this.totalTiles > 0 ? (this.downloadedTiles + this.cachedTiles + this.failedTiles) / this.totalTiles : 0, + status: this.status }; } - static async fetchTile(chartsPath: string, provider: any, tile: Tile): Promise { + static async getTileFromCacheOrRemote(chartsPath: string, provider: any, tile: Tile): Promise { const tilePath = path.join(chartsPath, `${provider.name}`, `${tile.z}`, `${tile.x}`, `${tile.y}.${provider.format}`); if (fs.existsSync(tilePath)) { try { @@ -148,10 +214,17 @@ export class ChartDownloader { console.error(`Error reading cached tile ${tilePath}:`, err); } } - if (!provider.remoteUrl) { - console.error(`No remote URL defined for cached provider ${provider.name}`); - return null; + const buffer = await this.fetchTileFromRemote(provider, tile); + if (buffer) { + if (!fs.existsSync(path.dirname(tilePath))) { + fs.mkdirSync(path.dirname(tilePath), { recursive: true }); + } + await fs.promises.writeFile(tilePath, buffer); } + return buffer; + } + + static async fetchTileFromRemote(provider: any, tile: Tile): Promise { const url = provider.remoteUrl .replace("{z}", tile.z.toString()) .replace("{x}", tile.x.toString()) @@ -165,13 +238,65 @@ export class ChartDownloader { } const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); - if (!fs.existsSync(path.dirname(tilePath))) { - fs.mkdirSync(path.dirname(tilePath), { recursive: true }); + return buffer + } + + getSubTiles(tile: Tile, maxZoom: number): Tile[] { + const tiles: Tile[] = [tile]; + + for (let z = tile.z + 1; z <= maxZoom; z++) { + const zoomDiff = z - tile.z; + const factor = Math.pow(2, zoomDiff); + + const startX = tile.x * factor; + const startY = tile.y * factor; + + for (let x = startX; x < startX + factor; x++) { + for (let y = startY; y < startY + factor; y++) { + tiles.push({ x, y, z }); + } + } } - await fs.promises.writeFile(tilePath, buffer); - return buffer; + + return tiles; } + /** + * Get all tiles that intersect a bounding box up to a maximum zoom level. + * bbox = [minLon, minLat, maxLon, maxLat] + */ + getTilesForBBox(bbox: BBox, maxZoom: number): Tile[] { + const tiles: Tile[] = []; + let [minLon, minLat, maxLon, maxLat] = bbox; + + const crossesAntiMeridian = minLon > maxLon; + + // Helper to process a lon/lat box normally + const processBBox = (lo1: number, la1: number, lo2: number, la2: number) => { + for (let z = 0; z <= maxZoom; z++) { + const [minX, maxY] = this.lonLatToTileXY(lo1, la1, z); + const [maxX, minY] = this.lonLatToTileXY(lo2, la2, z); + + for (let x = minX; x <= maxX; x++) { + for (let y = minY; y <= maxY; y++) { + tiles.push({ x, y, z }); + } + } + } + }; + + if (!crossesAntiMeridian) { + // normal + processBBox(minLon, minLat, maxLon, maxLat); + } else { + // crosses antimeridian — split into two boxes: + // [minLon -> 180] and [-180 -> maxLon] + processBBox(minLon, minLat, 180, maxLat); + processBBox(-180, minLat, maxLon, maxLat); + } + + return tiles; + } getTilesForGeoJSON( geojson: FeatureCollection, @@ -185,11 +310,11 @@ export class ChartDownloader { console.warn("Skipping non-polygon feature"); continue; } - - const bbox = this.getBBox(feature.geometry as Polygon); + + const boundingBox = bbox(feature.geometry as Polygon); // [minX, minY, maxX, maxY] for (let z = zoomMin; z <= zoomMax; z++) { - const [minX, minY] = this.lonLatToTileXY(bbox[0], bbox[3], z); // top-left - const [maxX, maxY] = this.lonLatToTileXY(bbox[2], bbox[1], z); // bottom-right + const [minX, minY] = this.lonLatToTileXY(boundingBox[0], boundingBox[3], z); // top-left + const [maxX, maxY] = this.lonLatToTileXY(boundingBox[2], boundingBox[1], z); // bottom-right for (let x = minX; x <= maxX; x++) { for (let y = minY; y <= maxY; y++) { @@ -207,7 +332,8 @@ export class ChartDownloader { return tiles; } - async getRegion(regionGUID: string): Promise> { + + private async getRegion(regionGUID: string): Promise> { let regionData: any const resp = await fetch(`${this.urlBase}/signalk/v2/api/resources/regions/${regionGUID}`) if (!resp.ok) { @@ -220,7 +346,7 @@ export class ChartDownloader { } - convertRegionToGeoJSON(region: Record): FeatureCollection { + private convertRegionToGeoJSON(region: Record): FeatureCollection { const feature = region.feature; if (!feature || feature.type !== "Feature" || !feature.geometry) { throw new Error("Invalid region: missing feature or geometry"); @@ -238,24 +364,35 @@ export class ChartDownloader { ...feature.properties, }, }; - - return { - type: "FeatureCollection" as const, - features: [geoFeature], + const splitGeoFeature = splitGeoJSON(geoFeature); + const features: Feature[] = []; + + const pushFeaturePolygon = (orig: Feature, coords: Position[][], idx?: number) => { + const poly: Feature = { + type: "Feature", + id: idx != null && orig.id ? `${orig.id}-${idx}` : orig.id, + geometry: { + type: "Polygon", + coordinates: coords + }, + properties: orig.properties || {} + }; + features.push(poly); }; - } - private getBBox(geometry: Polygon): BBox { - let minLon = 180, minLat = 90, maxLon = -180, maxLat = -90; - for (const ring of geometry.coordinates) { - for (const [lon, lat] of ring) { - minLon = Math.min(minLon, lon); - minLat = Math.min(minLat, lat); - maxLon = Math.max(maxLon, lon); - maxLat = Math.max(maxLat, lat); + const f = splitGeoFeature as Feature; + if (f.geometry && f.geometry.type === "MultiPolygon") { + for (let i = 0; i < (f.geometry as MultiPolygon).coordinates.length; i++) { + pushFeaturePolygon(f, (f.geometry as MultiPolygon).coordinates[i], i); } + } else if (f.geometry && f.geometry.type === "Polygon") { + features.push(f as Feature); } - return [minLon, minLat, maxLon, maxLat]; + + return { + type: "FeatureCollection" as const, + features + }; } private lonLatToTileXY(lon: number, lat: number, zoom: number): [number, number] { diff --git a/src/index.ts b/src/index.ts index 5e6ea50..d03d8b6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,7 @@ import * as _ from 'lodash' import { findCharts } from './charts' import { apiRoutePrefix } from './constants' import { ChartProvider, OnlineChartProvider } from './types' -import { ChartDownloader } from './chartDownloader' +import { ChartSeedingManager, ChartDownloader, Tile } from './chartDownloader' import { Request, Response, Application } from 'express' import { OutgoingHttpHeaders } from 'http' import { @@ -264,7 +264,7 @@ module.exports = (app: ChartProviderApp): Plugin => { return res.sendStatus(404) } if (provider.proxy === true) { - return serveTileFromCache(res, provider, iz, ix, iy) + return serveTileFromCacheOrRemote(res, provider, iz, ix, iy) } else { @@ -295,53 +295,58 @@ module.exports = (app: ChartProviderApp): Plugin => { } ) - app.post(`${chartTilesPath}/cache/:identifier/:regionGUID/:maxZoom`, async (req: Request, res: Response) => { - const { identifier, regionGUID, maxZoom } = req.params + app.post(`${chartTilesPath}/cache/:identifier`, async (req: Request, res: Response) => { + const { identifier } = req.params + const { regionGUID, tile, bbox, maxZoom } = req.body as { + regionGUID?: string; + tile?: Tile; // query params come in as strings + bbox?: { + minLon: number; + minLat: number; + maxLon: number; + maxLat: number; + }; + maxZoom?: string; + }; const provider = chartProviders[identifier] if (!provider) { - return res.sendStatus(500) + return res.sendStatus(500).send("Provider not found") } - const maxZoomParsed = maxZoom ? parseInt(maxZoom) : 17 - - const { jobId, downloader } = ChartDownloader.createAndRegister(urlBase, defaultChartsPath, provider) - ;(async () => { - await downloader.downloadTiles(regionGUID, maxZoomParsed) - })() - return res.status(200).json({ progress: `${chartTilesPath}/progress/${jobId}` }) - }) - - app.delete(`${chartTilesPath}/cache/:identifier/:regionGUID`, async (req: Request, res: Response) => { - const { identifier, regionGUID } = req.params - const provider = chartProviders[identifier] - if (!provider) { - return res.sendStatus(500) + if (!maxZoom) { + return res.status(400).send('maxZoom parameter is required') } - - const { jobId, downloader } = ChartDownloader.createAndRegister(urlBase, defaultChartsPath, provider) - // Long going process, create an endpoint to monitor progress - ;(async () => { - await downloader.deleteTiles(regionGUID) - })() - return res.status(200).json({ progress: `${chartTilesPath}/cache/progress/${jobId}` }) + const maxZoomParsed = parseInt(maxZoom) + const job = await ChartSeedingManager.createJob(urlBase, defaultChartsPath, provider, maxZoomParsed, regionGUID, bbox ? [bbox.minLon, bbox.minLat, bbox.maxLon, bbox.maxLat] : undefined, tile ) + return res.status(200) }) - app.get(`${chartTilesPath}/cache/progress`, (req: Request, res: Response) => { - const progresses = Object.entries(ChartDownloader.ActiveDownloads).map(([id, job]) => { - return job.progressInfo() + app.get(`${chartTilesPath}/cache/jobs`, (req: Request, res: Response) => { + const jobs = Object.entries(ChartSeedingManager.ActiveJobs).map(([id, job]) => { + return job.info() }) - return res.status(200).json(progresses) - + return res.status(200).json(jobs) + }) - app.put(`${chartTilesPath}/cache/progress/:id/cancel`, (req: Request, res: Response) => { + app.post(`${chartTilesPath}/cache/jobs/:id`, (req: Request, res: Response) => { const { id } = req.params + const { action } = req.body as { action: string } const parsedId = parseInt(id) - const job = ChartDownloader.ActiveDownloads[parsedId] - if (job) { - job.cancelDownload() - return res.status(200).send(`Cancelled job ${parsedId}`) - } else { - return res.status(404).send(`Job ${parsedId} not found`) + const job = ChartSeedingManager.ActiveJobs[parsedId] + if (job && action) { + if (action === 'start') { + job.seedCache() + } else if (action === 'stop') { + job.cancelJob() + } else if (action === 'delete') { + job.deleteCache() + } else if (action === 'remove') { + delete ChartSeedingManager.ActiveJobs[parsedId] + } + else { + return res.status(404).send(`Job ${parsedId} not found`) + } + return res.status(200).send(`Job ${parsedId} ${action}ed`) } }) @@ -408,14 +413,14 @@ module.exports = (app: ChartProviderApp): Plugin => { } } - const serveTileFromCache = async ( + const serveTileFromCacheOrRemote = async ( res: Response, provider: ChartProvider, z: number, x: number, y: number ) => { - const buffer = await ChartDownloader.fetchTile(defaultChartsPath, provider, { x, y, z }) + const buffer = await ChartDownloader.getTileFromCacheOrRemote(defaultChartsPath, provider, { x, y, z }) if (!buffer) { res.sendStatus(500) return @@ -446,21 +451,21 @@ const resolveUniqueChartPaths = ( const convertOnlineProviderConfig = (provider: OnlineChartProvider, urlBase: string) => { const id = _.kebabCase(_.deburr(provider.name)) - const parseHeaders = (arr?: string[] | null): { [key: string]: string } => { - const result: { [key: string]: string } = {} - if (!arr) return result - for (const entry of arr) { - if (typeof entry !== 'string') continue - const idx = entry.indexOf(':') - if (idx === -1) continue - const key = entry.slice(0, idx).trim() - const value = entry.slice(idx + 1).trim() - if (key) result[key] = value + const parseHeaders = (arr: string[] | undefined): { [key: string]: string } => { + if (arr === undefined) { + return {} } - return result + return arr.reduce<{ [key: string]: string }>((acc, entry) => { + if(typeof entry == 'string') { + const [key, value] = entry.split(':').map((s) => s.trim()) + if (key && value) { + acc[key] = value + } + } + return acc + }, {}) } - const data = { identifier: id, name: provider.name, From f16dd199de86de8fdc1fe4ed8333938efde01486 Mon Sep 17 00:00:00 2001 From: Batu Akan Date: Sat, 1 Nov 2025 05:29:43 +0100 Subject: [PATCH 7/8] Small fix for header parsing --- src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index d03d8b6..fd19a9b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -457,7 +457,9 @@ const convertOnlineProviderConfig = (provider: OnlineChartProvider, urlBase: str } return arr.reduce<{ [key: string]: string }>((acc, entry) => { if(typeof entry == 'string') { - const [key, value] = entry.split(':').map((s) => s.trim()) + const idx = entry.indexOf(':'); + const key = entry.slice(0, idx).trim(); + const value = entry.slice(idx + 1).trim(); if (key && value) { acc[key] = value } From 2d6b019653f3013fc56192b74cde6c950b8558bb Mon Sep 17 00:00:00 2001 From: Batu Akan Date: Sat, 1 Nov 2025 11:29:32 +0100 Subject: [PATCH 8/8] Fixed build issue after merge --- src/chartDownloader.ts | 1 + src/index.ts | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/chartDownloader.ts b/src/chartDownloader.ts index 54d8c2e..070ad71 100644 --- a/src/chartDownloader.ts +++ b/src/chartDownloader.ts @@ -116,6 +116,7 @@ export class ChartDownloader { this.status = Status.Running; this.tilesToDownload = await this.filterCachedTiles(this.tiles); this.downloadedTiles = 0; + this.failedTiles = 0; this.cachedTiles = this.totalTiles - this.tilesToDownload.length; const limit = pLimit(this.concurrentDownloadsLimit); // concurrent download limit const promises: Promise[] = []; diff --git a/src/index.ts b/src/index.ts index fd19a9b..6b6bced 100644 --- a/src/index.ts +++ b/src/index.ts @@ -221,10 +221,6 @@ module.exports = (app: ChartProviderApp): Plugin => { registerAsProvider() } - const urlBase = `${app.config.ssl ? 'https' : 'http'}://localhost:${ - 'getExternalPort' in app.config ? app.config.getExternalPort() : 3000 - }` - app.debug(`**urlBase** ${urlBase}`) app.setPluginStatus('Started') const loadProviders = bluebird