diff --git a/package-lock.json b/package-lock.json index ebb1e3aa78..bb900fc07a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4011,6 +4011,13 @@ "node": ">=8.0.0" } }, + "node_modules/@petamoriken/float16": { + "version": "3.9.2", + "resolved": "https://registry.npmjs.org/@petamoriken/float16/-/float16-3.9.2.tgz", + "integrity": "sha512-VgffxawQde93xKxT3qap3OH+meZf7VaSB5Sqd4Rqc+FP5alWbpOyan/7tRbOAvynjpG3GpdtAuGU/NdhQpmrog==", + "license": "MIT", + "optional": true + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -4255,6 +4262,12 @@ "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0" } }, + "node_modules/@repeaterjs/repeater": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@repeaterjs/repeater/-/repeater-3.0.6.tgz", + "integrity": "sha512-Javneu5lsuhwNCryN+pXH93VPQ8g0dBX7wItHFgYiwQmzE1sVdg5tWHiOgHywzL2W21XQopa7IwIEnNbmeUJYA==", + "license": "MIT" + }, "node_modules/@rushstack/ts-command-line": { "version": "4.11.1", "resolved": "https://registry.npmjs.org/@rushstack/ts-command-line/-/ts-command-line-4.11.1.tgz", @@ -5436,6 +5449,13 @@ "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==", "dev": true }, + "node_modules/@types/rbush": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/rbush/-/rbush-4.0.0.tgz", + "integrity": "sha512-+N+2H39P8X+Hy1I5mC6awlTX54k3FhiUmvt7HWzGJZvF+syUAAxP/stwppS8JE84YHqFgRMv6fCy31202CMFxQ==", + "license": "MIT", + "optional": true + }, "node_modules/@types/react": { "version": "18.2.43", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.43.tgz", @@ -8819,7 +8839,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/earcut/-/earcut-3.0.0.tgz", "integrity": "sha512-41Fs7Q/PLq1SDbqjsgcY7GA42T0jvaCNGXgGtsNdvg+Yv8eIu06bxv4/PoREkZ9nMDNwnUSG9OFB9+yv8eKhDg==", - "dev": true + "devOptional": true }, "node_modules/eastasianwidth": { "version": "0.2.0", @@ -10055,9 +10075,24 @@ } }, "node_modules/flatbuffers": { - "version": "24.3.25", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-24.3.25.tgz", - "integrity": "sha512-3HDgPbgiwWMI9zVB7VYBHaMrbOO7Gm0v+yD2FV/sCKj+9NDeVL7BOBYUuhWAQGKWOzBo8S9WdMvV0eixO233XQ==" + "version": "24.12.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-24.12.23.tgz", + "integrity": "sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==", + "license": "Apache-2.0" + }, + "node_modules/flatgeobuf": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flatgeobuf/-/flatgeobuf-4.0.1.tgz", + "integrity": "sha512-KhK+34kFaCw1knOSq1vAq3eWf3UzsNlQ38+ursdQMG2iWmCV67RsovrLZKG29XHysQVLIoFHaV7dLEbdUpLwcw==", + "license": "BSD-3-Clause", + "dependencies": { + "@repeaterjs/repeater": "3.0.6", + "flatbuffers": "24.12.23", + "slice-source": "0.4.1" + }, + "optionalDependencies": { + "ol": ">=3" + } }, "node_modules/flatted": { "version": "3.3.1", @@ -10356,6 +10391,46 @@ "integrity": "sha512-AV9ROqlNqoZEIJGfm1ncNjEXfkz2hdFlZf0qkVfmkwdKa8vj7H16YUOT81rJw1rdFhyEDlN2Tds91p/glzbl5A==", "dev": true }, + "node_modules/geotiff": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/geotiff/-/geotiff-2.1.3.tgz", + "integrity": "sha512-PT6uoF5a1+kbC3tHmZSUsLHBp2QJlHasxxxxPW47QIY1VBKpFB+FcDvX+MxER6UzgLQZ0xDzJ9s48B9JbOCTqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@petamoriken/float16": "^3.4.7", + "lerc": "^3.0.0", + "pako": "^2.0.4", + "parse-headers": "^2.0.2", + "quick-lru": "^6.1.1", + "web-worker": "^1.2.0", + "xml-utils": "^1.0.2", + "zstddec": "^0.1.0" + }, + "engines": { + "node": ">=10.19" + } + }, + "node_modules/geotiff/node_modules/lerc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lerc/-/lerc-3.0.0.tgz", + "integrity": "sha512-Rm4J/WaHhRa93nCN2mwWDZFoRVF18G1f47C+kvQWyHGEZxFpTUi73p7lMVSAndyxGt6lJ2/CFbOcf9ra5p8aww==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/geotiff/node_modules/quick-lru": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-6.1.2.tgz", + "integrity": "sha512-AAFUA5O1d83pIHEhJwWCq/RQcRukCkn/NSm2QsTEMle5f2hP0ChI2+3Xb051PZCkLryI/Ir1MVKviT2FIloaTQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -14947,6 +15022,37 @@ "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==" }, + "node_modules/ol": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/ol/-/ol-10.5.0.tgz", + "integrity": "sha512-nHFx8gkGmvYImsa7iKkwUnZidd5gn1XbMZd9GNOorvm9orjW9gQvT3Naw/MjIasVJ3cB9EJUdCGR2EFAulMHsQ==", + "license": "BSD-2-Clause", + "optional": true, + "dependencies": { + "@types/rbush": "4.0.0", + "earcut": "^3.0.0", + "geotiff": "^2.1.3", + "pbf": "4.0.1", + "rbush": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/openlayers" + } + }, + "node_modules/ol/node_modules/pbf": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pbf/-/pbf-4.0.1.tgz", + "integrity": "sha512-SuLdBvS42z33m8ejRbInMapQe8n0D3vN/Xd5fmWM3tufNgRQFBpaW2YVJxQZV4iPNqb0vEFvssMEo5w9c6BTIA==", + "license": "BSD-3-Clause", + "optional": true, + "dependencies": { + "resolve-protobuf-schema": "^2.1.0" + }, + "bin": { + "pbf": "bin/pbf" + } + }, "node_modules/on-exit-leak-free": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz", @@ -15482,6 +15588,13 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/pako": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", + "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==", + "license": "(MIT AND Zlib)", + "optional": true + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -15502,6 +15615,13 @@ "dev": true, "license": "MIT" }, + "node_modules/parse-headers": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.6.tgz", + "integrity": "sha512-Tz11t3uKztEW5FEVZnj1ox8GKblWn+PvHY9TmJV5Mll2uHEwRdR/5Li1OlXoECjLYkApdhWy44ocONwXLiKO5A==", + "license": "MIT", + "optional": true + }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -16149,7 +16269,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/quickselect/-/quickselect-3.0.0.tgz", "integrity": "sha512-XdjUArbK4Bm5fLLvlm5KpTFOiOThgfWWI4axAZDWg4E/0mKdZyI9tNEfds27qCi1ze/vwTR16kvmmGhRra3c2g==", - "dev": true + "devOptional": true }, "node_modules/range-parser": { "version": "1.2.1", @@ -16161,6 +16281,40 @@ "node": ">= 0.6" } }, + "node_modules/rbush": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/rbush/-/rbush-4.0.1.tgz", + "integrity": "sha512-IP0UpfeWQujYC8Jg162rMNc01Rf0gWMMAb2Uxus/Q0qOFw4lCcq6ZnQEZwUoJqWyUGJ9th7JjwI4yIWo+uvoAQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "quickselect": "^3.0.0" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/react": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", @@ -17634,6 +17788,12 @@ "node": ">=8" } }, + "node_modules/slice-source": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/slice-source/-/slice-source-0.4.1.tgz", + "integrity": "sha512-YiuPbxpCj4hD9Qs06hGAz/OZhQ0eDuALN0lRWJez0eD/RevzKqGdUx1IOMUnXgpr+sXZLq3g8ERwbAH0bCb8vg==", + "license": "BSD-3-Clause" + }, "node_modules/slugify": { "version": "1.6.5", "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.5.tgz", @@ -19055,6 +19215,13 @@ "node": ">= 8" } }, + "node_modules/web-worker": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/web-worker/-/web-worker-1.5.0.tgz", + "integrity": "sha512-RiMReJrTAiA+mBjGONMnjVDP2u3p9R1vkcGz6gDIrOMT3oGuYwX2WRMYI9ipkphSuE5XKEhydbhNEJh4NY9mlw==", + "license": "Apache-2.0", + "optional": true + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -19356,6 +19523,13 @@ "node": ">=6" } }, + "node_modules/xml-utils": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/xml-utils/-/xml-utils-1.10.2.tgz", + "integrity": "sha512-RqM+2o1RYs6T8+3DzDSoTRAUfrvaejbVHcp3+thnAtDKo8LskR+HomLajEy5UjTz24rpka7AxVBRR3g2wTUkJA==", + "license": "CC0-1.0", + "optional": true + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -19449,6 +19623,19 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "node_modules/zod-geojson": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/zod-geojson/-/zod-geojson-0.0.3.tgz", + "integrity": "sha512-swL9Tatws3djCG6yux98SuaIdfa8dHh3lEITNeI8HOCxM26XBgqOVrw4KC5YtNwxE1kmfaJnR8rG+fRc5y7Eog==", + "license": "MIT" + }, + "node_modules/zstddec": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/zstddec/-/zstddec-0.1.0.tgz", + "integrity": "sha512-w2NTI8+3l3eeltKAdK8QpiLo/flRAr2p8AGeakfMZOXBxOg9HIu4LVDxBi81sYgVhFhdJjv1OrB5ssI8uFPoLg==", + "license": "MIT AND BSD-3-Clause", + "optional": true + }, "packages/__tests__": { "name": "@basemaps/test", "version": "8.0.0", @@ -19613,10 +19800,10 @@ "@cotar/builder": "^6.0.1", "@cotar/core": "^6.0.1", "@linzjs/docker-command": "^7.5.0", - "@linzjs/geojson": "^8.0.0", - "cmd-ts": "^0.12.1", - "mustache": "^4.2.0", - "object-sizeof": "^2.6.5", + "@mapbox/geojson-area": "^0.2.2", + "better-sqlite3": "^9.4.3", + "flatgeobuf": "^4.0.1", + "geojson": "^0.5.0", "p-limit": "^6.2.0", "polylabel": "^1.1.0", "stac-ts": "^1.0.0", diff --git a/packages/cli-vector/package.json b/packages/cli-vector/package.json index 4e7ca9bff2..6d9309cb4c 100644 --- a/packages/cli-vector/package.json +++ b/packages/cli-vector/package.json @@ -36,10 +36,17 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, "devDependencies": { - "@types/geojson": "^7946.0.7", - "@types/mustache": "^4.2.6", - "@types/polylabel": "^1.0.5", - "@types/tar-stream": "^2.2.2" + "@basemaps/config": "^8.1.0", + "@basemaps/geo": "^8.0.0", + "@basemaps/shared": "^8.1.0", + "@types/better-sqlite3": "^7.6.9", + "@types/geojson": "^7946.0.16", + "@types/mapbox__geojson-area": "^0.2.6", + "@types/p-limit": "^2.1.0", + "@types/polylabel": "^1.1.3", + "@types/tar-stream": "^2.2.2", + "cmd-ts": "^0.12.1", + "stac-ts": "^1.0.0" }, "publishConfig": { "access": "public" @@ -55,14 +62,14 @@ "@cotar/builder": "^6.0.1", "@cotar/core": "^6.0.1", "@linzjs/docker-command": "^7.5.0", - "@linzjs/geojson": "^8.0.0", - "cmd-ts": "^0.12.1", - "mustache": "^4.2.0", - "object-sizeof": "^2.6.5", + "@mapbox/geojson-area": "^0.2.2", + "better-sqlite3": "^9.4.3", + "flatgeobuf": "^4.0.1", + "geojson": "^0.5.0", "p-limit": "^6.2.0", - "polylabel": "^1.1.0", - "stac-ts": "^1.0.0", + "polylabel": "^2.0.1", "tar-stream": "^2.2.0", - "zod": "^3.24.4" + "zod": "^3.24.4", + "zod-geojson": "^0.0.3" } } diff --git a/packages/cli-vector/src/cli/cli.create.ts b/packages/cli-vector/src/cli/cli.create.ts index 90db511430..4cb39cb36a 100644 --- a/packages/cli-vector/src/cli/cli.create.ts +++ b/packages/cli-vector/src/cli/cli.create.ts @@ -9,7 +9,7 @@ import { createGunzip } from 'zlib'; import { generalize } from '../generalization/generalization.js'; import { Metrics } from '../schema-loader/schema.js'; import { VectorStacItem } from '../stac.js'; -import { ogr2ogrNDJson } from '../transform/ogr2ogr.js'; +import { ogr2ogrfgb } from '../transform/ogr2ogr.js'; import { tileJoin, tippecanoe } from '../transform/tippecanoe.js'; import { ContentType, prepareTmpPaths, TmpPaths } from '../util.js'; @@ -37,6 +37,11 @@ export const CreateArgs = { defaultValue: () => false, defaultValueIsSerializable: true, }), + force: flag({ + long: 'force', + defaultValue: () => false, + defaultValueIsSerializable: true, + }), }; export const CreateCommand = command({ @@ -62,7 +67,7 @@ export const CreateCommand = command({ // generate ndjsons and mbtiles logger.info({ items: items.length }, 'Create Mbtiles Files: Start'); - await Promise.all(items.map((item) => q(() => createMbtilesFile(item, logger)))); + await Promise.all(items.map((item) => q(() => createMbtilesFile(item, args.force, logger)))); logger.info({ items: items.length }, 'Create Mbtiles Files: End'); // if applicable, combine all mbtiles files into a single mbtiles file @@ -167,6 +172,7 @@ async function downloadSourceFile( */ async function createMbtilesFile( { stac, tmpPaths }: { stac: VectorStacItem; tmpPaths: TmpPaths }, + force: boolean, logger: LogType, ): Promise { const options = stac.properties['linz_basemaps:options']; @@ -180,34 +186,34 @@ async function createMbtilesFile( logger.info({ shortbreadLayer, dataset: layer.name }, 'CreateMbtilesFile: Start'); /** - * Convert the source file into an ndjson + * Convert the source file into a flatgeobuf */ logger.info({ source: tmpPaths.source.path, dataset: layer.name }, '[1/5] Convert source file to ndjson: Start'); - if (!(await fsa.exists(tmpPaths.ndjson))) { - await ogr2ogrNDJson(tmpPaths.source.path, tmpPaths.ndjson, layer, logger); + if (!(await fsa.exists(tmpPaths.fgb))) { + await ogr2ogrfgb(tmpPaths.source.path, tmpPaths.fgb, logger); } - logger.info({ destination: tmpPaths.ndjson, dataset: layer.name }, '[1/5] Convert source file to ndjson: End'); + logger.info({ destination: tmpPaths.fgb, dataset: layer.name }, '[1/5] Convert source file to ndjson: End'); /** * Parse the ndjson file and apply the generalization options */ - logger.info({ source: tmpPaths.ndjson, dataset: layer.name }, '[2/5] Generalise ndjson features: Start'); + logger.info({ source: tmpPaths.fgb, dataset: layer.name }, '[2/5] Generalise ndjson features: Start'); let metrics: Metrics | null = null; - if (!(await fsa.exists(tmpPaths.genNdjson))) { - metrics = await generalize(tmpPaths.ndjson, tmpPaths.genNdjson, tileMatrix, options, logger); - if (metrics.output === 0) throw new Error(`Failed to generalize ndjson file ${tmpPaths.ndjson.href}`); + if (force || !(await fsa.exists(tmpPaths.ndjson))) { + metrics = await generalize(tmpPaths.fgb, tmpPaths.ndjson, options, logger); + if (metrics.output === 0) throw new Error(`Failed to generalize ndjson file ${tmpPaths.fgb.href}`); } - logger.info({ destination: tmpPaths.genNdjson, dataset: layer.name }, '[2/5] Generalise ndjson features: End'); + logger.info({ destination: tmpPaths.ndjson, dataset: layer.name }, '[2/5] Generalise ndjson features: End'); /** * Transform the generalized ndjson file to an mbtiles file */ logger.info( - { source: tmpPaths.genNdjson, dataset: layer.name }, + { source: tmpPaths.ndjson, dataset: layer.name }, '[3/5] Transform generalised ndjson into mbtiles: Start', ); - if (!(await fsa.exists(tmpPaths.mbtiles))) { - await tippecanoe(tmpPaths.genNdjson, tmpPaths.mbtiles, layer, tileMatrix, logger); + if (force || !(await fsa.exists(tmpPaths.mbtiles))) { + await tippecanoe(tmpPaths.ndjson, tmpPaths.mbtiles, layer, logger); } logger.info( { destination: tmpPaths.mbtiles, dataset: layer.name }, diff --git a/packages/cli-vector/src/generalization/generalization.ts b/packages/cli-vector/src/generalization/generalization.ts index 201385a252..c2e1ba773f 100644 --- a/packages/cli-vector/src/generalization/generalization.ts +++ b/packages/cli-vector/src/generalization/generalization.ts @@ -1,15 +1,15 @@ import { TileMatrixSet } from '@basemaps/geo'; import { LogType } from '@basemaps/shared'; -import { createWriteStream } from 'fs'; -import { Feature, Geometry, LineString, MultiPolygon, Polygon } from 'geojson'; -import readline from 'readline'; +import { IGeoJsonFeature } from 'flatgeobuf'; +import { geojson } from 'flatgeobuf'; +import { createWriteStream, readFileSync } from 'fs'; +import { Geometry, LineString, MultiPolygon, Polygon } from 'geojson'; import { modifyFeature } from '../modify/modify.js'; import { Metrics, Simplify } from '../schema-loader/schema.js'; import { VectorCreationOptions } from '../stac.js'; import { transformNdJson, transformZoom } from '../transform/nztm.js'; import { VectorGeoFeature } from '../types/VectorGeoFeature.js'; -import { createReadStreamSafe } from '../util.js'; import { Point, simplify } from './simplify.js'; /** @@ -25,35 +25,28 @@ export async function generalize( logger: LogType, ): Promise { logger.info({}, 'Generalize:Start'); - const fileStream = await createReadStreamSafe(input.pathname); + const buffer = readFileSync(input); + const featureStream = geojson.deserialize(buffer); const simplify = options.layer.simplify; - const rl = readline.createInterface({ - input: fileStream, - crlfDelay: Infinity, - }); - const writeStream = createWriteStream(output); let inputCount = 0; let outputCount = 0; - for await (const line of rl) { - if (line === '') continue; - const feature = JSON.parse(line) as Feature; - if (tileMatrix.identifier === 'NZTM2000Quad') transformNdJson(feature); + for await (const feature of featureStream) { inputCount++; // For simplify, duplicate feature for each zoom level with different tolerance if (simplify != null) { for (const s of simplify) { - const vectorGeofeature = tag(tileMatrix, options, feature, s, logger); - if (vectorGeofeature == null) continue; + const result = tag(options, feature, s, logger); + if (result == null) continue; - writeStream.write(JSON.stringify(vectorGeofeature) + '\n'); + writeStream.write(JSON.stringify(result) + '\n'); outputCount++; } } else { - const vectorGeofeature = tag(tileMatrix, options, feature, null, logger); - if (vectorGeofeature == null) continue; + const result = tag(options, feature, null, logger); + if (result == null) continue; writeStream.write(JSON.stringify(vectorGeofeature) + '\n'); outputCount++; @@ -79,12 +72,14 @@ export async function generalize( function tag( tileMatrix: TileMatrixSet, options: VectorCreationOptions, - feature: Feature, + iFeature: IGeoJsonFeature, simplify: Simplify | null, logger: LogType, ): VectorGeoFeature | null { - const vectorGeofeature = { - ...structuredClone(feature), + const feature = { + id: iFeature.id, + geometry: iFeature.geometry, + properties: iFeature.properties, tippecanoe: { layer: options.name, minzoom: options.layer.style.minZoom, diff --git a/packages/cli-vector/src/transform/ogr2ogr.ts b/packages/cli-vector/src/transform/ogr2ogr.ts index 36a6018a81..2a9da805ac 100644 --- a/packages/cli-vector/src/transform/ogr2ogr.ts +++ b/packages/cli-vector/src/transform/ogr2ogr.ts @@ -14,15 +14,8 @@ export async function ogr2ogrNDJson(input: URL, output: URL, layer: Layer, logge cmd.args.push('-f', 'GeoJSONSeq'); cmd.args.push(output.pathname); - cmd.args.push('-t_srs', Epsg.Wgs84.toEpsgString()); - - // Calculate area for lake polygons - if (layer.includeDerivedArea) { - const table = await getTableName(input, logger); - cmd.args.push('-dialect', 'SQLite'); - cmd.args.push('-sql', `SELECT *, ST_Area(geom) AS _derived_area FROM "${table}"`); - } + cmd.args.push('-t_srs', Epsg.Wgs84.toEpsgString()); cmd.args.push(input.pathname); const res = await cmd.run(); @@ -32,25 +25,23 @@ export async function ogr2ogrNDJson(input: URL, output: URL, layer: Layer, logge } } -export async function getTableName(input: URL, logger: LogType): Promise { - const cmd = Command.create('ogrinfo'); +/** + * ogr2ogr flatgeobuf usage, return cmd for ogr2ogr + */ +export async function ogr2ogrfgb(input: URL, output: URL, logger: LogType): Promise { + const cmd = Command.create('ogr2ogr'); + + // format + cmd.args.push('-f', 'FlatGeobuf'); + cmd.args.push(output.pathname); - cmd.args.push('-ro'); - cmd.args.push('-q'); - cmd.args.push('-json'); + // spatial reference system + cmd.args.push('-t_srs', Epsg.Wgs84.toEpsgString()); cmd.args.push(input.pathname); const res = await cmd.run(); - if (res.exitCode !== 0) { logger.fatal({ Gdal: res }, 'Failure'); throw new Error('Gdal failed to run'); } - - const info = JSON.parse(res.stdout) as { layers: { name: string }[] }; - if (info.layers == null || info.layers.length === 0) { - throw new Error(`No layers found in ${input.pathname}`); - } - - return info.layers[0].name; // Return the first layer name } diff --git a/packages/cli-vector/src/util.ts b/packages/cli-vector/src/util.ts index 6a605febaa..acfbd11df0 100644 --- a/packages/cli-vector/src/util.ts +++ b/packages/cli-vector/src/util.ts @@ -38,11 +38,11 @@ export interface TmpPaths { contentType: (typeof ContentType)[keyof typeof ContentType]; }; - /** @example "tmp/create/layers/50248/50248.ndjson" */ - ndjson: URL; + /** @example "tmp/create/layers/50248/50248.fgb" */ + fgb: URL; - /** @example "tmp/create/transform/aerialways/50248/50248-gen.ndjson" */ - genNdjson: URL; + /** @example "tmp/create/transform/aerialways/50248/50248.ndjson" */ + ndjson: URL; /** @example "tmp/create/transform/aerialways/50248/50248.mbtiles" */ mbtiles: URL; @@ -80,8 +80,8 @@ export function prepareTmpPaths( format, contentType: ContentType[format], }, - ndjson: new URL(`${layerId}.ndjson`, LayersDir), - genNdjson: new URL(`${layerId}-gen.ndjson`, TransformDir), + fgb: new URL(`${layerId}.fgb`, LayersDir), + ndjson: new URL(`${layerId}.ndjson`, TransformDir), mbtiles: new URL(`${layerId}.mbtiles`, TransformDir), mbtilesCopy: new URL(path.href.replace(/\.json$/, '.mbtiles')), };