From dda3ddeacbebb548a3ccf2a939961f1ff08556da Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Wed, 14 Jan 2026 21:25:53 +0100 Subject: [PATCH 1/5] Enhance JSONL endpoint with tile metadata and error information - Add errorCode field to TileLayer for numeric error codes - Enhance TileFeatureLayer::toJson() to include: - mapgetTileId (decimal uint64) - mapgetLayerId, mapId - idPrefix (common ID parts) - timestamp (ISO 8601) - ttl (milliseconds) - error object with code and message - Update Python bindings with error_code getter/setter - Document new JSONL response format in mapget-api.md - Add unit tests for enhanced toJson() output --- docs/mapget-api.md | 40 +++++++++++++ libs/model/include/mapget/model/layer.h | 9 +++ libs/model/src/featurelayer.cpp | 53 +++++++++++++++-- libs/model/src/layer.cpp | 18 ++++++ libs/pymapget/binding/py-layer.h | 15 +++++ test/unit/test-model.cpp | 78 +++++++++++++++++++++++++ 6 files changed, 209 insertions(+), 4 deletions(-) diff --git a/docs/mapget-api.md b/docs/mapget-api.md index d03d610c..e4e7ce11 100644 --- a/docs/mapget-api.md +++ b/docs/mapget-api.md @@ -47,6 +47,46 @@ If `Accept-Encoding: gzip` is set, the server compresses responses where possibl JSON Lines is better suited to streaming large responses than a single JSON array. Clients can start processing the first tiles immediately, do not need to buffer the complete response in memory, and can naturally consume the stream with incremental parsers. +### JSONL response format + +Each line in the JSONL response is a GeoJSON-like FeatureCollection with additional metadata: + +```json +{ + "type": "FeatureCollection", + "mapgetTileId": 281479271743500, + "mapId": "EuropeHD", + "mapgetLayerId": "Roads", + "idPrefix": { + "areaId": 123, + "tileId": 456 + }, + "timestamp": "2025-01-14T10:30:00.000000Z", + "ttl": 3600000, + "error": { + "code": 404, + "message": "Error while contacting remote data source: not found" + }, + "features": [...] +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `type` | string | Always `"FeatureCollection"` | +| `mapgetTileId` | integer | The mapget tile ID (64-bit decimal) | +| `mapId` | string | Map identifier | +| `mapgetLayerId` | string | Layer identifier within the map | +| `idPrefix` | object | Common ID parts shared by all features in this tile (optional) | +| `timestamp` | string | ISO 8601 timestamp when the tile was created | +| `ttl` | integer | Time-to-live in milliseconds (optional) | +| `error` | object | Error information if tile creation failed (optional) | +| `error.code` | integer | Numeric error code, e.g., HTTP status or database error (optional) | +| `error.message` | string | Human-readable error message (optional) | +| `features` | array | Array of GeoJSON Feature objects | + +The `error` object is only present if an error occurred while filling the tile. When present, the `features` array may be empty or contain partial data. + ## `/abort` – cancel tile streaming `POST /abort` cancels a running `/tiles` request that was started with a matching `clientId`. It is useful when the viewport changes and the previous stream should be abandoned. diff --git a/libs/model/include/mapget/model/layer.h b/libs/model/include/mapget/model/layer.h index 30677651..5a2acfa5 100644 --- a/libs/model/include/mapget/model/layer.h +++ b/libs/model/include/mapget/model/layer.h @@ -145,6 +145,14 @@ class TileLayer [[nodiscard]] std::optional error() const; void setError(const std::optional& err); + /** + * Getter and setter for 'errorCode' member variable. + * It's used to provide a numeric error code (e.g., HTTP status code, + * SQLite error code) when an error occurred while filling the tile. + */ + [[nodiscard]] std::optional errorCode() const; + void setErrorCode(const std::optional& code); + /** * Getter and setter for 'timestamp' member variable. * It represents when this layer was created. @@ -191,6 +199,7 @@ class TileLayer std::string mapId_; std::shared_ptr layerInfo_; std::optional error_; + std::optional errorCode_; std::chrono::time_point timestamp_; std::optional ttl_; nlohmann::json info_; diff --git a/libs/model/src/featurelayer.cpp b/libs/model/src/featurelayer.cpp index b000c12b..826883b4 100644 --- a/libs/model/src/featurelayer.cpp +++ b/libs/model/src/featurelayer.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -787,13 +788,57 @@ tl::expected TileFeatureLayer::write(std::ostream& outputSt nlohmann::json TileFeatureLayer::toJson() const { + auto result = nlohmann::json::object(); + + result["type"] = "FeatureCollection"; + result["mapgetTileId"] = tileId_.value_; + result["mapId"] = mapId_; + result["mapgetLayerId"] = layerInfo_->layerId_; + + // Add ID prefix if set + if (impl_->featureIdPrefix_) { + auto prefix = const_cast(this)->getIdPrefix(); + if (prefix) + result["idPrefix"] = prefix->toJson(); + } + + // Add timestamp as ISO 8601 string + { + auto time_t_val = std::chrono::system_clock::to_time_t(timestamp_); + auto microseconds = std::chrono::duration_cast( + timestamp_.time_since_epoch()).count() % 1000000; + std::tm tm_val{}; +#ifdef _WIN32 + gmtime_s(&tm_val, &time_t_val); +#else + gmtime_r(&time_t_val, &tm_val); +#endif + char buf[32]; + std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tm_val); + result["timestamp"] = fmt::format("{}.{:06d}Z", buf, microseconds); + } + + // Add TTL if set (in milliseconds) + if (ttl_) + result["ttl"] = ttl_->count(); + + // Add error information if present + if (error_ || errorCode_) { + auto errorObj = nlohmann::json::object(); + if (errorCode_) + errorObj["code"] = *errorCode_; + if (error_) + errorObj["message"] = *error_; + result["error"] = errorObj; + } + + // Add features auto features = nlohmann::json::array(); for (auto f : *this) features.push_back(f->toJson()); - return nlohmann::json::object({ - {"type", "FeatureCollection"}, - {"features", features} - }); + result["features"] = features; + + return result; } size_t TileFeatureLayer::size() const diff --git a/libs/model/src/layer.cpp b/libs/model/src/layer.cpp index 6a83f9a6..be94dbac 100644 --- a/libs/model/src/layer.cpp +++ b/libs/model/src/layer.cpp @@ -138,6 +138,13 @@ TileLayer::TileLayer( s.text1b(*error_, std::numeric_limits::max()); } + bool hasErrorCode = false; + s.value1b(hasErrorCode); + if (hasErrorCode) { + errorCode_ = 0; // Tell the optional that it has a value. + s.value4b(*errorCode_); + } + bool hasLegalInfo = false; s.value1b(hasLegalInfo); if (hasLegalInfo) { @@ -207,6 +214,14 @@ void TileLayer::setError(const std::optional& err) { error_ = err; } +std::optional TileLayer::errorCode() const { + return errorCode_; +} + +void TileLayer::setErrorCode(const std::optional& code) { + errorCode_ = code; +} + void TileLayer::setTimestamp(const std::chrono::time_point& ts) { timestamp_ = ts; } @@ -247,6 +262,9 @@ tl::expected TileLayer::write(std::ostream& outputStream) s.value1b(error_.has_value()); if (error_) s.text1b(*error_, std::numeric_limits::max()); + s.value1b(errorCode_.has_value()); + if (errorCode_) + s.value4b(*errorCode_); s.value1b(legalInfo_.has_value()); if (legalInfo_.has_value()) { s.text1b(legalInfo_.value(), std::numeric_limits::max()); diff --git a/libs/pymapget/binding/py-layer.h b/libs/pymapget/binding/py-layer.h index 63799355..bc641193 100644 --- a/libs/pymapget/binding/py-layer.h +++ b/libs/pymapget/binding/py-layer.h @@ -51,6 +51,21 @@ void bindTileLayer(py::module_& m) R"pbdoc( Set the error occurred while the tile was filled. )pbdoc") + .def( + "error_code", + [](TileFeatureLayer const& self) { return self.errorCode(); }, + R"pbdoc( + Get the error code (e.g., HTTP status code, SQLite error code) + if an error occurred while the tile was filled. + )pbdoc") + .def( + "set_error_code", + [](TileFeatureLayer& self, int code) { self.setErrorCode(code); }, + py::arg("code"), + R"pbdoc( + Set the error code (e.g., HTTP status code, SQLite error code) + for an error that occurred while the tile was filled. + )pbdoc") .def( "timestamp", [](TileFeatureLayer const& self) {return self.timestamp(); }, diff --git a/test/unit/test-model.cpp b/test/unit/test-model.cpp index 2697dcbf..5f86d734 100644 --- a/test/unit/test-model.cpp +++ b/test/unit/test-model.cpp @@ -236,6 +236,7 @@ TEST_CASE("FeatureLayer", "[test.featurelayer]") REQUIRE(deserializedTile->mapId() == tile->mapId()); REQUIRE(deserializedTile->layerInfo() == tile->layerInfo()); REQUIRE(deserializedTile->error() == tile->error()); + REQUIRE(deserializedTile->errorCode() == tile->errorCode()); REQUIRE(deserializedTile->timestamp().time_since_epoch() == tile->timestamp().time_since_epoch()); REQUIRE(deserializedTile->ttl() == tile->ttl()); REQUIRE(deserializedTile->mapVersion() == tile->mapVersion()); @@ -329,6 +330,83 @@ TEST_CASE("FeatureLayer", "[test.featurelayer]") auto foundFeature10 = tile->find("Way", KeyValueViewPairs{{"wayId", 42}}); REQUIRE(!foundFeature10); } + + SECTION("toJson with enhanced metadata") + { + // Set TTL + tile->setTtl(std::chrono::milliseconds(3600000)); + + auto json = tile->toJson(); + + // Verify required fields + REQUIRE(json["type"] == "FeatureCollection"); + REQUIRE(json["mapgetTileId"].is_number_unsigned()); + REQUIRE(json["mapgetTileId"].get() == tile->tileId().value_); + REQUIRE(json["mapId"] == "Tropico"); + REQUIRE(json["mapgetLayerId"] == "WayLayer"); + + // Verify idPrefix + REQUIRE(json.contains("idPrefix")); + REQUIRE(json["idPrefix"]["areaId"] == "TheBestArea"); + + // Verify timestamp is ISO 8601 format + REQUIRE(json["timestamp"].is_string()); + std::string timestamp = json["timestamp"]; + REQUIRE(timestamp.find("T") != std::string::npos); + REQUIRE(timestamp.back() == 'Z'); + + // Verify TTL + REQUIRE(json["ttl"] == 3600000); + + // Verify features array exists + REQUIRE(json["features"].is_array()); + REQUIRE(json["features"].size() == 2); // feature0 and feature1 + + // Verify no error object when no error is set + REQUIRE(!json.contains("error")); + } + + SECTION("toJson with error information") + { + // Set error message and code + tile->setError("Test error message"); + tile->setErrorCode(404); + + auto json = tile->toJson(); + + // Verify error object + REQUIRE(json.contains("error")); + REQUIRE(json["error"]["message"] == "Test error message"); + REQUIRE(json["error"]["code"] == 404); + } + + SECTION("Serialization with errorCode") + { + // Set error information + tile->setError("Connection timeout"); + tile->setErrorCode(504); + tile->setTtl(std::chrono::milliseconds(60000)); + + std::stringstream tileBytes; + tile->write(tileBytes); + + auto deserializedTile = std::make_shared( + tileBytes, + [&](auto&& mapName, auto&& layerName){ + return layerInfo; + }, + [&](auto&& nodeId){ + return strings; + } + ); + + REQUIRE(deserializedTile->error() == tile->error()); + REQUIRE(deserializedTile->error().value() == "Connection timeout"); + REQUIRE(deserializedTile->errorCode() == tile->errorCode()); + REQUIRE(deserializedTile->errorCode().value() == 504); + REQUIRE(deserializedTile->ttl() == tile->ttl()); + REQUIRE(deserializedTile->ttl().value().count() == 60000); + } } // Helper function to compare two points with some tolerance From 521f3008f6f94f0d55d46f1c95da3b27912e25cd Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Fri, 16 Jan 2026 19:47:43 +0100 Subject: [PATCH 2/5] Add manifest.json support to GeoJsonSource for flexible file mapping and multi-layer --- .../include/geojsonsource/geojsonsource.h | 139 +++++++- libs/geojsonsource/src/geojsonsource.cpp | 268 ++++++++++++--- test/unit/test-geojsonsource.cpp | 308 +++++++++++++++++- 3 files changed, 653 insertions(+), 62 deletions(-) diff --git a/libs/geojsonsource/include/geojsonsource/geojsonsource.h b/libs/geojsonsource/include/geojsonsource/geojsonsource.h index 42f9efeb..b2e046fd 100644 --- a/libs/geojsonsource/include/geojsonsource/geojsonsource.h +++ b/libs/geojsonsource/include/geojsonsource/geojsonsource.h @@ -1,7 +1,9 @@ #pragma once #include +#include #include +#include #include "mapget/model/featurelayer.h" #include "mapget/model/sourcedatalayer.h" @@ -11,29 +13,111 @@ namespace mapget::geojsonsource { /** - * Data Source which may be used to test a directory which - * contains feature tiles as legacy DBI/LiveLab GeoJSON exports. - * Each tile in the designated directory must be named `.geojson`. + * Entry describing a single GeoJSON file in the manifest. + */ +struct FileEntry +{ + std::string filename; + uint64_t tileId = 0; + std::string layer; // Empty means use default layer +}; + +/** + * Metadata section of the manifest (all fields optional). + */ +struct ManifestMetadata +{ + std::optional name; + std::optional description; + std::optional source; + std::optional created; + std::optional author; + std::optional license; +}; + +/** + * Parsed manifest.json structure. + */ +struct Manifest +{ + int version = 1; + ManifestMetadata metadata; + std::string defaultLayer = "GeoJsonAny"; + std::vector files; +}; + +/** + * Key for looking up files by (tileId, layer). + */ +struct TileLayerKey +{ + uint64_t tileId; + std::string layer; + + bool operator==(const TileLayerKey& other) const { + return tileId == other.tileId && layer == other.layer; + } +}; + +struct TileLayerKeyHash +{ + std::size_t operator()(const TileLayerKey& k) const { + return std::hash()(k.tileId) ^ (std::hash()(k.layer) << 1); + } +}; + +/** + * Data Source which may be used to load GeoJSON files from a directory. + * + * Supports two modes of operation: + * + * 1. **Manifest mode** (recommended): If a `manifest.json` file exists in the + * input directory, it is used to map filenames to tile IDs and layers. + * This allows arbitrary filenames and multi-layer support. + * + * Example manifest.json: + * ```json + * { + * "version": 1, + * "metadata": { + * "name": "My Dataset", + * "source": "OpenStreetMap", + * "created": "2024-01-15" + * }, + * "index": { + * "defaultLayer": "GeoJsonAny", + * "files": { + * "roads.geojson": { "tileId": 121212121212, "layer": "Road" }, + * "lanes.geojson": { "tileId": 121212121212, "layer": "Lane" }, + * "other.geojson": { "tileId": 343434343434 } + * } + * } + * } + * ``` * - * You may use the script docs/export-classic-routing-tiles.py, to - * generate an export of NDS.Classic tiles which can be used - * with this class. + * 2. **Legacy mode**: If no manifest.json exists, falls back to scanning for + * files named `.geojson`. All files go into a single + * "GeoJsonAny" layer. * * Note: This data source was mainly developed as a scalability test - * scenario for erdblick, and should NOT BE USED IN PRODUCTION. - * In the future, the DBI will export the same GeoJSON feature model that - * is understood by mapget, and a GeoJSON data source will be part - * of the mapget code base. + * scenario for erdblick. In the future, the DBI will export the same + * GeoJSON feature model that is understood by mapget, and a GeoJSON + * data source will be part of the mapget code base. */ class GeoJsonSource : public mapget::DataSource { public: /** - * Construct a GeoJSON data source from a directory containing - * GeoJSON tiles like .geojson. - * @param inputDir The directory with the GeoJSON tiles. - * @param withAttrLayers Flag indicating, whether compound GeoJSON + * Construct a GeoJSON data source from a directory. + * + * If a manifest.json exists in the directory, it will be used for + * file-to-tile mapping and layer configuration. Otherwise, falls back + * to legacy mode where files must be named `.geojson`. + * + * @param inputDir The directory with the GeoJSON files (and optional manifest.json). + * @param withAttrLayers Flag indicating whether compound GeoJSON * properties shall be converted to mapget attribute layers. + * @param mapId Optional map ID override. If empty, derived from inputDir. */ GeoJsonSource(const std::string& inputDir, bool withAttrLayers, const std::string& mapId=""); @@ -42,11 +126,36 @@ class GeoJsonSource : public mapget::DataSource void fill(mapget::TileFeatureLayer::Ptr const&) override; void fill(mapget::TileSourceDataLayer::Ptr const&) override; + /** Returns true if a manifest.json was found and used. */ + [[nodiscard]] bool hasManifest() const { return hasManifest_; } + + /** Returns the parsed manifest (only valid if hasManifest() is true). */ + [[nodiscard]] const Manifest& manifest() const { return manifest_; } + private: + /** Parse manifest.json from the input directory. Returns true if found and valid. */ + bool parseManifest(); + + /** Initialize coverage from manifest entries. */ + void initFromManifest(); + + /** Initialize coverage by scanning directory for .geojson files (legacy). */ + void initFromDirectory(); + + /** Create LayerInfo JSON for a given layer name. */ + static nlohmann::json createLayerInfoJson(const std::string& layerName); + mapget::DataSourceInfo info_; - std::unordered_set coveredMapgetTileIds_; std::string inputDir_; bool withAttrLayers_ = true; + bool hasManifest_ = false; + Manifest manifest_; + + // Mapping from (tileId, layer) -> filename + std::unordered_map tileLayerToFile_; + + // Set of covered tile IDs per layer (for legacy single-layer mode compatibility) + std::unordered_map> layerCoverage_; }; } // namespace mapget::geojsonsource diff --git a/libs/geojsonsource/src/geojsonsource.cpp b/libs/geojsonsource/src/geojsonsource.cpp index 4ba2d101..47274c17 100644 --- a/libs/geojsonsource/src/geojsonsource.cpp +++ b/libs/geojsonsource/src/geojsonsource.cpp @@ -48,59 +48,239 @@ simfil::ModelNode::Ptr jsonToMapget( // NOLINT (recursive) return {}; } -auto geoJsonLayerInfo = R"json( +constexpr auto manifestFilename = "manifest.json"; + +} // namespace + +namespace mapget::geojsonsource { + +nlohmann::json GeoJsonSource::createLayerInfoJson(const std::string& layerName) +{ + // Create feature type name from layer name (e.g., "Road" -> "RoadFeature") + std::string featureTypeName = layerName; + if (layerName != "GeoJsonAny") { + featureTypeName = layerName + "Feature"; + } else { + featureTypeName = "AnyFeature"; + } + + return nlohmann::json::parse(fmt::format(R"json( +{{ "featureTypes": [ - { - "name": "AnyFeature", + {{ + "name": "{}", "uniqueIdCompositions": [ [ - { + {{ "partId": "tileId", "description": "Mapget Tile ID.", "datatype": "U64" - }, - { + }}, + {{ "partId": "featureIndex", "description": "Index of the feature within the GeoJSON collection.", "datatype": "U32" - } + }} ] ] - } + }} ] -})json"; +}})json", featureTypeName)); +} -} // namespace +bool GeoJsonSource::parseManifest() +{ + auto manifestPath = std::filesystem::path(inputDir_) / manifestFilename; + if (!std::filesystem::exists(manifestPath)) { + return false; + } -namespace mapget::geojsonsource + try { + std::ifstream manifestFile(manifestPath); + nlohmann::json manifestJson; + manifestFile >> manifestJson; + + // Parse version (required) + manifest_.version = manifestJson.value("version", 1); + + // Parse metadata (optional) + if (manifestJson.contains("metadata")) { + auto& meta = manifestJson["metadata"]; + if (meta.contains("name")) + manifest_.metadata.name = meta["name"].get(); + if (meta.contains("description")) + manifest_.metadata.description = meta["description"].get(); + if (meta.contains("source")) + manifest_.metadata.source = meta["source"].get(); + if (meta.contains("created")) + manifest_.metadata.created = meta["created"].get(); + if (meta.contains("author")) + manifest_.metadata.author = meta["author"].get(); + if (meta.contains("license")) + manifest_.metadata.license = meta["license"].get(); + } + + // Parse index (optional - if missing, will fall back to directory scan) + if (manifestJson.contains("index")) { + auto& index = manifestJson["index"]; + + // Default layer name + manifest_.defaultLayer = index.value("defaultLayer", "GeoJsonAny"); + + // Parse files + if (index.contains("files")) { + for (auto& [filename, fileInfo] : index["files"].items()) { + FileEntry entry; + entry.filename = filename; + + if (fileInfo.is_object()) { + // Full format: { "tileId": 123, "layer": "Road" } + entry.tileId = fileInfo.value("tileId", uint64_t{0}); + entry.layer = fileInfo.value("layer", std::string{}); + } else if (fileInfo.is_number()) { + // Short format: just the tile ID + entry.tileId = fileInfo.get(); + } else { + mapget::log().warn( + "Invalid file entry in manifest for '{}': expected object or number", + filename); + continue; + } + + // Use default layer if not specified + if (entry.layer.empty()) { + entry.layer = manifest_.defaultLayer; + } + + // Validate file exists + auto filePath = std::filesystem::path(inputDir_) / filename; + if (!std::filesystem::exists(filePath)) { + mapget::log().warn( + "File '{}' listed in manifest does not exist, skipping", + filename); + continue; + } + + manifest_.files.push_back(std::move(entry)); + } + } + } + + mapget::log().info( + "Loaded manifest.json with {} file entries", + manifest_.files.size()); + + return true; + + } catch (const std::exception& e) { + mapget::log().error("Failed to parse manifest.json: {}", e.what()); + return false; + } +} + +void GeoJsonSource::initFromManifest() { + // Build layer coverage and file mapping from manifest entries + for (const auto& entry : manifest_.files) { + // Track coverage per layer + layerCoverage_[entry.layer].insert(entry.tileId); + + // Map (tileId, layer) -> filename + TileLayerKey key{entry.tileId, entry.layer}; + tileLayerToFile_[key] = entry.filename; + + mapget::log().debug( + "Registered file '{}' -> tile {} in layer '{}'", + entry.filename, entry.tileId, entry.layer); + } + + // Create LayerInfo for each discovered layer + for (const auto& [layerName, tileIds] : layerCoverage_) { + auto layerJson = createLayerInfoJson(layerName); + auto layerInfo = mapget::LayerInfo::fromJson(layerJson, layerName); + + // Add coverage entries + for (uint64_t tileId : tileIds) { + mapget::Coverage coverage({tileId, tileId, std::vector()}); + layerInfo->coverage_.emplace_back(coverage); + } + + info_.layers_.emplace(layerName, layerInfo); + mapget::log().info( + "Layer '{}' initialized with {} tiles", + layerName, tileIds.size()); + } +} + +void GeoJsonSource::initFromDirectory() +{ + // Legacy mode: scan for .geojson files + const std::string defaultLayer = "GeoJsonAny"; + auto layerJson = createLayerInfoJson(defaultLayer); + auto layerInfo = mapget::LayerInfo::fromJson(layerJson, defaultLayer); + + for (const auto& file : std::filesystem::directory_iterator(inputDir_)) { + mapget::log().debug("Found file {}", file.path().string()); + if (file.path().extension() == ".geojson") { + try { + auto tileId = static_cast(std::stoull(file.path().stem())); + layerCoverage_[defaultLayer].insert(tileId); + + TileLayerKey key{tileId, defaultLayer}; + tileLayerToFile_[key] = file.path().filename().string(); + + mapget::Coverage coverage({tileId, tileId, std::vector()}); + layerInfo->coverage_.emplace_back(coverage); + mapget::log().debug("Added tile {}", tileId); + } catch (const std::exception& e) { + mapget::log().debug( + "Skipping file '{}': filename is not a valid tile ID", + file.path().filename().string()); + } + } + } + + info_.layers_.emplace(defaultLayer, layerInfo); +} GeoJsonSource::GeoJsonSource(const std::string& inputDir, bool withAttrLayers, const std::string& mapId) : inputDir_(inputDir), withAttrLayers_(withAttrLayers) { - // Initialize DataSourceInfo from JSON - info_.layers_.emplace( - "GeoJsonAny", - mapget::LayerInfo::fromJson(nlohmann::json::parse(geoJsonLayerInfo), "GeoJsonAny")); - // Compromise between performance and resource usage, so that we don't overload the system. - // TODO: Find a more sophisticated way to determine the number of parallel jobs. + // Compromise between performance and resource usage info_.maxParallelJobs_ = std::max((int)(0.33*std::thread::hardware_concurrency()), 2); info_.mapId_ = mapId.empty() ? mapNameFromUri(inputDir) : mapId; info_.nodeId_ = generateNodeHexUuid(); - // Initialize coverage - auto layer = info_.getLayer("GeoJsonAny"); - for (const auto& file : std::filesystem::directory_iterator(inputDir)) { - mapget::log().debug("Found file {}", file.path().string()); - if (file.path().extension() == ".geojson") { - auto tileId = static_cast(std::stoull(file.path().stem())); - coveredMapgetTileIds_.insert(tileId); - mapget::Coverage coverage({tileId, tileId, std::vector()}); - layer->coverage_.emplace_back(coverage); - mapget::log().debug("Added tile {}", tileId); + // Try to load manifest.json first + hasManifest_ = parseManifest(); + + if (hasManifest_ && !manifest_.files.empty()) { + // Use manifest-based initialization + initFromManifest(); + } else { + // Fallback to directory scanning + if (!hasManifest_) { + mapget::log().warn( + "No manifest.json found in '{}'. " + "Falling back to filename-based tile ID detection. " + "Consider adding a manifest.json for better control over file mapping and layers.", + inputDir); + } else { + mapget::log().info( + "manifest.json found but has no index/files section, scanning directory"); } + initFromDirectory(); + } + + // Log summary + size_t totalTiles = 0; + for (const auto& [layer, tileIds] : layerCoverage_) { + totalTiles += tileIds.size(); } + mapget::log().info( + "GeoJsonSource initialized: {} layers, {} total tile entries", + info_.layers_.size(), totalTiles); } mapget::DataSourceInfo GeoJsonSource::info() @@ -112,34 +292,48 @@ void GeoJsonSource::fill(const mapget::TileFeatureLayer::Ptr& tile) { using namespace mapget; - mapget::log().debug("Starting... "); + auto tileId = tile->tileId().value_; + auto layerName = tile->layerInfo()->layerId_; + + mapget::log().debug("Filling tile {} for layer '{}'", tileId, layerName); - auto tileIdIt = coveredMapgetTileIds_.find(tile->tileId().value_); - if (tileIdIt == coveredMapgetTileIds_.end()) { - mapget::log().error("Tile not available: {}", tile->tileId().value_); + // Look up the file for this (tileId, layer) combination + TileLayerKey key{tileId, layerName}; + auto fileIt = tileLayerToFile_.find(key); + if (fileIt == tileLayerToFile_.end()) { + mapget::log().error( + "No file registered for tile {} in layer '{}'", + tileId, layerName); return; } - auto tileId = *tileIdIt; - // All features share the same tile id. + // All features share the same tile id tile->setIdPrefix({{"tileId", static_cast(tileId)}}); - // Parse the GeoJSON file - auto path = fmt::format("{}/{}.geojson", inputDir_, std::to_string(tileId)); + // Build the full path + auto path = (std::filesystem::path(inputDir_) / fileIt->second).string(); mapget::log().debug("Opening: {}", path); std::ifstream geojsonFile(path); + if (!geojsonFile) { + mapget::log().error("Failed to open file: {}", path); + return; + } + nlohmann::json geojsonData; geojsonFile >> geojsonData; mapget::log().debug("Processing {} features...", geojsonData["features"].size()); + // Get the feature type name for this layer + std::string featureTypeName = (layerName == "GeoJsonAny") ? "AnyFeature" : (layerName + "Feature"); + // Iterate over each feature in the GeoJSON data - int featureId = 0; // Initialize the running index + int featureId = 0; for (auto& feature_data : geojsonData["features"]) { // Create a new feature - auto feature = tile->newFeature("AnyFeature", {{"featureIndex", featureId}}); + auto feature = tile->newFeature(featureTypeName, {{"featureIndex", featureId}}); featureId++; // Get geometry data diff --git a/test/unit/test-geojsonsource.cpp b/test/unit/test-geojsonsource.cpp index 9b0b5fdc..4cc86fde 100644 --- a/test/unit/test-geojsonsource.cpp +++ b/test/unit/test-geojsonsource.cpp @@ -13,6 +13,7 @@ namespace // Sample GeoJSON with a 64-bit tile ID (37392110387213 > UINT32_MAX) constexpr uint64_t largeTileId = 37392110387213; +constexpr uint64_t secondTileId = 37392110387214; auto sampleGeoJson = R"json({"type": "FeatureCollection", "features": [{ "geometry": { @@ -31,41 +32,60 @@ auto sampleGeoJson = R"json({"type": "FeatureCollection", "features": [{ "type": "Feature" }]})json"; -std::filesystem::path createTempGeoJsonDir() +auto sampleGeoJson2 = R"json({"type": "FeatureCollection", "features": [{ + "geometry": { + "coordinates": [11.30, 48.04, 0.0], + "type": "Point" + }, + "properties": { + "name": "Test Point" + }, + "type": "Feature" +}]})json"; + +std::filesystem::path createTempDir() { - // Use timestamp for unique directory name (same pattern as test-cache.cpp) auto now = std::chrono::system_clock::now(); auto epochTime = std::chrono::system_clock::to_time_t(now); auto tempDir = std::filesystem::temp_directory_path() / - ("mapget_geojson_test_" + std::to_string(epochTime)); + ("mapget_geojson_test_" + std::to_string(epochTime) + "_" + + std::to_string(std::rand())); if (std::filesystem::exists(tempDir)) { std::filesystem::remove_all(tempDir); } std::filesystem::create_directories(tempDir); - auto geojsonPath = tempDir / (std::to_string(largeTileId) + ".geojson"); - std::ofstream file(geojsonPath); - file << sampleGeoJson; - file.close(); - return tempDir; } +void writeFile(const std::filesystem::path& path, const std::string& content) +{ + std::ofstream file(path); + file << content; + file.close(); +} + } // namespace TEST_CASE("GeoJsonSource", "[GeoJsonSource]") { - SECTION("64-bit tile ID support") + SECTION("64-bit tile ID support (legacy mode)") { // Verify our test tile ID exceeds 32-bit max REQUIRE(largeTileId > UINT32_MAX); - auto tempDir = createTempGeoJsonDir(); + auto tempDir = createTempDir(); + + // Create GeoJSON file with tile ID as filename (legacy mode) + writeFile(tempDir / (std::to_string(largeTileId) + ".geojson"), sampleGeoJson); // Create GeoJsonSource - should not throw with 64-bit tile IDs geojsonsource::GeoJsonSource source(tempDir.string(), false); + // Should be in legacy mode (no manifest) + REQUIRE_FALSE(source.hasManifest()); + // Get source info and verify coverage includes our tile auto info = source.info(); auto layer = info.getLayer("GeoJsonAny"); @@ -90,4 +110,272 @@ TEST_CASE("GeoJsonSource", "[GeoJsonSource]") // Cleanup std::filesystem::remove_all(tempDir); } + + SECTION("Manifest with single layer") + { + auto tempDir = createTempDir(); + + // Create GeoJSON file with custom name + writeFile(tempDir / "my_roads.geojson", sampleGeoJson); + + // Create manifest.json + auto manifest = R"json({ + "version": 1, + "metadata": { + "name": "Test Dataset", + "source": "Unit Test" + }, + "index": { + "files": { + "my_roads.geojson": { "tileId": 37392110387213 } + } + } + })json"; + writeFile(tempDir / "manifest.json", manifest); + + geojsonsource::GeoJsonSource source(tempDir.string(), false); + + REQUIRE(source.hasManifest()); + REQUIRE(source.manifest().metadata.name == "Test Dataset"); + REQUIRE(source.manifest().metadata.source == "Unit Test"); + + auto info = source.info(); + auto layer = info.getLayer("GeoJsonAny"); + REQUIRE(layer != nullptr); + + auto strings = std::make_shared(info.nodeId_); + auto tile = std::make_shared( + TileId(largeTileId), + info.nodeId_, + info.mapId_, + layer, + strings); + + REQUIRE_NOTHROW(source.fill(tile)); + REQUIRE(tile->numRoots() > 0); + + std::filesystem::remove_all(tempDir); + } + + SECTION("Manifest with multiple layers") + { + auto tempDir = createTempDir(); + + // Create GeoJSON files for different layers + writeFile(tempDir / "roads.geojson", sampleGeoJson); + writeFile(tempDir / "lanes.geojson", sampleGeoJson2); + + // Create manifest with multiple layers + auto manifest = R"json({ + "version": 1, + "index": { + "defaultLayer": "GeoJsonAny", + "files": { + "roads.geojson": { "tileId": 37392110387213, "layer": "Road" }, + "lanes.geojson": { "tileId": 37392110387213, "layer": "Lane" } + } + } + })json"; + writeFile(tempDir / "manifest.json", manifest); + + geojsonsource::GeoJsonSource source(tempDir.string(), false); + + REQUIRE(source.hasManifest()); + + auto info = source.info(); + + // Verify both layers exist + auto roadLayer = info.getLayer("Road"); + auto laneLayer = info.getLayer("Lane"); + REQUIRE(roadLayer != nullptr); + REQUIRE(laneLayer != nullptr); + + // Verify feature type names + REQUIRE(roadLayer->featureTypes_.size() == 1); + REQUIRE(roadLayer->featureTypes_[0].name_ == "RoadFeature"); + REQUIRE(laneLayer->featureTypes_.size() == 1); + REQUIRE(laneLayer->featureTypes_[0].name_ == "LaneFeature"); + + // Fill Road layer + auto strings = std::make_shared(info.nodeId_); + auto roadTile = std::make_shared( + TileId(largeTileId), + info.nodeId_, + info.mapId_, + roadLayer, + strings); + + REQUIRE_NOTHROW(source.fill(roadTile)); + REQUIRE(roadTile->numRoots() > 0); + + // Fill Lane layer + auto laneTile = std::make_shared( + TileId(largeTileId), + info.nodeId_, + info.mapId_, + laneLayer, + strings); + + REQUIRE_NOTHROW(source.fill(laneTile)); + REQUIRE(laneTile->numRoots() > 0); + + std::filesystem::remove_all(tempDir); + } + + SECTION("Manifest with short tile ID format") + { + auto tempDir = createTempDir(); + + writeFile(tempDir / "data.geojson", sampleGeoJson); + + // Use short format (just tile ID number) + auto manifest = R"json({ + "version": 1, + "index": { + "files": { + "data.geojson": 37392110387213 + } + } + })json"; + writeFile(tempDir / "manifest.json", manifest); + + geojsonsource::GeoJsonSource source(tempDir.string(), false); + + REQUIRE(source.hasManifest()); + REQUIRE(source.manifest().files.size() == 1); + REQUIRE(source.manifest().files[0].tileId == largeTileId); + + std::filesystem::remove_all(tempDir); + } + + SECTION("Manifest with metadata only falls back to directory scan") + { + auto tempDir = createTempDir(); + + // Create GeoJSON file with tile ID as filename + writeFile(tempDir / (std::to_string(largeTileId) + ".geojson"), sampleGeoJson); + + // Create manifest with only metadata (no index) + auto manifest = R"json({ + "version": 1, + "metadata": { + "name": "Metadata Only", + "description": "Dataset with no index section" + } + })json"; + writeFile(tempDir / "manifest.json", manifest); + + geojsonsource::GeoJsonSource source(tempDir.string(), false); + + // Manifest was found but has no index, so falls back to directory scan + REQUIRE(source.hasManifest()); + REQUIRE(source.manifest().files.empty()); + + // But tiles should still be discovered from filenames + auto info = source.info(); + auto layer = info.getLayer("GeoJsonAny"); + REQUIRE(layer != nullptr); + REQUIRE(!layer->coverage_.empty()); + + std::filesystem::remove_all(tempDir); + } + + SECTION("Manifest with missing file warns and skips") + { + auto tempDir = createTempDir(); + + // Create only one of the two files listed in manifest + writeFile(tempDir / "existing.geojson", sampleGeoJson); + + auto manifest = R"json({ + "version": 1, + "index": { + "files": { + "existing.geojson": { "tileId": 37392110387213 }, + "missing.geojson": { "tileId": 37392110387214 } + } + } + })json"; + writeFile(tempDir / "manifest.json", manifest); + + geojsonsource::GeoJsonSource source(tempDir.string(), false); + + REQUIRE(source.hasManifest()); + // Only the existing file should be registered + REQUIRE(source.manifest().files.size() == 1); + REQUIRE(source.manifest().files[0].filename == "existing.geojson"); + + std::filesystem::remove_all(tempDir); + } + + SECTION("Legacy mode skips non-numeric filenames") + { + auto tempDir = createTempDir(); + + // Create files with valid and invalid names + writeFile(tempDir / (std::to_string(largeTileId) + ".geojson"), sampleGeoJson); + writeFile(tempDir / "not_a_number.geojson", sampleGeoJson2); + writeFile(tempDir / "readme.txt", "Not a geojson file"); + + geojsonsource::GeoJsonSource source(tempDir.string(), false); + + REQUIRE_FALSE(source.hasManifest()); + + auto info = source.info(); + auto layer = info.getLayer("GeoJsonAny"); + REQUIRE(layer != nullptr); + // Only the valid tile ID file should be registered + REQUIRE(layer->coverage_.size() == 1); + + std::filesystem::remove_all(tempDir); + } + + SECTION("Multiple tiles same layer") + { + auto tempDir = createTempDir(); + + writeFile(tempDir / "tile1.geojson", sampleGeoJson); + writeFile(tempDir / "tile2.geojson", sampleGeoJson2); + + auto manifest = R"json({ + "version": 1, + "index": { + "files": { + "tile1.geojson": { "tileId": 37392110387213, "layer": "Road" }, + "tile2.geojson": { "tileId": 37392110387214, "layer": "Road" } + } + } + })json"; + writeFile(tempDir / "manifest.json", manifest); + + geojsonsource::GeoJsonSource source(tempDir.string(), false); + + auto info = source.info(); + auto roadLayer = info.getLayer("Road"); + REQUIRE(roadLayer != nullptr); + REQUIRE(roadLayer->coverage_.size() == 2); + + // Fill both tiles + auto strings = std::make_shared(info.nodeId_); + + auto tile1 = std::make_shared( + TileId(largeTileId), + info.nodeId_, + info.mapId_, + roadLayer, + strings); + REQUIRE_NOTHROW(source.fill(tile1)); + REQUIRE(tile1->numRoots() > 0); + + auto tile2 = std::make_shared( + TileId(secondTileId), + info.nodeId_, + info.mapId_, + roadLayer, + strings); + REQUIRE_NOTHROW(source.fill(tile2)); + REQUIRE(tile2->numRoots() > 0); + + std::filesystem::remove_all(tempDir); + } } From e55dfa5d4e81b89e8fa2661f6334feb9b02b8aca Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Mon, 19 Jan 2026 15:41:00 +0100 Subject: [PATCH 3/5] Document GeoJsonSource manifest.json format and deprecate legacy mode --- docs/mapget-config.md | 60 +++++++++++++++++++++++- libs/geojsonsource/src/geojsonsource.cpp | 5 +- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/docs/mapget-config.md b/docs/mapget-config.md index d4c61e20..d44fddb2 100644 --- a/docs/mapget-config.md +++ b/docs/mapget-config.md @@ -194,7 +194,7 @@ The generator will produce deterministic but varied features for any requested t -`GeoJsonFolder` serves tiles from a directory containing GeoJSON files. Each file represents one tile and must be named with the tile’s numeric ID in the mapget tiling scheme, for example `123456.geojson`. +`GeoJsonFolder` serves tiles from a directory containing GeoJSON files. Required fields: @@ -214,7 +214,63 @@ sources: withAttrLayers: true ``` -The datasource scans the directory, infers coverage from the file names and converts each GeoJSON feature into mapget’s internal feature model when the corresponding tile is requested. +#### Manifest Mode (Recommended) + +If a `manifest.json` file exists in the input directory, it is used to map filenames to tile IDs and layers. This allows arbitrary filenames and multi‑layer support. + +**Manifest Structure:** + +```json +{ + "version": 1, + "metadata": { + "name": "My Dataset", + "description": "Optional description of the dataset", + "source": "OpenStreetMap", + "created": "2024-01-15", + "author": "Your Name", + "license": "CC-BY-4.0" + }, + "index": { + "defaultLayer": "GeoJsonAny", + "files": { + "roads.geojson": { "tileId": 121212121212, "layer": "Road" }, + "lanes.geojson": { "tileId": 121212121212, "layer": "Lane" }, + "other.geojson": { "tileId": 343434343434 }, + "simple.geojson": 565656565656 + } + } +} +``` + +**Manifest Fields:** + +| Field | Required | Description | +|-------|----------|-------------| +| `version` | No | Manifest format version (default: `1`). | +| `metadata` | No | Optional metadata about the dataset. All sub‑fields (`name`, `description`, `source`, `created`, `author`, `license`) are optional strings. | +| `index` | No | File‑to‑tile mapping configuration. | +| `index.defaultLayer` | No | Default layer name for files without explicit layer (default: `"GeoJsonAny"`). | +| `index.files` | No | Object mapping filenames to tile information. | + +**File Entry Formats:** + +Each entry in `index.files` maps a filename to tile information. Two formats are supported: + +1. **Full format** (object): `{ "tileId": , "layer": "" }` + - `tileId` (required): The mapget tile ID as a 64‑bit unsigned integer. + - `layer` (optional): Layer name. If omitted, uses `defaultLayer`. + +2. **Short format** (number): Just the tile ID as a number. Uses `defaultLayer` for the layer name. + +This allows multiple GeoJSON files to contribute features to the same tile in different layers, enabling separation of feature types (e.g., roads, lanes, buildings) while sharing the same tile coordinate. + +#### Legacy Mode (Deprecated) + +!!! warning "Deprecation Notice" + Legacy mode is deprecated and will be removed in a future release. Please migrate to manifest mode by adding a `manifest.json` file to your data directory. When legacy mode is used, a warning is logged to help identify directories that need migration. + +If no `manifest.json` exists, the datasource falls back to scanning for files named `.geojson` (e.g., `123456.geojson`). All files are served from a single `GeoJsonAny` layer. diff --git a/libs/geojsonsource/src/geojsonsource.cpp b/libs/geojsonsource/src/geojsonsource.cpp index 47274c17..6a43038d 100644 --- a/libs/geojsonsource/src/geojsonsource.cpp +++ b/libs/geojsonsource/src/geojsonsource.cpp @@ -263,8 +263,9 @@ GeoJsonSource::GeoJsonSource(const std::string& inputDir, bool withAttrLayers, c if (!hasManifest_) { mapget::log().warn( "No manifest.json found in '{}'. " - "Falling back to filename-based tile ID detection. " - "Consider adding a manifest.json for better control over file mapping and layers.", + "Using deprecated legacy mode with filename-based tile ID detection. " + "Legacy mode will be removed in a future release. " + "Please add a manifest.json for file mapping and multi-layer support.", inputDir); } else { mapget::log().info( From bce03a7b86acd439dec1241ca0fbe251734d651d Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Wed, 21 Jan 2026 19:46:58 +0100 Subject: [PATCH 4/5] Fix GeoJsonSource to never use legacy mode when manifest.json is present --- libs/geojsonsource/src/geojsonsource.cpp | 24 +++++++------- test/unit/test-geojsonsource.cpp | 41 ++++++++++++++++++++---- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/libs/geojsonsource/src/geojsonsource.cpp b/libs/geojsonsource/src/geojsonsource.cpp index 6a43038d..6a0bd37c 100644 --- a/libs/geojsonsource/src/geojsonsource.cpp +++ b/libs/geojsonsource/src/geojsonsource.cpp @@ -255,22 +255,20 @@ GeoJsonSource::GeoJsonSource(const std::string& inputDir, bool withAttrLayers, c // Try to load manifest.json first hasManifest_ = parseManifest(); - if (hasManifest_ && !manifest_.files.empty()) { - // Use manifest-based initialization - initFromManifest(); - } else { - // Fallback to directory scanning - if (!hasManifest_) { - mapget::log().warn( - "No manifest.json found in '{}'. " - "Using deprecated legacy mode with filename-based tile ID detection. " - "Legacy mode will be removed in a future release. " - "Please add a manifest.json for file mapping and multi-layer support.", - inputDir); + if (hasManifest_) { + if (!manifest_.files.empty()) { + initFromManifest(); } else { mapget::log().info( - "manifest.json found but has no index/files section, scanning directory"); + "manifest.json found but has no index/files section - no tiles available"); } + } else { + mapget::log().warn( + "No manifest.json found in '{}'. " + "Using deprecated legacy mode with filename-based tile ID detection. " + "Legacy mode will be removed in a future release. " + "Please add a manifest.json for file mapping and multi-layer support.", + inputDir); initFromDirectory(); } diff --git a/test/unit/test-geojsonsource.cpp b/test/unit/test-geojsonsource.cpp index 4cc86fde..b41c6d13 100644 --- a/test/unit/test-geojsonsource.cpp +++ b/test/unit/test-geojsonsource.cpp @@ -248,7 +248,7 @@ TEST_CASE("GeoJsonSource", "[GeoJsonSource]") std::filesystem::remove_all(tempDir); } - SECTION("Manifest with metadata only falls back to directory scan") + SECTION("Manifest with metadata only does not fall back to directory scan") { auto tempDir = createTempDir(); @@ -267,15 +267,13 @@ TEST_CASE("GeoJsonSource", "[GeoJsonSource]") geojsonsource::GeoJsonSource source(tempDir.string(), false); - // Manifest was found but has no index, so falls back to directory scan + // Manifest was found but has no index - should NOT fall back to directory scan REQUIRE(source.hasManifest()); REQUIRE(source.manifest().files.empty()); - // But tiles should still be discovered from filenames + // No tiles should be available (no fallback to legacy filename parsing) auto info = source.info(); - auto layer = info.getLayer("GeoJsonAny"); - REQUIRE(layer != nullptr); - REQUIRE(!layer->coverage_.empty()); + REQUIRE(info.layers_.empty()); std::filesystem::remove_all(tempDir); } @@ -330,6 +328,37 @@ TEST_CASE("GeoJsonSource", "[GeoJsonSource]") std::filesystem::remove_all(tempDir); } + SECTION("Manifest prevents legacy filename parsing for non-numeric names") + { + auto tempDir = createTempDir(); + + // Create GeoJSON file with non-numeric name (would fail stoull in legacy mode) + writeFile(tempDir / "mytestdata.geojson", sampleGeoJson); + + // Create manifest that maps the file correctly + auto manifest = R"json({ + "version": 1, + "index": { + "files": { + "mytestdata.geojson": { "tileId": 62530591326221, "layer": "Road" } + } + } + })json"; + writeFile(tempDir / "manifest.json", manifest); + + // Should not throw - manifest mode should be used, not legacy filename parsing + geojsonsource::GeoJsonSource source(tempDir.string(), false); + + REQUIRE(source.hasManifest()); + + auto info = source.info(); + auto layer = info.getLayer("Road"); + REQUIRE(layer != nullptr); + REQUIRE(layer->coverage_.size() == 1); + + std::filesystem::remove_all(tempDir); + } + SECTION("Multiple tiles same layer") { auto tempDir = createTempDir(); From 211fa321d0e0ecef43e977fe2f371cccc4bc0ad3 Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Thu, 22 Jan 2026 17:14:53 +0100 Subject: [PATCH 5/5] Version bump. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bdf99d9a..1d73bb5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(mapget LANGUAGES CXX C) # Allow version to be set from command line for CI/CD # For local development, use the default version if(NOT DEFINED MAPGET_VERSION) - set(MAPGET_VERSION 2025.5.0) + set(MAPGET_VERSION 2025.5.1) endif() set(CMAKE_CXX_STANDARD 20)