From b782cf3d48ef4bedaee03c758cab4cdfacc72775 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 14 Jan 2026 17:55:35 +0100 Subject: [PATCH 01/14] Add cached tile size breakdown stats --- .../model/include/mapget/model/featurelayer.h | 4 + libs/model/src/featurelayer.cpp | 92 +++++++++++++++++++ libs/service/include/mapget/service/cache.h | 5 + .../service/include/mapget/service/memcache.h | 7 +- .../include/mapget/service/nullcache.h | 5 +- .../include/mapget/service/sqlitecache.h | 3 +- libs/service/src/memcache.cpp | 10 +- libs/service/src/nullcache.cpp | 7 +- libs/service/src/service.cpp | 78 +++++++++++++++- libs/service/src/sqlitecache.cpp | 28 +++++- 10 files changed, 231 insertions(+), 8 deletions(-) diff --git a/libs/model/include/mapget/model/featurelayer.h b/libs/model/include/mapget/model/featurelayer.h index 97c4f569..07888c60 100644 --- a/libs/model/include/mapget/model/featurelayer.h +++ b/libs/model/include/mapget/model/featurelayer.h @@ -20,6 +20,7 @@ #include "geometry.h" #include "sourcedatareference.h" #include "pointnode.h" +#include "nlohmann/json.hpp" namespace mapget { @@ -209,6 +210,9 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool /** Convert to (Geo-) JSON. */ nlohmann::json toJson() const override; + /** Report serialized size stats for feature-layer data and model-pool columns. */ + [[nodiscard]] nlohmann::json serializationSizeStats() const; + /** Access number of stored features */ size_t size() const; diff --git a/libs/model/src/featurelayer.cpp b/libs/model/src/featurelayer.cpp index 826883b4..cf891b66 100644 --- a/libs/model/src/featurelayer.cpp +++ b/libs/model/src/featurelayer.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -76,6 +77,39 @@ namespace throw std::out_of_range("Size out of range"); return (index << SourceAddressArenaSizeBits) | size; } + + class CountingStreambuf : public std::streambuf + { + public: + size_t size() const { return size_; } + + protected: + std::streamsize xsputn(const char* /*s*/, std::streamsize count) override + { + size_ += static_cast(count); + return count; + } + + int overflow(int ch) override + { + if (ch != EOF) + ++size_; + return ch; + } + + private: + size_t size_ = 0; + }; + + template + size_t measureBytes(Fn&& fn) + { + CountingStreambuf buf; + std::ostream os(&buf); + bitsery::Serializer s(os); + fn(s); + return buf.size(); + } } namespace mapget @@ -841,6 +875,64 @@ nlohmann::json TileFeatureLayer::toJson() const return result; } +nlohmann::json TileFeatureLayer::serializationSizeStats() const +{ + constexpr size_t maxColumnSize = std::numeric_limits::max(); + auto featureLayer = nlohmann::json::object(); + + featureLayer["features"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->features_, maxColumnSize); })); + featureLayer["attributes"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->attributes_, maxColumnSize); })); + featureLayer["validities"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->validities_, maxColumnSize); })); + featureLayer["feature-ids"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->featureIds_, maxColumnSize); })); + featureLayer["attribute-layers"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->attrLayers_, maxColumnSize); })); + featureLayer["attribute-layer-lists"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->attrLayerLists_, maxColumnSize); })); + featureLayer["feature-id-prefix"] = static_cast(measureBytes( + [&](auto& s) { s.object(impl_->featureIdPrefix_); })); + featureLayer["relations"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->relations_, maxColumnSize); })); + featureLayer["feature-hash-index"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->featureHashIndex_, maxColumnSize); })); + featureLayer["geometries"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->geom_, maxColumnSize); })); + featureLayer["point-buffers"] = static_cast(measureBytes( + [&](auto& s) { s.ext(impl_->pointBuffers_, bitsery::ext::ArrayArenaExt{}); })); + featureLayer["source-data-references"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->sourceDataReferences_, maxColumnSize); })); + + int64_t featureLayerTotal = 0; + for (const auto& [_, value] : featureLayer.items()) { + if (value.is_number_integer()) + featureLayerTotal += value.get(); + } + + auto modelStats = ModelPool::serializationSizeStats(); + auto modelPool = nlohmann::json::object({ + {"roots", static_cast(modelStats.rootsBytes)}, + {"int64", static_cast(modelStats.int64Bytes)}, + {"double", static_cast(modelStats.doubleBytes)}, + {"string-data", static_cast(modelStats.stringDataBytes)}, + {"string-ranges", static_cast(modelStats.stringRangeBytes)}, + {"object-members", static_cast(modelStats.objectMemberBytes)}, + {"array-members", static_cast(modelStats.arrayMemberBytes)}, + }); + + int64_t modelPoolTotal = static_cast(modelStats.totalBytes()); + + return { + {"feature-layer", featureLayer}, + {"model-pool", modelPool}, + {"feature-layer-total-bytes", featureLayerTotal}, + {"model-pool-total-bytes", modelPoolTotal}, + {"total-bytes", featureLayerTotal + modelPoolTotal} + }; +} + size_t TileFeatureLayer::size() const { return numRoots(); diff --git a/libs/service/include/mapget/service/cache.h b/libs/service/include/mapget/service/cache.h index bee02b1d..50d92a28 100644 --- a/libs/service/include/mapget/service/cache.h +++ b/libs/service/include/mapget/service/cache.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "mapget/model/info.h" #include "mapget/model/featurelayer.h" @@ -29,6 +30,7 @@ class Cache : public TileLayerStream::StringPoolCache, public std::enable_shared }; using Ptr = std::shared_ptr; + using TileBlobVisitor = std::function; // The following methods are already implemented, // they forward to the virtual methods on-demand. @@ -52,6 +54,9 @@ class Cache : public TileLayerStream::StringPoolCache, public std::enable_shared /** Abstract: Upsert (update or insert) a TileLayer blob. */ virtual void putTileLayerBlob(MapTileKey const& k, std::string const& v) = 0; + /** Abstract: Iterate through all cached tile layer blobs. */ + virtual void forEachTileLayerBlob(const TileBlobVisitor& cb) const = 0; + /** Abstract: Retrieve a string-pool blob for a sourceNodeId. */ virtual std::optional getStringPoolBlob(std::string_view const& sourceNodeId) = 0; diff --git a/libs/service/include/mapget/service/memcache.h b/libs/service/include/mapget/service/memcache.h index 283c0e11..bdc2b452 100644 --- a/libs/service/include/mapget/service/memcache.h +++ b/libs/service/include/mapget/service/memcache.h @@ -29,6 +29,9 @@ class MemCache : public Cache /** Upsert a TileLayer blob. */ void putTileLayerBlob(MapTileKey const& k, std::string const& v) override; + /** Iterate over cached tile layer blobs. */ + void forEachTileLayerBlob(const TileBlobVisitor& cb) const override; + /** Retrieve a string-pool blob for a sourceNodeId -> No-Op */ std::optional getStringPoolBlob(std::string_view const& sourceNodeId) override {return {};} @@ -40,10 +43,10 @@ class MemCache : public Cache private: // Cached tile blobs. - std::shared_mutex cacheMutex_; + mutable std::shared_mutex cacheMutex_; std::unordered_map cachedTiles_; std::deque fifo_; uint32_t maxCachedTiles_ = 0; }; -} \ No newline at end of file +} diff --git a/libs/service/include/mapget/service/nullcache.h b/libs/service/include/mapget/service/nullcache.h index 00aa8e1a..66e29b71 100644 --- a/libs/service/include/mapget/service/nullcache.h +++ b/libs/service/include/mapget/service/nullcache.h @@ -20,6 +20,9 @@ class NullCache : public Cache /** Upsert a TileLayer blob - does nothing. */ void putTileLayerBlob(MapTileKey const& k, std::string const& v) override; + /** Iterate cached tile blobs - no-op. */ + void forEachTileLayerBlob(const TileBlobVisitor& cb) const override; + /** Retrieve a string-pool blob for a sourceNodeId - always returns empty. */ std::optional getStringPoolBlob(std::string_view const& sourceNodeId) override; @@ -27,4 +30,4 @@ class NullCache : public Cache void putStringPoolBlob(std::string_view const& sourceNodeId, std::string const& v) override; }; -} \ No newline at end of file +} diff --git a/libs/service/include/mapget/service/sqlitecache.h b/libs/service/include/mapget/service/sqlitecache.h index 55b42879..85387dcd 100644 --- a/libs/service/include/mapget/service/sqlitecache.h +++ b/libs/service/include/mapget/service/sqlitecache.h @@ -25,6 +25,7 @@ class SQLiteCache : public Cache std::optional getTileLayerBlob(MapTileKey const& k) override; void putTileLayerBlob(MapTileKey const& k, std::string const& v) override; + void forEachTileLayerBlob(const TileBlobVisitor& cb) const override; std::optional getStringPoolBlob(std::string_view const& sourceNodeId) override; void putStringPoolBlob(std::string_view const& sourceNodeId, std::string const& v) override; @@ -53,4 +54,4 @@ class SQLiteCache : public Cache } stmts_; }; -} // namespace mapget \ No newline at end of file +} // namespace mapget diff --git a/libs/service/src/memcache.cpp b/libs/service/src/memcache.cpp index d9a49dc8..f3d95a88 100644 --- a/libs/service/src/memcache.cpp +++ b/libs/service/src/memcache.cpp @@ -31,6 +31,14 @@ void MemCache::putTileLayerBlob(const MapTileKey& k, const std::string& v) } } +void MemCache::forEachTileLayerBlob(const TileBlobVisitor& cb) const +{ + std::shared_lock cacheLock(cacheMutex_); + for (const auto& [key, value] : cachedTiles_) { + cb(MapTileKey(key), value); + } +} + nlohmann::json MemCache::getStatistics() const { auto result = Cache::getStatistics(); result["memcache-map-size"] = (int64_t)cachedTiles_.size(); @@ -38,4 +46,4 @@ nlohmann::json MemCache::getStatistics() const { return result; } -} \ No newline at end of file +} diff --git a/libs/service/src/nullcache.cpp b/libs/service/src/nullcache.cpp index 0862a5ea..e1dc99e6 100644 --- a/libs/service/src/nullcache.cpp +++ b/libs/service/src/nullcache.cpp @@ -13,6 +13,11 @@ void NullCache::putTileLayerBlob(MapTileKey const& k, std::string const& v) // Do nothing - no caching } +void NullCache::forEachTileLayerBlob(const TileBlobVisitor& cb) const +{ + // No cached tiles. +} + std::optional NullCache::getStringPoolBlob(std::string_view const& sourceNodeId) { return std::nullopt; @@ -23,4 +28,4 @@ void NullCache::putStringPoolBlob(std::string_view const& sourceNodeId, std::str // Do nothing - no caching } -} \ No newline at end of file +} diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index c8449d8c..9f0b1cbf 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include "simfil/types.h" @@ -673,10 +675,84 @@ nlohmann::json Service::getStatistics() const }); } - return { + auto result = nlohmann::json{ {"datasources", datasources}, {"active-requests", impl_->requests_.size()} }; + + auto layerInfoByMap = std::unordered_map>>{}; + for (auto const& [_, info] : impl_->dataSourceInfo_) { + auto& layers = layerInfoByMap[info.mapId_]; + for (auto const& [layerId, layerInfo] : info.layers_) { + layers[layerId] = layerInfo; + } + } + + auto resolveLayerInfo = [&](std::string_view mapId, std::string_view layerId) -> std::shared_ptr { + auto mapIt = layerInfoByMap.find(std::string(mapId)); + if (mapIt == layerInfoByMap.end()) + return std::make_shared(); + auto layerIt = mapIt->second.find(std::string(layerId)); + if (layerIt == mapIt->second.end()) { + auto fallback = std::make_shared(); + fallback->layerId_ = std::string(layerId); + return fallback; + } + return layerIt->second; + }; + + int64_t parsedTiles = 0; + int64_t totalTileBytes = 0; + int64_t parseErrors = 0; + auto featureLayerTotals = nlohmann::json::object(); + auto modelPoolTotals = nlohmann::json::object(); + + auto addTotals = [](nlohmann::json& totals, const nlohmann::json& stats) { + for (const auto& [key, value] : stats.items()) { + if (!value.is_number_integer()) + continue; + totals[key] = totals.value(key, 0) + value.get(); + } + }; + + impl_->cache_->forEachTileLayerBlob( + [&](const MapTileKey& key, const std::string& blob) + { + if (key.layer_ != LayerType::Features) + return; + ++parsedTiles; + totalTileBytes += static_cast(blob.size()); + + try { + std::istringstream inputStream(blob, std::ios::binary); + auto tile = std::make_shared( + inputStream, + [&](auto&& mapId, auto&& layerId) { + return resolveLayerInfo(mapId, layerId); + }, + [&](auto&& nodeId) { + return impl_->cache_->getStringPool(nodeId); + }); + auto sizeStats = tile->serializationSizeStats(); + addTotals(featureLayerTotals, sizeStats["feature-layer"]); + addTotals(modelPoolTotals, sizeStats["model-pool"]); + } + catch (const std::exception&) { + ++parseErrors; + } + }); + + if (parsedTiles > 0) { + result["cached-feature-tree-bytes"] = nlohmann::json{ + {"tile-count", parsedTiles}, + {"total-tile-bytes", totalTileBytes}, + {"parse-errors", parseErrors}, + {"feature-layer", featureLayerTotals}, + {"model-pool", modelPoolTotals} + }; + } + + return result; } } // namespace mapget diff --git a/libs/service/src/sqlitecache.cpp b/libs/service/src/sqlitecache.cpp index 74b2b308..c062e1ec 100644 --- a/libs/service/src/sqlitecache.cpp +++ b/libs/service/src/sqlitecache.cpp @@ -259,6 +259,32 @@ void SQLiteCache::putTileLayerBlob(MapTileKey const& k, std::string const& v) } } +void SQLiteCache::forEachTileLayerBlob(const TileBlobVisitor& cb) const +{ + std::lock_guard lock(dbMutex_); + + sqlite3_stmt* stmt = nullptr; + int rc = sqlite3_prepare_v2(db_, "SELECT key, data FROM tiles", -1, &stmt, nullptr); + if (rc != SQLITE_OK) { + raise(fmt::format("Failed to prepare tile iteration statement: {}", sqlite3_errmsg(db_))); + } + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + const char* key = reinterpret_cast(sqlite3_column_text(stmt, 0)); + const void* data = sqlite3_column_blob(stmt, 1); + int size = sqlite3_column_bytes(stmt, 1); + if (key && data && size >= 0) { + cb(MapTileKey(key), std::string(static_cast(data), size)); + } + } + if (rc != SQLITE_DONE) { + sqlite3_finalize(stmt); + raise(fmt::format("Error iterating cached tiles: {}", sqlite3_errmsg(db_))); + } + + sqlite3_finalize(stmt); +} + void SQLiteCache::cleanupOldestTiles() { // Delete the oldest tile @@ -315,4 +341,4 @@ void SQLiteCache::putStringPoolBlob(std::string_view const& sourceNodeId, std::s } } -} // namespace mapget \ No newline at end of file +} // namespace mapget From 4aeec718bbdecc70503426493fa0384229794e5e Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 15 Jan 2026 11:04:21 +0100 Subject: [PATCH 02/14] Use simfil release branch. --- cmake/deps.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 7473f2b9..1ed0717c 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -15,7 +15,7 @@ CPMAddPackage( "EXPECTED_BUILD_TESTS OFF" "EXPECTED_BUILD_PACKAGE_DEB OFF") CPMAddPackage( - URI "gh:Klebert-Engineering/simfil@0.6.2" + URI "gh:Klebert-Engineering/simfil@0.6.3#v0.6.3" OPTIONS "SIMFIL_WITH_MODEL_JSON ON" "SIMFIL_SHARED OFF") From 0659ee8ca0137ba97fb2c695f2673cb613320371 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 15 Jan 2026 12:15:51 +0100 Subject: [PATCH 03/14] Use TileLayerStream in Cache statistics generation. --- libs/service/src/service.cpp | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index 9f0b1cbf..4abbf9bd 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -709,12 +709,28 @@ nlohmann::json Service::getStatistics() const auto addTotals = [](nlohmann::json& totals, const nlohmann::json& stats) { for (const auto& [key, value] : stats.items()) { - if (!value.is_number_integer()) - continue; - totals[key] = totals.value(key, 0) + value.get(); + if (value.is_number_integer()) + { + totals[key] = totals.value(key, 0) + value.get(); + } + else if (value.is_number_float()) + { + totals[key] = totals.value(key, .0) + value.get(); + } } }; + TileLayerStream::Reader tileReader( + resolveLayerInfo, + [&](auto&& parsedLayer) + { + auto tile = std::static_pointer_cast(parsedLayer); + auto sizeStats = tile->serializationSizeStats(); + addTotals(featureLayerTotals, sizeStats["feature-layer"]); + addTotals(modelPoolTotals, sizeStats["model-pool"]); + }, + impl_->cache_); + impl_->cache_->forEachTileLayerBlob( [&](const MapTileKey& key, const std::string& blob) { @@ -722,20 +738,8 @@ nlohmann::json Service::getStatistics() const return; ++parsedTiles; totalTileBytes += static_cast(blob.size()); - try { - std::istringstream inputStream(blob, std::ios::binary); - auto tile = std::make_shared( - inputStream, - [&](auto&& mapId, auto&& layerId) { - return resolveLayerInfo(mapId, layerId); - }, - [&](auto&& nodeId) { - return impl_->cache_->getStringPool(nodeId); - }); - auto sizeStats = tile->serializationSizeStats(); - addTotals(featureLayerTotals, sizeStats["feature-layer"]); - addTotals(modelPoolTotals, sizeStats["model-pool"]); + tileReader.read(blob); } catch (const std::exception&) { ++parseErrors; From 516c2b6e3dcd62889b954976db4b26f7a2ce1dbd Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 21 Jan 2026 10:24:33 +0100 Subject: [PATCH 04/14] Introduce uWebSockets as dependency. --- cmake/deps.cmake | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 1ed0717c..9e22dc7a 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -64,6 +64,38 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) CPMAddPackage("gh:CLIUtils/CLI11@2.5.0") CPMAddPackage("gh:pboettch/json-schema-validator#2.3.0") CPMAddPackage("gh:okdshin/PicoSHA2@1.0.1") + + CPMAddPackage( + NAME uSockets + GIT_REPOSITORY https://github.com/uNetworking/uSockets + GIT_TAG v0.8.5 + GIT_SHALLOW ON + GIT_SUBMODULES "") + if (NOT TARGET uSockets) + file(GLOB_RECURSE U_SOCKETS_SOURCES "${uSockets_SOURCE_DIR}/src/*.c") + add_library(uSockets STATIC ${U_SOCKETS_SOURCES}) + target_include_directories(uSockets PUBLIC "${uSockets_SOURCE_DIR}/src") + target_compile_definitions(uSockets PRIVATE LIBUS_USE_OPENSSL) + target_link_libraries(uSockets PRIVATE OpenSSL::SSL OpenSSL::Crypto) + if (WIN32) + target_link_libraries(uSockets PRIVATE ws2_32) + endif() + endif() + + CPMAddPackage( + NAME uWebSockets + GIT_REPOSITORY https://github.com/uNetworking/uWebSockets + GIT_TAG v20.37.0 + GIT_SHALLOW ON + GIT_SUBMODULES "") + if (NOT TARGET uWebSockets) + add_library(uWebSockets INTERFACE) + target_include_directories(uWebSockets INTERFACE "${uWebSockets_SOURCE_DIR}/src") + target_link_libraries(uWebSockets INTERFACE uSockets ZLIB::ZLIB) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + target_compile_options(uWebSockets INTERFACE -Wno-deprecated-declarations) + endif() + endif() endif () if (MAPGET_WITH_WHEEL AND NOT TARGET pybind11) From fbc5d48060727451dc88a4150b94174d4727be03 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 22 Jan 2026 10:54:27 +0100 Subject: [PATCH 05/14] Backend transition: uWS for server, cpp-httplib for client. --- cmake/deps.cmake | 28 +- libs/http-datasource/CMakeLists.txt | 1 + .../include/mapget/detail/http-server.h | 9 +- .../http-datasource/datasource-server.h | 2 +- .../http-datasource/src/datasource-server.cpp | 175 ++-- libs/http-datasource/src/http-server.cpp | 335 +++++- libs/http-service/CMakeLists.txt | 1 + .../include/mapget/http-service/http-client.h | 2 +- .../mapget/http-service/http-service.h | 3 +- libs/http-service/src/http-client.cpp | 9 +- libs/http-service/src/http-service.cpp | 990 ++++++++++-------- libs/service/src/datasource.cpp | 7 +- 12 files changed, 1003 insertions(+), 559 deletions(-) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 9e22dc7a..a990b4a2 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -65,6 +65,19 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) CPMAddPackage("gh:pboettch/json-schema-validator#2.3.0") CPMAddPackage("gh:okdshin/PicoSHA2@1.0.1") + if (WIN32) + CPMAddPackage( + NAME libuv + GIT_REPOSITORY https://github.com/libuv/libuv + GIT_TAG v1.48.0 + GIT_SHALLOW ON + OPTIONS + "LIBUV_BUILD_TESTS OFF" + "LIBUV_BUILD_BENCH OFF" + "LIBUV_BUILD_SHARED OFF" + "LIBUV_BUILD_EXAMPLES OFF") + endif() + CPMAddPackage( NAME uSockets GIT_REPOSITORY https://github.com/uNetworking/uSockets @@ -72,13 +85,22 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) GIT_SHALLOW ON GIT_SUBMODULES "") if (NOT TARGET uSockets) - file(GLOB_RECURSE U_SOCKETS_SOURCES "${uSockets_SOURCE_DIR}/src/*.c") + file(GLOB_RECURSE U_SOCKETS_SOURCES CONFIGURE_DEPENDS + "${uSockets_SOURCE_DIR}/src/*.c" + "${uSockets_SOURCE_DIR}/src/*.cpp") add_library(uSockets STATIC ${U_SOCKETS_SOURCES}) target_include_directories(uSockets PUBLIC "${uSockets_SOURCE_DIR}/src") target_compile_definitions(uSockets PRIVATE LIBUS_USE_OPENSSL) - target_link_libraries(uSockets PRIVATE OpenSSL::SSL OpenSSL::Crypto) + target_link_libraries(uSockets PUBLIC OpenSSL::SSL OpenSSL::Crypto) if (WIN32) - target_link_libraries(uSockets PRIVATE ws2_32) + target_link_libraries(uSockets PUBLIC ws2_32) + if (TARGET uv_a) + target_link_libraries(uSockets PUBLIC uv_a) + elseif (TARGET uv) + target_link_libraries(uSockets PUBLIC uv) + else() + message(FATAL_ERROR "libuv was requested for uSockets on Windows, but no CMake target (uv_a/uv) was found.") + endif() endif() endif() diff --git a/libs/http-datasource/CMakeLists.txt b/libs/http-datasource/CMakeLists.txt index e46557b5..917eded2 100644 --- a/libs/http-datasource/CMakeLists.txt +++ b/libs/http-datasource/CMakeLists.txt @@ -18,6 +18,7 @@ target_include_directories(mapget-http-datasource target_link_libraries(mapget-http-datasource PUBLIC httplib::httplib + uWebSockets mapget-model mapget-service tiny-process-library) diff --git a/libs/http-datasource/include/mapget/detail/http-server.h b/libs/http-datasource/include/mapget/detail/http-server.h index c80b7b62..bcc9656f 100644 --- a/libs/http-datasource/include/mapget/detail/http-server.h +++ b/libs/http-datasource/include/mapget/detail/http-server.h @@ -3,9 +3,10 @@ #include #include -// Pre-declare httplib::Server to avoid including httplib.h in header -namespace httplib { -class Server; +// Forward declare uWebSockets app type to avoid including uWS headers in public headers. +namespace uWS { +template struct TemplatedApp; +using App = TemplatedApp; } namespace mapget { @@ -76,7 +77,7 @@ class HttpServer * This function is called upon the first call to go(), * and allows any derived server class to add endpoints. */ - virtual void setup(httplib::Server&) = 0; + virtual void setup(uWS::App&) = 0; /** * Derived servers can use this to control whether diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h index ed11adde..501f01ac 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h @@ -50,7 +50,7 @@ class DataSourceServer : public HttpServer DataSourceInfo const& info(); private: - void setup(httplib::Server&) override; + void setup(uWS::App&) override; struct Impl; std::unique_ptr impl_; diff --git a/libs/http-datasource/src/datasource-server.cpp b/libs/http-datasource/src/datasource-server.cpp index f6ea10f0..a8435058 100644 --- a/libs/http-datasource/src/datasource-server.cpp +++ b/libs/http-datasource/src/datasource-server.cpp @@ -1,54 +1,51 @@ #include "datasource-server.h" -#include "mapget/detail/http-server.h" -#include "mapget/model/sourcedatalayer.h" -#include "mapget/model/featurelayer.h" + +#include "mapget/log.h" #include "mapget/model/info.h" -#include "mapget/model/layer.h" #include "mapget/model/stream.h" -#include "httplib.h" +#include + #include #include +#include + +#include "fmt/format.h" -namespace mapget { +namespace mapget +{ struct DataSourceServer::Impl { DataSourceInfo info_; - std::function tileFeatureCallback_ = [](auto&&) - { + std::function tileFeatureCallback_ = [](auto&&) { throw std::runtime_error("TileFeatureLayer callback is unset!"); }; - std::function tileSourceDataCallback_ = [](auto&&) - { + std::function tileSourceDataCallback_ = [](auto&&) { throw std::runtime_error("TileSourceDataLayer callback is unset!"); }; std::function(const LocateRequest&)> locateCallback_; std::shared_ptr strings_; - explicit Impl(DataSourceInfo info) - : info_(std::move(info)), strings_(std::make_shared(info_.nodeId_)) + explicit Impl(DataSourceInfo info) : info_(std::move(info)), strings_(std::make_shared(info_.nodeId_)) { } }; -DataSourceServer::DataSourceServer(DataSourceInfo const& info) - : HttpServer(), impl_(new Impl(info)) +DataSourceServer::DataSourceServer(DataSourceInfo const& info) : HttpServer(), impl_(new Impl(info)) { printPortToStdOut(true); } DataSourceServer::~DataSourceServer() = default; -DataSourceServer& -DataSourceServer::onTileFeatureRequest(std::function const& callback) +DataSourceServer& DataSourceServer::onTileFeatureRequest(std::function const& callback) { impl_->tileFeatureCallback_ = callback; return *this; } -DataSourceServer& -DataSourceServer::onTileSourceDataRequest(std::function const& callback) +DataSourceServer& DataSourceServer::onTileSourceDataRequest(std::function const& callback) { impl_->tileSourceDataCallback_ = callback; return *this; @@ -61,50 +58,49 @@ DataSourceServer& DataSourceServer::onLocateRequest( return *this; } -DataSourceInfo const& DataSourceServer::info() { - return impl_->info_; -} +DataSourceInfo const& DataSourceServer::info() { return impl_->info_; } -void DataSourceServer::setup(httplib::Server& server) +void DataSourceServer::setup(uWS::App& app) { - // Set up GET /tile endpoint - server.Get( - "/tile", - [this](const httplib::Request& req, httplib::Response& res) { - // Extract parameters from request. - auto layerIdParam = req.get_param_value("layer"); - auto layer = impl_->info_.getLayer(layerIdParam); - - auto tileIdParam = TileId{std::stoull(req.get_param_value("tileId"))}; + app.get("/tile", [this](auto* res, auto* req) { + try { + auto layerIdParam = req->getQuery("layer"); + auto tileIdParam = req->getQuery("tileId"); + + if (layerIdParam.empty() || tileIdParam.empty()) { + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Missing query parameter: layer and/or tileId"); + return; + } + + auto layer = impl_->info_.getLayer(std::string(layerIdParam)); + + TileId tileId{std::stoull(std::string(tileIdParam))}; + auto stringPoolOffsetParam = (simfil::StringId)0; - if (req.has_param("stringPoolOffset")) - stringPoolOffsetParam = (simfil::StringId) - std::stoul(req.get_param_value("stringPoolOffset")); + auto stringPoolOffsetStr = req->getQuery("stringPoolOffset"); + if (!stringPoolOffsetStr.empty()) { + stringPoolOffsetParam = (simfil::StringId)std::stoul(std::string(stringPoolOffsetStr)); + } std::string responseType = "binary"; - if (req.has_param("responseType")) - responseType = req.get_param_value("responseType"); + auto responseTypeStr = req->getQuery("responseType"); + if (!responseTypeStr.empty()) { + responseType = std::string(responseTypeStr); + } - // Create response TileFeatureLayer. auto tileLayer = [&]() -> std::shared_ptr { switch (layer->type_) { case mapget::LayerType::Features: { auto tileFeatureLayer = std::make_shared( - tileIdParam, - impl_->info_.nodeId_, - impl_->info_.mapId_, - layer, - impl_->strings_); + tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); impl_->tileFeatureCallback_(tileFeatureLayer); return tileFeatureLayer; } case mapget::LayerType::SourceData: { auto tileSourceLayer = std::make_shared( - tileIdParam, - impl_->info_.nodeId_, - impl_->info_.mapId_, - layer, - impl_->strings_); + tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); impl_->tileSourceDataCallback_(tileSourceLayer); return tileSourceLayer; } @@ -113,45 +109,72 @@ void DataSourceServer::setup(httplib::Server& server) } }(); - // Serialize TileLayer using TileLayerStream. if (responseType == "binary") { - std::stringstream content; + std::string content; TileLayerStream::StringPoolOffsetMap stringPoolOffsets{ {impl_->info_.nodeId_, stringPoolOffsetParam}}; TileLayerStream::Writer layerWriter{ - [&](auto&& msg, auto&& msgType) { content << msg; }, + [&](std::string bytes, TileLayerStream::MessageType) { content.append(bytes); }, stringPoolOffsets}; layerWriter.write(tileLayer); - res.set_content(content.str(), "application/binary"); - } - else { - res.set_content(nlohmann::to_string(tileLayer->toJson()), "application/json"); - } - }); - - // Set up GET /info endpoint - server.Get( - "/info", - [this](const httplib::Request&, httplib::Response& res) { - nlohmann::json j = impl_->info_.toJson(); - res.set_content(j.dump(), "application/json"); - }); - // Set up POST /locate endpoint - server.Post( - "/locate", - [this](const httplib::Request& req, httplib::Response& res) { - LocateRequest parsedReq(nlohmann::json::parse(req.body)); - auto responseJson = nlohmann::json::array(); - - if (impl_->locateCallback_) { - for (auto const& response : impl_->locateCallback_(parsedReq)) { - responseJson.emplace_back(response.serialize()); - } + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/binary"); + res->end(content); + } else { + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(tileLayer->toJson().dump()); } + } + catch (std::exception const& e) { + res->writeStatus("500 Internal Server Error"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Error: ") + e.what()); + } + }); + + app.get("/info", [this](auto* res, auto* /*req*/) { + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(impl_->info_.toJson().dump()); + }); + + app.post("/locate", [this](auto* res, auto* /*req*/) { + auto aborted = std::make_shared(false); + res->onAborted([aborted]() { *aborted = true; }); + + res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { + if (*aborted) + return; + body.append(chunk.data(), chunk.size()); + if (!last) + return; + try { + LocateRequest parsedReq(nlohmann::json::parse(body)); + auto responseJson = nlohmann::json::array(); + + if (impl_->locateCallback_) { + for (auto const& response : impl_->locateCallback_(parsedReq)) { + responseJson.emplace_back(response.serialize()); + } + } - res.set_content(responseJson.dump(), "application/json"); + if (*aborted) + return; + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(responseJson.dump()); + } + catch (std::exception const& e) { + if (*aborted) + return; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Invalid request: ") + e.what()); + } }); + }); } } // namespace mapget diff --git a/libs/http-datasource/src/http-server.cpp b/libs/http-datasource/src/http-server.cpp index 6b49eb66..cdceab2c 100644 --- a/libs/http-datasource/src/http-server.cpp +++ b/libs/http-datasource/src/http-server.cpp @@ -1,10 +1,23 @@ #include "mapget/detail/http-server.h" #include "mapget/log.h" -#include "httplib.h" -#include +#include +#include + #include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include #include "fmt/format.h" @@ -14,14 +27,120 @@ namespace mapget // initialize the atomic activeHttpServer with nullptr static std::atomic activeHttpServer = nullptr; +namespace +{ +struct MountPoint +{ + std::string urlPrefix; + std::filesystem::path fsRoot; +}; + +[[nodiscard]] bool startsWith(std::string_view s, std::string_view prefix) +{ + return s.size() >= prefix.size() && s.substr(0, prefix.size()) == prefix; +} + +[[nodiscard]] std::string normalizeUrlPrefix(std::string prefix) +{ + if (prefix.empty()) + prefix = "/"; + if (prefix.front() != '/') + prefix.insert(prefix.begin(), '/'); + if (prefix.size() > 1 && prefix.back() == '/') + prefix.pop_back(); + return prefix; +} + +[[nodiscard]] std::string_view guessMimeType(std::filesystem::path const& filePath) +{ + auto ext = filePath.extension().string(); + std::ranges::transform(ext, ext.begin(), [](unsigned char c) { return (char)std::tolower(c); }); + + if (ext == ".html" || ext == ".htm") + return "text/html"; + if (ext == ".css") + return "text/css"; + if (ext == ".js") + return "application/javascript"; + if (ext == ".json") + return "application/json"; + if (ext == ".svg") + return "image/svg+xml"; + if (ext == ".png") + return "image/png"; + if (ext == ".jpg" || ext == ".jpeg") + return "image/jpeg"; + if (ext == ".ico") + return "image/x-icon"; + if (ext == ".woff2") + return "font/woff2"; + if (ext == ".woff") + return "font/woff"; + if (ext == ".ttf") + return "font/ttf"; + if (ext == ".txt") + return "text/plain"; + + return "application/octet-stream"; +} + +[[nodiscard]] std::optional resolveStaticFile( + std::vector const& mounts, + std::string_view urlPath) +{ + if (mounts.empty()) + return std::nullopt; + if (!startsWith(urlPath, "/")) + return std::nullopt; + + // Longest-prefix match. + MountPoint const* best = nullptr; + for (auto const& m : mounts) { + if (startsWith(urlPath, m.urlPrefix) && (!best || m.urlPrefix.size() > best->urlPrefix.size())) + best = &m; + } + if (!best) + return std::nullopt; + + std::string_view remainder = urlPath.substr(best->urlPrefix.size()); + if (!remainder.empty() && remainder.front() == '/') + remainder.remove_prefix(1); + + std::filesystem::path relativePath = std::filesystem::path(std::string(remainder)).lexically_normal(); + if (relativePath.empty() || urlPath.back() == '/') + relativePath /= "index.html"; + + // Basic path traversal protection: reject any ".." segments. + for (auto const& part : relativePath) { + if (part == "..") + return std::nullopt; + } + + std::filesystem::path candidate = (best->fsRoot / relativePath).lexically_normal(); + return candidate; +} + +} // namespace + struct HttpServer::Impl { - httplib::Server server_; std::thread serverThread_; + std::atomic_bool running_{false}; + + std::mutex startMutex_; + std::condition_variable startCv_; + bool startNotified_ = false; + std::string startError_; + uint16_t port_ = 0; - bool setupWasCalled_ = false; bool printPortToStdout_ = false; + std::mutex mountsMutex_; + std::vector mounts_; + + uWS::Loop* loop_ = nullptr; + us_listen_socket_t* listenSocket_ = nullptr; + static void handleSignal(int) { // Temporarily holds the current active HttpServer @@ -35,69 +154,174 @@ struct HttpServer::Impl } } } + + void notifyStart(std::string errorMessage = {}) + { + std::lock_guard lock(startMutex_); + startError_ = std::move(errorMessage); + startNotified_ = true; + startCv_.notify_one(); + } }; HttpServer::HttpServer() : impl_(new Impl()) {} -HttpServer::~HttpServer() { +HttpServer::~HttpServer() +{ if (isRunning()) stop(); } -void HttpServer::go( - std::string const& interfaceAddr, - uint16_t port, - uint32_t waitMs) +void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t waitMs) { - if (!impl_->setupWasCalled_) { - // Allow derived class to set up the server - setup(impl_->server_); - impl_->setupWasCalled_ = true; - } - - if (impl_->server_.is_running() || impl_->serverThread_.joinable()) + if (impl_->running_ || impl_->serverThread_.joinable()) raise("HttpServer is already running"); - if (port == 0) { - impl_->port_ = impl_->server_.bind_to_any_port(interfaceAddr); - } - else { - impl_->port_ = port; - impl_->server_.bind_to_port(interfaceAddr, port); + // Reset start state. + { + std::lock_guard lock(impl_->startMutex_); + impl_->startNotified_ = false; + impl_->startError_.clear(); } impl_->serverThread_ = std::thread( - [this, interfaceAddr] + [this, interfaceAddr, port] { - if (impl_->printPortToStdout_) - std::cout << "====== Running on port " << impl_->port_ << " ======" << std::endl; - else - log().info("====== Running on port {} ======", impl_->port_); - impl_->server_.listen_after_bind(); + try { + uWS::App app; + + // Allow derived class to set up the server + setup(app); + + // Copy mounts to avoid locking in the hot path. + std::vector mountsCopy; + { + std::lock_guard lock(impl_->mountsMutex_); + mountsCopy = impl_->mounts_; + } + + if (!mountsCopy.empty()) { + app.get( + "/*", + [mounts = std::move(mountsCopy)](auto* res, auto* req) mutable + { + auto urlPath = req->getUrl(); + auto candidate = resolveStaticFile(mounts, urlPath); + if (!candidate || !std::filesystem::exists(*candidate) || + !std::filesystem::is_regular_file(*candidate)) { + res->writeStatus("404 Not Found"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Not found"); + return; + } + + std::ifstream ifs(*candidate, std::ios::binary); + if (!ifs) { + res->writeStatus("500 Internal Server Error"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Failed to open file"); + return; + } + + std::string content; + ifs.seekg(0, std::ios::end); + content.resize(static_cast(ifs.tellg())); + ifs.seekg(0, std::ios::beg); + if (!content.empty()) { + ifs.read(content.data(), static_cast(content.size())); + } + + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", guessMimeType(*candidate)); + res->end(content); + }); + } + + app.listen( + interfaceAddr, + port, + [this, interfaceAddr, port](us_listen_socket_t* listenSocket) + { + if (!listenSocket) { + impl_->notifyStart( + fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); + return; + } + + impl_->listenSocket_ = listenSocket; + impl_->loop_ = uWS::Loop::get(); + + // Determine actual port (port may be 0 for ephemeral). + impl_->port_ = static_cast( + us_socket_local_port(0, reinterpret_cast(listenSocket))); + + impl_->running_ = true; + impl_->notifyStart(); + + if (impl_->printPortToStdout_) + std::cout << "====== Running on port " << impl_->port_ << " ======" << std::endl; + else + log().info("====== Running on port {} ======", impl_->port_); + }); + + // If listen failed, exit without running the loop. + if (!impl_->running_) { + if (!impl_->startNotified_) { + impl_->notifyStart(fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); + } + return; + } + + app.run(); + } + catch (std::exception const& e) { + impl_->notifyStart(e.what()); + } + + impl_->running_ = false; + impl_->listenSocket_ = nullptr; + impl_->loop_ = nullptr; }); - std::this_thread::sleep_for(std::chrono::milliseconds(waitMs)); - if (!impl_->server_.is_running() || !impl_->server_.is_valid()) - raise(fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); + std::unique_lock lk(impl_->startMutex_); + if (!impl_->startCv_.wait_for( + lk, + std::chrono::milliseconds(waitMs), + [this] { return impl_->startNotified_; })) { + raise(fmt::format("Could not start HttpServer on {}:{} (timeout)", interfaceAddr, port)); + } + + if (!impl_->startError_.empty()) + raise(impl_->startError_); } -bool HttpServer::isRunning() { - return impl_->server_.is_running(); +bool HttpServer::isRunning() +{ + return impl_->running_; } -void HttpServer::stop() { - if (!impl_->server_.is_running()) +void HttpServer::stop() +{ + if (!impl_->serverThread_.joinable()) return; - impl_->server_.stop(); - impl_->serverThread_.join(); + if (impl_->loop_ && impl_->listenSocket_) { + auto* loop = impl_->loop_; + auto* listenSocket = impl_->listenSocket_; + loop->defer([listenSocket]() { us_listen_socket_close(0, listenSocket); }); + } + + if (impl_->serverThread_.get_id() != std::this_thread::get_id()) + impl_->serverThread_.join(); } -uint16_t HttpServer::port() const { +uint16_t HttpServer::port() const +{ return impl_->port_; } -void HttpServer::waitForSignal() { +void HttpServer::waitForSignal() +{ // So the signal handler knows what to call activeHttpServer = this; @@ -107,24 +331,43 @@ void HttpServer::waitForSignal() { // Wait for the signal handler to stop us, or the server to shut down on its own. while (isRunning()) { - std::this_thread::sleep_for(std::chrono::milliseconds (200)); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); } activeHttpServer = nullptr; } -bool HttpServer::mountFileSystem(const std::string& pathFromTo) +bool HttpServer::mountFileSystem(std::string const& pathFromTo) { using namespace std::ranges; - auto parts = pathFromTo | views::split(':') | views::transform([](auto&& s){return std::string(&*s.begin(), distance(s));}); + auto parts = pathFromTo | views::split(':') | + views::transform([](auto&& s) { return std::string(&*s.begin(), distance(s)); }); auto partsVec = std::vector(parts.begin(), parts.end()); - if (partsVec.size() == 1) - return impl_->server_.set_mount_point("/", partsVec[0]); - return impl_->server_.set_mount_point(partsVec[0], partsVec[1]); + std::string urlPrefix; + std::filesystem::path fsRoot; + if (partsVec.size() == 1) { + urlPrefix = "/"; + fsRoot = partsVec[0]; + } else if (partsVec.size() == 2) { + urlPrefix = partsVec[0]; + fsRoot = partsVec[1]; + } else { + return false; + } + + urlPrefix = normalizeUrlPrefix(std::move(urlPrefix)); + + if (!std::filesystem::exists(fsRoot) || !std::filesystem::is_directory(fsRoot)) + return false; + + std::lock_guard lock(impl_->mountsMutex_); + impl_->mounts_.push_back(MountPoint{std::move(urlPrefix), std::move(fsRoot)}); + return true; } -void HttpServer::printPortToStdOut(bool enabled) { +void HttpServer::printPortToStdOut(bool enabled) +{ impl_->printPortToStdout_ = enabled; } diff --git a/libs/http-service/CMakeLists.txt b/libs/http-service/CMakeLists.txt index ce3d733e..23c2f847 100644 --- a/libs/http-service/CMakeLists.txt +++ b/libs/http-service/CMakeLists.txt @@ -18,6 +18,7 @@ target_include_directories(mapget-http-service target_link_libraries(mapget-http-service PUBLIC httplib::httplib + uWebSockets yaml-cpp CLI11::CLI11 nlohmann_json_schema_validator diff --git a/libs/http-service/include/mapget/http-service/http-client.h b/libs/http-service/include/mapget/http-service/http-client.h index 67c293d6..7fad612d 100644 --- a/libs/http-service/include/mapget/http-service/http-client.h +++ b/libs/http-service/include/mapget/http-service/http-client.h @@ -18,7 +18,7 @@ class HttpClient * endpoint, and caches the result for the lifetime of this object. * @param enableCompression Enable gzip compression for responses (default: true) */ - explicit HttpClient(std::string const& host, uint16_t port, httplib::Headers headers = {}, bool enableCompression = true); + explicit HttpClient(std::string const& host, uint16_t port, AuthHeaders headers = {}, bool enableCompression = true); ~HttpClient(); /** diff --git a/libs/http-service/include/mapget/http-service/http-service.h b/libs/http-service/include/mapget/http-service/http-service.h index 41f834c8..1e87225d 100644 --- a/libs/http-service/include/mapget/http-service/http-service.h +++ b/libs/http-service/include/mapget/http-service/http-service.h @@ -1,6 +1,5 @@ #pragma once -#include "httplib.h" #include "mapget/detail/http-server.h" #include "mapget/model/featurelayer.h" #include "mapget/model/stream.h" @@ -56,7 +55,7 @@ class HttpService : public HttpServer, public Service ~HttpService() override; protected: - void setup(httplib::Server& server) override; + void setup(uWS::App& app) override; private: struct Impl; diff --git a/libs/http-service/src/http-client.cpp b/libs/http-service/src/http-client.cpp index a002c34f..95ebdbea 100644 --- a/libs/http-service/src/http-client.cpp +++ b/libs/http-service/src/http-client.cpp @@ -11,10 +11,13 @@ struct HttpClient::Impl { std::shared_ptr stringPoolProvider_; httplib::Headers headers_; - Impl(std::string const& host, uint16_t port, httplib::Headers headers, bool enableCompression) : + Impl(std::string const& host, uint16_t port, AuthHeaders headers, bool enableCompression) : client_(host, port), - headers_(std::move(headers)) + headers_() { + for (auto const& [k, v] : headers) { + headers_.emplace(k, v); + } // Add Accept-Encoding header if compression is enabled and not already present if (enableCompression) { bool hasAcceptEncoding = false; @@ -51,7 +54,7 @@ struct HttpClient::Impl { } }; -HttpClient::HttpClient(const std::string& host, uint16_t port, httplib::Headers headers, bool enableCompression) : impl_( +HttpClient::HttpClient(const std::string& host, uint16_t port, AuthHeaders headers, bool enableCompression) : impl_( std::make_unique(host, port, std::move(headers), enableCompression)) {} HttpClient::~HttpClient() = default; diff --git a/libs/http-service/src/http-service.cpp b/libs/http-service/src/http-service.cpp index 0137a1b7..9f09586f 100644 --- a/libs/http-service/src/http-service.cpp +++ b/libs/http-service/src/http-service.cpp @@ -1,19 +1,29 @@ #include "http-service.h" + +#include "cli.h" #include "mapget/log.h" #include "mapget/service/config.h" +#include + +#include #include -#include +#include #include #include #include +#include #include +#include +#include +#include +#include #include -#include "cli.h" -#include "httplib.h" + #include "nlohmann/json-schema.hpp" #include "nlohmann/json.hpp" #include "yaml-cpp/yaml.h" + #include #ifdef __linux__ @@ -27,27 +37,28 @@ namespace { /** - * Simple gzip compressor for streaming compression + * Simple gzip compressor for streaming compression. */ -class GzipCompressor { +class GzipCompressor +{ public: - GzipCompressor() { + GzipCompressor() + { strm_.zalloc = Z_NULL; strm_.zfree = Z_NULL; strm_.opaque = Z_NULL; // 16+MAX_WBITS enables gzip format (not just deflate) - int ret = deflateInit2(&strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, - 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY); + int ret = deflateInit2( + &strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY); if (ret != Z_OK) { throw std::runtime_error("Failed to initialize gzip compressor"); } } - ~GzipCompressor() { - deflateEnd(&strm_); - } + ~GzipCompressor() { deflateEnd(&strm_); } - std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) { + std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) + { std::string result; if (size == 0 && flush_mode == Z_NO_FLUSH) { return result; @@ -60,12 +71,12 @@ class GzipCompressor { do { strm_.avail_out = sizeof(outbuf); strm_.next_out = reinterpret_cast(outbuf); - + int ret = deflate(&strm_, flush_mode); if (ret == Z_STREAM_ERROR) { throw std::runtime_error("Gzip compression failed"); } - + size_t have = sizeof(outbuf) - strm_.avail_out; result.append(outbuf, have); } while (strm_.avail_out == 0); @@ -73,19 +84,26 @@ class GzipCompressor { return result; } - std::string finish() { - return compress(nullptr, 0, Z_FINISH); - } + std::string finish() { return compress(nullptr, 0, Z_FINISH); } private: z_stream strm_{}; }; -/** - * Recursively convert a YAML node to a JSON object, - * with special handling for sensitive fields. - * The function returns a nlohmann::json object and updates maskedSecretMap. - */ +[[nodiscard]] AuthHeaders authHeadersFromRequest(uWS::HttpRequest* req) +{ + AuthHeaders headers; + for (auto const& [k, v] : *req) { + headers.emplace(std::string(k), std::string(v)); + } + return headers; +} + +[[nodiscard]] bool containsGzip(std::string_view acceptEncoding) +{ + return !acceptEncoding.empty() && acceptEncoding.find("gzip") != std::string_view::npos; +} + } // namespace struct HttpService::Impl @@ -95,64 +113,47 @@ struct HttpService::Impl mutable std::atomic binaryRequestCounter_{0}; mutable std::atomic jsonRequestCounter_{0}; - explicit Impl(HttpService& self, const HttpServiceConfig& config) - : self_(self), config_(config) {} + explicit Impl(HttpService& self, const HttpServiceConfig& config) : self_(self), config_(config) {} - enum class ResponseType { - Binary, - Json - }; + enum class ResponseType { Binary, Json }; + + void tryMemoryTrim(ResponseType responseType) const + { + uint64_t interval = + (responseType == ResponseType::Binary) ? config_.memoryTrimIntervalBinary : config_.memoryTrimIntervalJson; + + if (interval == 0) { + return; + } + + auto& counter = (responseType == ResponseType::Binary) ? binaryRequestCounter_ : jsonRequestCounter_; + auto count = counter.fetch_add(1, std::memory_order_relaxed); + if ((count % interval) != 0) { + return; + } - void tryMemoryTrim(ResponseType responseType) const { - uint64_t interval = (responseType == ResponseType::Binary) - ? config_.memoryTrimIntervalBinary - : config_.memoryTrimIntervalJson; - - if (interval > 0) { - auto& counter = (responseType == ResponseType::Binary) - ? binaryRequestCounter_ - : jsonRequestCounter_; - - auto count = counter.fetch_add(1, std::memory_order_relaxed); - if ((count % interval) == 0) { #ifdef __linux__ - // Only log in debug builds to reduce overhead - #ifndef NDEBUG - const char* typeStr = (responseType == ResponseType::Binary) ? "binary" : "JSON"; - log().debug("Trimming memory after {} {} requests (interval: {})", count, typeStr, interval); - #endif - malloc_trim(0); +#ifndef NDEBUG + const char* typeStr = (responseType == ResponseType::Binary) ? "binary" : "JSON"; + log().debug("Trimming memory after {} {} requests (interval: {})", count, typeStr, interval); +#endif + malloc_trim(0); #endif - // On non-Linux platforms, this is a no-op but we still track the counter - } - } } - // Use a shared buffer for the responses and a mutex for thread safety. - struct HttpTilesRequestState + struct TilesStreamState : std::enable_shared_from_this { static constexpr auto binaryMimeType = "application/binary"; static constexpr auto jsonlMimeType = "application/jsonl"; static constexpr auto anyMimeType = "*/*"; - std::mutex mutex_; - std::condition_variable resultEvent_; - - uint64_t requestId_; - std::stringstream buffer_; - std::string responseType_; - std::unique_ptr writer_; - std::vector requests_; - TileLayerStream::StringPoolOffsetMap stringOffsets_; - std::unique_ptr compressor_; // Store compressor per request - - HttpTilesRequestState() + explicit TilesStreamState(Impl const& impl, uWS::HttpResponse* res, uWS::Loop* loop) + : impl_(impl), res_(res), loop_(loop) { static std::atomic_uint64_t nextRequestId; - writer_ = std::make_unique( - [&, this](auto&& msg, auto&& msgType) { buffer_ << msg; }, - stringOffsets_); requestId_ = nextRequestId++; + writer_ = std::make_unique( + [this](auto&& msg, auto&& /*msgType*/) { appendOutgoingUnlocked(msg); }, stringOffsets_); } void parseRequestFromJson(nlohmann::json const& requestJson) @@ -161,57 +162,194 @@ struct HttpService::Impl std::string layerId = requestJson["layerId"]; std::vector tileIds; tileIds.reserve(requestJson["tileIds"].size()); - for (auto const& tid : requestJson["tileIds"].get>()) + for (auto const& tid : requestJson["tileIds"].get>()) { tileIds.emplace_back(tid); - requests_ - .push_back(std::make_shared(mapId, layerId, std::move(tileIds))); + } + requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); } - void setResponseType(std::string const& s) + [[nodiscard]] bool setResponseTypeFromAccept(std::string_view acceptHeader, std::string& error) { - responseType_ = s; - if (responseType_ == HttpTilesRequestState::binaryMimeType) - return; - if (responseType_ == HttpTilesRequestState::jsonlMimeType) - return; - if (responseType_ == HttpTilesRequestState::anyMimeType) { + responseType_ = std::string(acceptHeader); + if (responseType_.empty()) + responseType_ = anyMimeType; + if (responseType_ == anyMimeType) responseType_ = binaryMimeType; + + if (responseType_ == binaryMimeType) { + trimResponseType_ = ResponseType::Binary; + return true; + } + if (responseType_ == jsonlMimeType) { + trimResponseType_ = ResponseType::Json; + return true; + } + + error = "Unknown Accept header value: " + responseType_; + return false; + } + + void enableGzip() { compressor_ = std::make_unique(); } + + void onAborted() + { + if (aborted_.exchange(true)) return; + for (auto const& req : requests_) { + if (!req->isDone()) { + impl_.self_.abort(req); + } } - raise(fmt::format("Unknown Accept-Header value {}", responseType_)); } void addResult(TileLayer::Ptr const& result) { - std::unique_lock lock(mutex_); - log().debug("Response ready: {}", MapTileKey(*result).toString()); - if (responseType_ == binaryMimeType) { - // Binary response - writer_->write(result); - } - else { - // JSON response - optimize with compact dump settings - // TODO: Implement direct streaming with result->writeGeoJsonTo(buffer_) - // to avoid intermediate JSON object creation entirely - auto json = result->toJson(); - // Use compact dump: no indentation, no spaces, ignore errors - // This reduces string allocation overhead - buffer_ << json.dump(-1, ' ', false, nlohmann::json::error_handler_t::ignore) << "\n"; - } - resultEvent_.notify_one(); + { + std::lock_guard lock(mutex_); + if (aborted_) + return; + + log().debug("Response ready: {}", MapTileKey(*result).toString()); + if (responseType_ == binaryMimeType) { + writer_->write(result); + } else { + auto dumped = result->toJson().dump( + -1, ' ', false, nlohmann::json::error_handler_t::ignore); + appendOutgoingUnlocked(dumped); + appendOutgoingUnlocked("\n"); + } + } + scheduleDrain(); + } + + void onRequestDone() + { + { + std::lock_guard lock(mutex_); + if (aborted_) + return; + + bool allDoneNow = std::all_of( + requests_.begin(), requests_.end(), [](auto const& r) { return r->isDone(); }); + + if (allDoneNow && !allDone_) { + allDone_ = true; + if (responseType_ == binaryMimeType && !endOfStreamSent_) { + writer_->sendEndOfStream(); + endOfStreamSent_ = true; + } + } + } + scheduleDrain(); + } + + void scheduleDrain() + { + if (aborted_ || responseEnded_) + return; + if (drainScheduled_.exchange(true)) + return; + + auto weak = weak_from_this(); + loop_->defer([weak = std::move(weak)]() mutable { + if (auto self = weak.lock()) { + self->drainOnLoop(); + } + }); + } + + void drainOnLoop() + { + drainScheduled_ = false; + if (aborted_ || responseEnded_) + return; + + constexpr size_t maxChunk = 64 * 1024; + + for (;;) { + std::string chunk; + bool done = false; + { + std::lock_guard lock(mutex_); + if (!pending_.empty()) { + size_t n = std::min(pending_.size(), maxChunk); + chunk.assign(pending_.data(), n); + pending_.erase(0, n); + } else { + if (allDone_ && compressor_ && !compressionFinished_) { + pending_.append(compressor_->finish()); + compressionFinished_ = true; + continue; + } + done = allDone_; + } + } + + if (!chunk.empty()) { + bool ok = res_->write(chunk); + if (!ok) { + // Backpressure: resume in onWritable. + return; + } + continue; + } + + if (done) { + responseEnded_ = true; + res_->end(); + impl_.tryMemoryTrim(trimResponseType_); + } + return; + } } + + void appendOutgoingUnlocked(std::string_view bytes) + { + if (bytes.empty()) + return; + + if (compressor_) { + pending_.append(compressor_->compress(bytes.data(), bytes.size())); + } else { + pending_.append(bytes); + } + } + + Impl const& impl_; + uWS::HttpResponse* res_; + uWS::Loop* loop_; + + std::mutex mutex_; + uint64_t requestId_ = 0; + + std::string responseType_; + ResponseType trimResponseType_ = ResponseType::Binary; + + std::string pending_; + std::unique_ptr writer_; + std::vector requests_; + TileLayerStream::StringPoolOffsetMap stringOffsets_; + + std::unique_ptr compressor_; + bool compressionFinished_ = false; + bool endOfStreamSent_ = false; + bool allDone_ = false; + + std::atomic_bool aborted_{false}; + std::atomic_bool drainScheduled_{false}; + std::atomic_bool responseEnded_{false}; }; mutable std::mutex clientRequestMapMutex_; - mutable std::unordered_map> requestStatePerClientId_; + mutable std::unordered_map> requestStatePerClientId_; - void abortRequestsForClientId(std::string clientId, std::shared_ptr newState = nullptr) const + void abortRequestsForClientId( + std::string const& clientId, + std::shared_ptr newState = nullptr) const { std::unique_lock clientRequestMapAccess(clientRequestMapMutex_); auto clientRequestIt = requestStatePerClientId_.find(clientId); if (clientRequestIt != requestStatePerClientId_.end()) { - // Ensure that any previous requests from the same clientId - // are finished post-haste! bool anySoftAbort = false; for (auto const& req : clientRequestIt->second->requests_) { if (!req->isDone()) { @@ -224,205 +362,173 @@ struct HttpService::Impl requestStatePerClientId_.erase(clientRequestIt); } if (newState) { - requestStatePerClientId_.emplace(clientId, newState); + requestStatePerClientId_.emplace(clientId, std::move(newState)); } } - /** - * Wraps around the generic mapget service's request() function - * to include httplib request decoding and response encoding. - */ - void handleTilesRequest(const httplib::Request& req, httplib::Response& res) const + void handleTilesRequest(uWS::HttpResponse* res, uWS::HttpRequest* req) const { - // Parse the JSON request. - nlohmann::json j = nlohmann::json::parse(req.body); - auto requestsJson = j["requests"]; - - // TODO: Limit number of requests to avoid DoS to other users. - // Within one HTTP request, all requested tiles from the same map+layer - // combination should be in a single LayerTilesRequest. - auto state = std::make_shared(); - log().info("Processing tiles request {}", state->requestId_); - for (auto& requestJson : requestsJson) { - state->parseRequestFromJson(requestJson); - } + auto* loop = uWS::Loop::get(); + auto state = std::make_shared(*this, res, loop); + + std::string accept = std::string(req->getHeader("accept")); + std::string acceptEncoding = std::string(req->getHeader("accept-encoding")); + auto clientHeaders = authHeadersFromRequest(req); + + res->onAborted([state]() { state->onAborted(); }); + + res->onData([this, + res, + state, + clientHeaders = std::move(clientHeaders), + accept = std::move(accept), + acceptEncoding = std::move(acceptEncoding), + body = std::string()](std::string_view chunk, bool last) mutable { + if (state->aborted_ || state->responseEnded_) + return; + + body.append(chunk.data(), chunk.size()); + if (!last) + return; - // Parse stringPoolOffsets. - if (j.contains("stringPoolOffsets")) { - for (auto& item : j["stringPoolOffsets"].items()) { - state->stringOffsets_[item.key()] = item.value().get(); + nlohmann::json j; + try { + j = nlohmann::json::parse(body); + } + catch (const std::exception& e) { + state->responseEnded_ = true; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Invalid JSON: ") + e.what()); + return; } - } - // Determine response type. - state->setResponseType(req.get_header_value("Accept")); + auto requestsIt = j.find("requests"); + if (requestsIt == j.end() || !requestsIt->is_array()) { + state->responseEnded_ = true; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Missing or invalid 'requests' array"); + return; + } - // Process requests. - for (auto& request : state->requests_) { - request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); - request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); - request->onDone_ = [state](RequestStatus r) - { - state->resultEvent_.notify_one(); - }; - } - auto canProcess = self_.request( - state->requests_, - AuthHeaders{req.headers.begin(), req.headers.end()}); - - if (!canProcess) { - // Send a status report detailing for each request - // whether its data source is unavailable or it was aborted. - res.status = 400; - std::vector> requestStatuses{}; - for (const auto& r : state->requests_) { - requestStatuses.push_back(static_cast>(r->getStatus())); - if (r->getStatus() == RequestStatus::Unauthorized) { - res.status = 403; // Forbidden. - } + log().info("Processing tiles request {}", state->requestId_); + for (auto& requestJson : *requestsIt) { + state->parseRequestFromJson(requestJson); } - res.set_content( - nlohmann::json::object({{"requestStatuses", requestStatuses}}).dump(), - "application/json"); - return; - } - // Parse/Process clientId. - if (j.contains("clientId")) { - auto clientId = j["clientId"].get(); - abortRequestsForClientId(clientId, state); - } + if (j.contains("stringPoolOffsets")) { + for (auto& item : j["stringPoolOffsets"].items()) { + state->stringOffsets_[item.key()] = item.value().get(); + } + } - // Check if client accepts gzip compression - bool enableGzip = false; - if (req.has_header("Accept-Encoding")) { - std::string acceptEncoding = req.get_header_value("Accept-Encoding"); - enableGzip = acceptEncoding.find("gzip") != std::string::npos; - log().debug("Accept-Encoding header: '{}', enableGzip: {}", acceptEncoding, enableGzip); - } else { - log().debug("No Accept-Encoding header present"); - } + std::string acceptError; + if (!state->setResponseTypeFromAccept(accept, acceptError)) { + state->responseEnded_ = true; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end(acceptError); + return; + } - // Set Content-Encoding header if compression is enabled - if (enableGzip) { - res.set_header("Content-Encoding", "gzip"); - state->compressor_ = std::make_unique(); - log().debug("Set Content-Encoding: gzip header"); - } + const bool gzip = containsGzip(acceptEncoding); + if (gzip) { + state->enableGzip(); + } - // For efficiency, set up httplib to stream tile layer responses to client: - // (1) Lambda continuously supplies response data to httplib's DataSink, - // picking up data from state->buffer_ until all tile requests are done. - // Then, signal sink->done() to close the stream with a 200 status. - // Using chunked transfer encoding with optional manual compression. - // (2) Lambda acts as a cleanup routine, triggered by httplib upon request wrap-up. - // The success flag indicates if wrap-up was due to sink->done() or external factors - // like network errors or request aborts in lengthy tile requests (e.g., map-viewer). - res.set_chunked_content_provider( - state->responseType_, - [state](size_t offset, httplib::DataSink& sink) - { - std::unique_lock lock(state->mutex_); - - // Wait until there is data to be read. - std::string strBuf; - bool allDone = false; - state->resultEvent_.wait( - lock, - [&] - { - allDone = std::all_of( - state->requests_.begin(), - state->requests_.end(), - [](const auto& r) { return r->isDone(); }); - if (allDone && state->responseType_ == HttpTilesRequestState::binaryMimeType) - state->writer_->sendEndOfStream(); - strBuf = state->buffer_.str(); - return !strBuf.empty() || allDone; - }); + for (auto& request : state->requests_) { + request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); + request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); + request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; + } - if (!strBuf.empty()) { - // Compress data if gzip is enabled - if (state->compressor_) { - std::string compressed = state->compressor_->compress(strBuf.data(), strBuf.size()); - if (!compressed.empty()) { - log().debug("Compressing: {} bytes -> {} bytes (request {})", - strBuf.size(), compressed.size(), state->requestId_); - sink.write(compressed.data(), compressed.size()); - } - } else { - log().debug("Streaming {} bytes (no compression)...", strBuf.size()); - sink.write(strBuf.data(), strBuf.size()); - } - sink.os.flush(); - state->buffer_.str(""); // Clear buffer content - state->buffer_.clear(); // Clear error flags - // Force release of internal buffer memory - std::stringstream().swap(state->buffer_); + auto canProcess = self_.request(state->requests_, clientHeaders); + if (!canProcess) { + state->responseEnded_ = true; + std::vector> requestStatuses{}; + bool anyUnauthorized = false; + for (auto const& r : state->requests_) { + auto status = r->getStatus(); + requestStatuses.emplace_back(static_cast>(status)); + anyUnauthorized |= (status == RequestStatus::Unauthorized); } + res->writeStatus(anyUnauthorized ? "403 Forbidden" : "400 Bad Request"); + res->writeHeader("Content-Type", "application/json"); + res->end(nlohmann::json::object({{"status", requestStatuses}}).dump()); + return; + } - // Call sink.done() when all requests are done. - if (allDone) { - // Finish compression if enabled - if (state->compressor_) { - std::string finalChunk = state->compressor_->finish(); - log().debug( - "Final compression chunk is {} bytes.", - strBuf.size(), finalChunk.size(), state->requestId_); - if (!finalChunk.empty()) { - sink.write(finalChunk.data(), finalChunk.size()); - } - } - sink.done(); - } + if (j.contains("clientId")) { + abortRequestsForClientId(j["clientId"].get(), state); + } - return true; - }, - // Network error/timeout of request to datasource: - // cleanup callback to abort the requests. - [state, this](bool success) - { - if (!success) { - log().warn("Aborting tiles request {}", state->requestId_); - for (auto& request : state->requests_) { - self_.abort(request); - } - } - else { - log().info("Tiles request {} was successful.", state->requestId_); - // Determine response type and trim accordingly - ResponseType respType = (state->responseType_ == HttpTilesRequestState::binaryMimeType) - ? ResponseType::Binary - : ResponseType::Json; - tryMemoryTrim(respType); - } + if (gzip) { + res->writeHeader("Content-Encoding", "gzip"); + } + + res->writeHeader("Content-Type", state->responseType_); + res->onWritable([state](uintmax_t) { + state->drainOnLoop(); + return !state->responseEnded_.load(); }); + + state->scheduleDrain(); + }); } - void handleAbortRequest(const httplib::Request& req, httplib::Response& res) const + void handleAbortRequest(uWS::HttpResponse* res) const { - // Parse the JSON request. - nlohmann::json j = nlohmann::json::parse(req.body); - if (j.contains("clientId")) { - auto const clientId = j["clientId"].get(); - abortRequestsForClientId(clientId); - } - else { - res.status = 400; - res.set_content("Missing clientId", "text/plain"); - } + auto aborted = std::make_shared(false); + res->onAborted([aborted]() { *aborted = true; }); + + res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { + if (*aborted) + return; + body.append(chunk.data(), chunk.size()); + if (!last) + return; + + try { + auto j = nlohmann::json::parse(body); + if (j.contains("clientId")) { + abortRequestsForClientId(j["clientId"].get()); + if (*aborted) + return; + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "text/plain"); + res->end("OK"); + return; + } + + if (*aborted) + return; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Missing clientId"); + } + catch (const std::exception& e) { + if (*aborted) + return; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Invalid JSON: ") + e.what()); + } + }); } - void handleSourcesRequest(const httplib::Request& req, httplib::Response& res) const + void handleSourcesRequest(uWS::HttpResponse* res, uWS::HttpRequest* req) const { auto sourcesInfo = nlohmann::json::array(); - for (auto& source : self_.info(AuthHeaders{req.headers.begin(), req.headers.end()})) { + for (auto& source : self_.info(authHeadersFromRequest(req))) { sourcesInfo.push_back(source.toJson()); } - res.set_content(sourcesInfo.dump(), "application/json"); + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(sourcesInfo.dump()); } - void handleStatusRequest(const httplib::Request&, httplib::Response& res) const + void handleStatusRequest(uWS::HttpResponse* res) const { auto serviceStats = self_.getStatistics(); auto cacheStats = self_.cache()->getStatistics(); @@ -430,74 +536,93 @@ struct HttpService::Impl std::ostringstream oss; oss << ""; oss << "

Status Information

"; - - // Output serviceStats oss << "

Service Statistics

"; - oss << "
" << serviceStats.dump(4) << "
"; // Indentation of 4 for pretty printing - - // Output cacheStats + oss << "
" << serviceStats.dump(4) << "
"; oss << "

Cache Statistics

"; - oss << "
" << cacheStats.dump(4) << "
"; // Indentation of 4 for pretty printing - + oss << "
" << cacheStats.dump(4) << "
"; oss << ""; - res.set_content(oss.str(), "text/html"); + + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "text/html"); + res->end(oss.str()); } - void handleLocateRequest(const httplib::Request& req, httplib::Response& res) const + void handleLocateRequest(uWS::HttpResponse* res) const { - // Parse the JSON request. - nlohmann::json j = nlohmann::json::parse(req.body); - auto requestsJson = j["requests"]; - auto allResponsesJson = nlohmann::json::array(); - - for (auto const& locateReqJson : requestsJson) { - LocateRequest locateReq{locateReqJson}; - auto responsesJson = nlohmann::json::array(); - for (auto const& resp : self_.locate(locateReq)) - responsesJson.emplace_back(resp.serialize()); - allResponsesJson.emplace_back(responsesJson); - } + auto aborted = std::make_shared(false); + res->onAborted([aborted]() { *aborted = true; }); - res.set_content( - nlohmann::json::object({{"responses", allResponsesJson}}).dump(), - "application/json"); + res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { + if (*aborted) + return; + body.append(chunk.data(), chunk.size()); + if (!last) + return; + + try { + nlohmann::json j = nlohmann::json::parse(body); + auto requestsJson = j["requests"]; + auto allResponsesJson = nlohmann::json::array(); + + for (auto const& locateReqJson : requestsJson) { + LocateRequest locateReq{locateReqJson}; + auto responsesJson = nlohmann::json::array(); + for (auto const& resp : self_.locate(locateReq)) + responsesJson.emplace_back(resp.serialize()); + allResponsesJson.emplace_back(responsesJson); + } + + if (*aborted) + return; + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); + } + catch (const std::exception& e) { + if (*aborted) + return; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Invalid JSON: ") + e.what()); + } + }); } - static bool openConfigFile(std::ifstream& configFile, httplib::Response& res) + static bool openConfigFile(std::ifstream& configFile, uWS::HttpResponse* res) { auto configFilePath = DataSourceConfigService::get().getConfigFilePath(); if (!configFilePath.has_value()) { - res.status = 404; // Not found. - res.set_content( - "The config file path is not set. Check the server configuration.", - "text/plain"); + res->writeStatus("404 Not Found"); + res->writeHeader("Content-Type", "text/plain"); + res->end("The config file path is not set. Check the server configuration."); return false; } std::filesystem::path path = *configFilePath; - if (!configFilePath || !std::filesystem::exists(path)) { - res.status = 404; // Not found. - res.set_content("The server does not have a config file.", "text/plain"); + if (!std::filesystem::exists(path)) { + res->writeStatus("404 Not Found"); + res->writeHeader("Content-Type", "text/plain"); + res->end("The server does not have a config file."); return false; } configFile.open(*configFilePath); if (!configFile) { - res.status = 500; // Internal Server Error. - res.set_content("Failed to open config file.", "text/plain"); + res->writeStatus("500 Internal Server Error"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Failed to open config file."); return false; } return true; } - static void handleGetConfigRequest(const httplib::Request& req, httplib::Response& res) + static void handleGetConfigRequest(uWS::HttpResponse* res) { if (!isGetConfigEndpointEnabled()) { - res.status = 403; // Forbidden. - res.set_content( - "The GET /config endpoint is disabled by the server administrator.", - "text/plain"); + res->writeStatus("403 Forbidden"); + res->writeHeader("Content-Type", "text/plain"); + res->end("The GET /config endpoint is disabled by the server administrator."); return; } @@ -505,10 +630,10 @@ struct HttpService::Impl if (!openConfigFile(configFile, res)) { return; } + nlohmann::json jsonSchema = DataSourceConfigService::get().getDataSourceConfigSchema(); try { - // Load config YAML, expose the parts which clients may edit. YAML::Node configYaml = YAML::Load(configFile); nlohmann::json jsonConfig; std::unordered_map maskedSecretMap; @@ -522,147 +647,168 @@ struct HttpService::Impl combinedJson["model"] = jsonConfig; combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); - // Set the response - res.status = 200; // OK - res.set_content(combinedJson.dump(2), "application/json"); + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(combinedJson.dump(2)); } catch (const std::exception& e) { - res.status = 500; // Internal Server Error - res.set_content("Error processing config file: " + std::string(e.what()), "text/plain"); + res->writeStatus("500 Internal Server Error"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Error processing config file: ") + e.what()); } } - static void handlePostConfigRequest(const httplib::Request& req, httplib::Response& res) + void handlePostConfigRequest(uWS::HttpResponse* res) const { if (!isPostConfigEndpointEnabled()) { - res.status = 403; // Forbidden. - res.set_content( - "The POST /config endpoint is not enabled by the server administrator.", - "text/plain"); + res->writeStatus("403 Forbidden"); + res->writeHeader("Content-Type", "text/plain"); + res->end("The POST /config endpoint is not enabled by the server administrator."); return; } - std::mutex mtx; - std::condition_variable cv; - bool update_done = false; - - std::ifstream configFile; - if (!openConfigFile(configFile, res)) { - return; - } + struct ConfigUpdateState : std::enable_shared_from_this + { + uWS::HttpResponse* res = nullptr; + uWS::Loop* loop = nullptr; + std::atomic_bool aborted{false}; + std::atomic_bool done{false}; + std::atomic_bool wroteConfig{false}; + std::unique_ptr subscription; + std::string body; + }; + + auto state = std::make_shared(); + state->res = res; + state->loop = uWS::Loop::get(); + + res->onAborted([state]() { + state->aborted = true; + state->done = true; + state->subscription.reset(); + }); + + res->onData([state](std::string_view chunk, bool last) mutable { + if (state->aborted) + return; + state->body.append(chunk.data(), chunk.size()); + if (!last) + return; - // Subscribe to configuration changes. - auto subscription = DataSourceConfigService::get().subscribe( - [&](const std::vector& serviceConfigNodes) - { - std::lock_guard lock(mtx); - res.status = 200; - res.set_content("Configuration updated and applied successfully.", "text/plain"); - update_done = true; - cv.notify_one(); - }, - [&](const std::string& error) - { - std::lock_guard lock(mtx); - res.status = 500; - res.set_content("Error applying the configuration: " + error, "text/plain"); - update_done = true; - cv.notify_one(); - }); + std::ifstream configFile; + if (!Impl::openConfigFile(configFile, state->res)) { + state->done = true; + return; + } - // Parse the JSON from the request body. - nlohmann::json jsonConfig; - try { - jsonConfig = nlohmann::json::parse(req.body); - } - catch (const nlohmann::json::parse_error& e) { - res.status = 400; // Bad Request - res.set_content("Invalid JSON format: " + std::string(e.what()), "text/plain"); - return; - } + nlohmann::json jsonConfig; + try { + jsonConfig = nlohmann::json::parse(state->body); + } + catch (const nlohmann::json::parse_error& e) { + state->res->writeStatus("400 Bad Request"); + state->res->writeHeader("Content-Type", "text/plain"); + state->res->end(std::string("Invalid JSON format: ") + e.what()); + state->done = true; + return; + } - // Validate JSON against schema. - try { - DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); - } - catch (const std::exception& e) { - res.status = 500; // Internal Server Error. - res.set_content("Validation failed: " + std::string(e.what()), "text/plain"); - return; - } + try { + DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); + } + catch (const std::exception& e) { + state->res->writeStatus("500 Internal Server Error"); + state->res->writeHeader("Content-Type", "text/plain"); + state->res->end(std::string("Validation failed: ") + e.what()); + state->done = true; + return; + } - // Load the YAML, parse the secrets. - auto yamlConfig = YAML::Load(configFile); - std::unordered_map maskedSecrets; - yamlToJson(yamlConfig, true, &maskedSecrets); + auto yamlConfig = YAML::Load(configFile); + std::unordered_map maskedSecrets; + yamlToJson(yamlConfig, true, &maskedSecrets); - // Create YAML nodes from JSON nodes. - for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { - if (jsonConfig.contains(key)) - yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); - } + for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + if (jsonConfig.contains(key)) + yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); + } - // Write the YAML to configFilePath. - update_done = false; - configFile.close(); - log().trace("Writing new config."); - std::ofstream newConfigFile(*DataSourceConfigService::get().getConfigFilePath()); - newConfigFile << yamlConfig; - newConfigFile.close(); - - // Wait for the subscription callback. - std::unique_lock lk(mtx); - if (!cv.wait_for(lk, std::chrono::seconds(60), [&] { return update_done; })) { - res.status = 500; // Internal Server Error. - res.set_content("Timeout while waiting for config to update.", "text/plain"); - } + // Subscribe before writing; ignore any callbacks that happen before we write. + state->subscription = DataSourceConfigService::get().subscribe( + [state](std::vector const&) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true) || state->aborted) + return; + state->loop->defer([state]() mutable { + if (state->aborted) + return; + state->res->writeStatus("200 OK"); + state->res->writeHeader("Content-Type", "text/plain"); + state->res->end("Configuration updated and applied successfully."); + state->subscription.reset(); + }); + }, + [state](std::string const& error) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true) || state->aborted) + return; + state->loop->defer([state, error]() mutable { + if (state->aborted) + return; + state->res->writeStatus("500 Internal Server Error"); + state->res->writeHeader("Content-Type", "text/plain"); + state->res->end(std::string("Error applying the configuration: ") + error); + state->subscription.reset(); + }); + }); + + configFile.close(); + log().trace("Writing new config."); + state->wroteConfig = true; + std::ofstream newConfigFile(*DataSourceConfigService::get().getConfigFilePath()); + newConfigFile << yamlConfig; + newConfigFile.close(); + + // Timeout fail-safe (rare endpoint; ok to spawn a thread). + std::thread([weak = state->weak_from_this()]() { + std::this_thread::sleep_for(std::chrono::seconds(60)); + if (auto state = weak.lock()) { + if (state->done.exchange(true) || state->aborted) + return; + state->loop->defer([state]() mutable { + if (state->aborted) + return; + state->res->writeStatus("500 Internal Server Error"); + state->res->writeHeader("Content-Type", "text/plain"); + state->res->end("Timeout while waiting for config to update."); + state->subscription.reset(); + }); + } + }).detach(); + }); } }; HttpService::HttpService(Cache::Ptr cache, const HttpServiceConfig& config) - : Service(std::move(cache), config.watchConfig, config.defaultTtl), - impl_(std::make_unique(*this, config)) + : Service(std::move(cache), config.watchConfig, config.defaultTtl), impl_(std::make_unique(*this, config)) { } HttpService::~HttpService() = default; -void HttpService::setup(httplib::Server& server) +void HttpService::setup(uWS::App& app) { - server.Post( - "/tiles", - [&](const httplib::Request& req, httplib::Response& res) - { impl_->handleTilesRequest(req, res); }); - - server.Post( - "/abort", - [&](const httplib::Request& req, httplib::Response& res) - { impl_->handleAbortRequest(req, res); }); - - server.Get( - "/sources", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleSourcesRequest(req, res); }); - - server.Get( - "/status", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleStatusRequest(req, res); }); - - server.Post( - "/locate", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleLocateRequest(req, res); }); - - server.Get( - "/config", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleGetConfigRequest(req, res); }); - - server.Post( - "/config", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handlePostConfigRequest(req, res); }); + app.post("/tiles", [this](auto* res, auto* req) { impl_->handleTilesRequest(res, req); }); + app.post("/abort", [this](auto* res, auto* /*req*/) { impl_->handleAbortRequest(res); }); + app.get("/sources", [this](auto* res, auto* req) { impl_->handleSourcesRequest(res, req); }); + app.get("/status", [this](auto* res, auto* /*req*/) { impl_->handleStatusRequest(res); }); + app.post("/locate", [this](auto* res, auto* /*req*/) { impl_->handleLocateRequest(res); }); + app.get("/config", [](auto* res, auto* /*req*/) { Impl::handleGetConfigRequest(res); }); + app.post("/config", [this](auto* res, auto* /*req*/) { impl_->handlePostConfigRequest(res); }); } } // namespace mapget diff --git a/libs/service/src/datasource.cpp b/libs/service/src/datasource.cpp index 2e15d0a6..8074fcd4 100644 --- a/libs/service/src/datasource.cpp +++ b/libs/service/src/datasource.cpp @@ -1,4 +1,6 @@ #include "datasource.h" +#include +#include #include #include #include @@ -54,6 +56,7 @@ TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourc void DataSource::requireAuthHeaderRegexMatchOption(std::string header, std::regex re) { + std::ranges::transform(header, header.begin(), [](unsigned char c) { return (char)std::tolower(c); }); authHeaderAlternatives_.insert({std::move(header), std::move(re)}); } @@ -64,7 +67,9 @@ bool DataSource::isDataSourceAuthorized( return true; for (auto const& [k, v] : clientHeaders) { - auto authHeaderPatternIt = authHeaderAlternatives_.find(k); + auto key = k; + std::ranges::transform(key, key.begin(), [](unsigned char c) { return (char)std::tolower(c); }); + auto authHeaderPatternIt = authHeaderAlternatives_.find(key); if (authHeaderPatternIt != authHeaderAlternatives_.end()) { if (std::regex_match(v, authHeaderPatternIt->second)) { return true; From 5126872ee950350c67999a4b77ccc40039b19bfb Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Fri, 23 Jan 2026 17:29:34 +0100 Subject: [PATCH 06/14] Use Drogon instead of uWebSockets+httplib. --- CMakeLists.txt | 14 +- cmake/deps.cmake | 142 ++-- libs/http-datasource/CMakeLists.txt | 3 +- .../include/mapget/detail/http-server.h | 10 +- .../http-datasource/datasource-client.h | 15 +- .../http-datasource/datasource-server.h | 2 +- .../http-datasource/src/datasource-client.cpp | 77 +- .../http-datasource/src/datasource-server.cpp | 204 ++--- libs/http-datasource/src/http-server.cpp | 243 ++---- libs/http-service/CMakeLists.txt | 3 +- .../mapget/http-service/http-service.h | 2 +- libs/http-service/src/http-client.cpp | 137 +-- libs/http-service/src/http-service.cpp | 780 ++++++++++-------- libs/pymapget/CMakeLists.txt | 12 +- test/unit/CMakeLists.txt | 15 +- test/unit/test-datasource-server.cpp | 70 ++ test/unit/test-http-datasource.cpp | 548 ++++++------ test/unit/test-http-service-fixture.h | 17 + test/unit/test-main.cpp | 50 ++ 19 files changed, 1305 insertions(+), 1039 deletions(-) create mode 100644 test/unit/test-datasource-server.cpp create mode 100644 test/unit/test-http-service-fixture.h create mode 100644 test/unit/test-main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d73bb5b..f97065b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,10 @@ endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) +if (WIN32) + add_compile_definitions(NOMINMAX) +endif() + include(FetchContent) include(GNUInstallDirs) @@ -22,12 +26,21 @@ if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) set(MAPGET_WITH_HTTPLIB ON CACHE BOOL "Enable mapget-http-datasource and mapget-http-service libraries.") set(MAPGET_ENABLE_TESTING ON CACHE BOOL "Enable testing.") set(MAPGET_BUILD_EXAMPLES ON CACHE BOOL "Build examples.") + # Prevent CPM dependencies (e.g. Drogon/Trantor) from registering their own + # tests into this project's ctest run. + set(BUILD_TESTING OFF CACHE BOOL "Disable dependency tests" FORCE) endif() option(MAPGET_WITH_WHEEL "Enable mapget Python wheel (output to WHEEL_DEPLOY_DIRECTORY).") option(MAPGET_WITH_SERVICE "Enable mapget-service library. Requires threads.") option(MAPGET_WITH_HTTPLIB "Enable mapget-http-datasource and mapget-http-service libraries.") +if (MAPGET_ENABLE_TESTING) + # Enable testing before adding CPM dependencies so stale/third-party CTest + # files don't linger in the build tree. + enable_testing() +endif() + set(Python3_FIND_STRATEGY LOCATION) if (NOT MSVC) @@ -112,7 +125,6 @@ endif() # tests if (MAPGET_ENABLE_TESTING) - enable_testing() add_subdirectory(test/unit) if (MAPGET_WITH_WHEEL) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index a990b4a2..629fc978 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -25,6 +25,21 @@ CPMAddPackage( "BUILD_TESTING OFF") if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) + # OpenSSL's Configure script needs a "full" Perl distribution. Git for + # Windows ships a minimal perl that is missing required modules (e.g. + # Locale::Maketext::Simple), causing OpenSSL builds to fail. + if (WIN32) + if (NOT DEFINED PERL_EXECUTABLE OR PERL_EXECUTABLE MATCHES "[\\\\/]Git[\\\\/]usr[\\\\/]bin[\\\\/]perl\\.exe$") + find_program(_MAPGET_STRAWBERRY_PERL + NAMES perl.exe + PATHS "C:/Strawberry/perl/bin" + NO_DEFAULT_PATH) + if (_MAPGET_STRAWBERRY_PERL) + set(PERL_EXECUTABLE "${_MAPGET_STRAWBERRY_PERL}" CACHE FILEPATH "" FORCE) + endif() + endif() + endif() + set (OPENSSL_VERSION openssl-3.5.2) CPMAddPackage("gh:klebert-engineering/openssl-cmake@1.0.0") CPMAddPackage( @@ -40,19 +55,67 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) endif() CPMAddPackage( - URI "gh:yhirose/cpp-httplib@0.15.3" + NAME jsoncpp + GIT_REPOSITORY https://github.com/open-source-parsers/jsoncpp + GIT_TAG 1.9.5 + GIT_SHALLOW ON OPTIONS - "CPPHTTPLIB_USE_POLL ON" - "HTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN OFF" - "HTTPLIB_INSTALL OFF" - "HTTPLIB_USE_OPENSSL_IF_AVAILABLE OFF" - "HTTPLIB_USE_ZLIB_IF_AVAILABLE OFF") - # Manually enable openssl/zlib in httplib to avoid FindPackage calls. - target_compile_definitions(httplib INTERFACE - CPPHTTPLIB_OPENSSL_SUPPORT - CPPHTTPLIB_ZLIB_SUPPORT) - target_link_libraries(httplib INTERFACE - OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB) + "JSONCPP_WITH_TESTS OFF" + "JSONCPP_WITH_POST_BUILD_UNITTEST OFF" + "JSONCPP_WITH_PKGCONFIG_SUPPORT OFF" + "JSONCPP_WITH_CMAKE_PACKAGE OFF" + "BUILD_SHARED_LIBS OFF" + "BUILD_STATIC_LIBS ON" + "BUILD_OBJECT_LIBS OFF") + # Help Drogon's FindJsoncpp.cmake locate jsoncpp when built via CPM. + set(JSONCPP_INCLUDE_DIRS "${jsoncpp_SOURCE_DIR}/include" CACHE PATH "" FORCE) + set(JSONCPP_LIBRARIES jsoncpp_static CACHE STRING "" FORCE) + # CPM generates a dummy package redirect config at + # `${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/jsoncpp-config.cmake`. Drogon uses + # `find_package(Jsoncpp)` (config-first), so make that redirect actually + # define the expected `Jsoncpp_lib` target. + if (DEFINED CMAKE_FIND_PACKAGE_REDIRECTS_DIR) + file(MAKE_DIRECTORY "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}") + file(WRITE "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/jsoncpp-extra.cmake" [=[ +if(NOT TARGET Jsoncpp_lib) + add_library(Jsoncpp_lib INTERFACE) + target_include_directories(Jsoncpp_lib INTERFACE "${JSONCPP_INCLUDE_DIRS}") + target_link_libraries(Jsoncpp_lib INTERFACE ${JSONCPP_LIBRARIES}) +endif() +]=]) + endif() + + # Drogon defines install(EXPORT ...) rules unconditionally, which fail when + # used as a subproject with CPM-provided dependencies (zlib/jsoncpp/etc). + # Since mapget only needs Drogon for building, temporarily suppress install + # rule generation while configuring Drogon. + set(_MAPGET_PREV_SKIP_INSTALL_RULES "${CMAKE_SKIP_INSTALL_RULES}") + if (DEFINED BUILD_TESTING) + set(_MAPGET_PREV_BUILD_TESTING "${BUILD_TESTING}") + endif() + set(CMAKE_SKIP_INSTALL_RULES ON) + set(BUILD_TESTING OFF) + + CPMAddPackage( + URI "gh:drogonframework/drogon@1.9.7" + OPTIONS + "BUILD_CTL OFF" + "BUILD_EXAMPLES OFF" + "BUILD_ORM OFF" + "BUILD_BROTLI OFF" + "BUILD_YAML_CONFIG OFF" + "BUILD_SHARED_LIBS OFF" + "USE_SUBMODULE ON" + "USE_STATIC_LIBS_ONLY OFF" + "USE_POSTGRESQL OFF" + "USE_MYSQL OFF" + "USE_SQLITE3 OFF" + GIT_SUBMODULES "trantor") + + set(CMAKE_SKIP_INSTALL_RULES "${_MAPGET_PREV_SKIP_INSTALL_RULES}") + if (DEFINED _MAPGET_PREV_BUILD_TESTING) + set(BUILD_TESTING "${_MAPGET_PREV_BUILD_TESTING}") + endif() CPMAddPackage( URI "gh:jbeder/yaml-cpp#aa8d4e@0.8.0" # Use > 0.8.0 once available. @@ -65,59 +128,6 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) CPMAddPackage("gh:pboettch/json-schema-validator#2.3.0") CPMAddPackage("gh:okdshin/PicoSHA2@1.0.1") - if (WIN32) - CPMAddPackage( - NAME libuv - GIT_REPOSITORY https://github.com/libuv/libuv - GIT_TAG v1.48.0 - GIT_SHALLOW ON - OPTIONS - "LIBUV_BUILD_TESTS OFF" - "LIBUV_BUILD_BENCH OFF" - "LIBUV_BUILD_SHARED OFF" - "LIBUV_BUILD_EXAMPLES OFF") - endif() - - CPMAddPackage( - NAME uSockets - GIT_REPOSITORY https://github.com/uNetworking/uSockets - GIT_TAG v0.8.5 - GIT_SHALLOW ON - GIT_SUBMODULES "") - if (NOT TARGET uSockets) - file(GLOB_RECURSE U_SOCKETS_SOURCES CONFIGURE_DEPENDS - "${uSockets_SOURCE_DIR}/src/*.c" - "${uSockets_SOURCE_DIR}/src/*.cpp") - add_library(uSockets STATIC ${U_SOCKETS_SOURCES}) - target_include_directories(uSockets PUBLIC "${uSockets_SOURCE_DIR}/src") - target_compile_definitions(uSockets PRIVATE LIBUS_USE_OPENSSL) - target_link_libraries(uSockets PUBLIC OpenSSL::SSL OpenSSL::Crypto) - if (WIN32) - target_link_libraries(uSockets PUBLIC ws2_32) - if (TARGET uv_a) - target_link_libraries(uSockets PUBLIC uv_a) - elseif (TARGET uv) - target_link_libraries(uSockets PUBLIC uv) - else() - message(FATAL_ERROR "libuv was requested for uSockets on Windows, but no CMake target (uv_a/uv) was found.") - endif() - endif() - endif() - - CPMAddPackage( - NAME uWebSockets - GIT_REPOSITORY https://github.com/uNetworking/uWebSockets - GIT_TAG v20.37.0 - GIT_SHALLOW ON - GIT_SUBMODULES "") - if (NOT TARGET uWebSockets) - add_library(uWebSockets INTERFACE) - target_include_directories(uWebSockets INTERFACE "${uWebSockets_SOURCE_DIR}/src") - target_link_libraries(uWebSockets INTERFACE uSockets ZLIB::ZLIB) - if (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") - target_compile_options(uWebSockets INTERFACE -Wno-deprecated-declarations) - endif() - endif() endif () if (MAPGET_WITH_WHEEL AND NOT TARGET pybind11) @@ -130,7 +140,7 @@ if (MAPGET_WITH_SERVICE OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) endif() if (MAPGET_WITH_WHEEL AND NOT TARGET python-cmake-wheel) - CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel@1.1.0") + CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel@1.2.0") endif() if (MAPGET_ENABLE_TESTING) diff --git a/libs/http-datasource/CMakeLists.txt b/libs/http-datasource/CMakeLists.txt index 917eded2..7b323329 100644 --- a/libs/http-datasource/CMakeLists.txt +++ b/libs/http-datasource/CMakeLists.txt @@ -17,8 +17,7 @@ target_include_directories(mapget-http-datasource target_link_libraries(mapget-http-datasource PUBLIC - httplib::httplib - uWebSockets + drogon mapget-model mapget-service tiny-process-library) diff --git a/libs/http-datasource/include/mapget/detail/http-server.h b/libs/http-datasource/include/mapget/detail/http-server.h index bcc9656f..cc6db8cb 100644 --- a/libs/http-datasource/include/mapget/detail/http-server.h +++ b/libs/http-datasource/include/mapget/detail/http-server.h @@ -1,12 +1,12 @@ #pragma once +#include #include #include -// Forward declare uWebSockets app type to avoid including uWS headers in public headers. -namespace uWS { -template struct TemplatedApp; -using App = TemplatedApp; +// Forward declare Drogon app type to avoid including drogon headers in public headers. +namespace drogon { +class HttpAppFramework; } namespace mapget { @@ -77,7 +77,7 @@ class HttpServer * This function is called upon the first call to go(), * and allows any derived server class to add endpoints. */ - virtual void setup(uWS::App&) = 0; + virtual void setup(drogon::HttpAppFramework&) = 0; /** * Derived servers can use this to control whether diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h index 5bb03aee..91038d73 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h @@ -3,15 +3,24 @@ #include "mapget/model/sourcedatalayer.h" #include "mapget/model/featurelayer.h" #include "mapget/service/datasource.h" -#include "httplib.h" #include +#include #include +#include namespace TinyProcessLib { class Process; } +namespace drogon { +class HttpClient; +} + +namespace trantor { +class EventLoopThread; +} + namespace mapget { @@ -32,6 +41,7 @@ class RemoteDataSource : public DataSource * fails for any reason. */ RemoteDataSource(std::string const& host, uint16_t port); + ~RemoteDataSource(); // DataSource method overrides DataSourceInfo info() override; @@ -48,7 +58,8 @@ class RemoteDataSource : public DataSource std::string error_; // Multiple http clients allow parallel GET requests - std::vector httpClients_; + std::unique_ptr httpClientLoop_; + std::vector> httpClients_; std::atomic_uint64_t nextClient_{0}; }; diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h index 501f01ac..a0deda49 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h @@ -50,7 +50,7 @@ class DataSourceServer : public HttpServer DataSourceInfo const& info(); private: - void setup(uWS::App&) override; + void setup(drogon::HttpAppFramework&) override; struct Impl; std::unique_ptr impl_; diff --git a/libs/http-datasource/src/datasource-client.cpp b/libs/http-datasource/src/datasource-client.cpp index 1199c48c..128a5446 100644 --- a/libs/http-datasource/src/datasource-client.cpp +++ b/libs/http-datasource/src/datasource-client.cpp @@ -3,6 +3,10 @@ #include "process.hpp" #include "mapget/log.h" +#include +#include +#include + #include #include @@ -11,25 +15,43 @@ namespace mapget RemoteDataSource::RemoteDataSource(const std::string& host, uint16_t port) { + httpClientLoop_ = std::make_unique("MapgetRemoteDataSource"); + httpClientLoop_->run(); + + const auto hostString = fmt::format("http://{}:{}/", host, port); + // Fetch data source info. - httplib::Client client(host, port); - auto fetchedInfoJson = client.Get("/info"); - if (!fetchedInfoJson || fetchedInfoJson->status >= 300) - raise("Failed to fetch datasource info."); - info_ = DataSourceInfo::fromJson(nlohmann::json::parse(fetchedInfoJson->body)); + auto infoClient = drogon::HttpClient::newHttpClient(hostString, httpClientLoop_->getLoop()); + auto infoReq = drogon::HttpRequest::newHttpRequest(); + infoReq->setMethod(drogon::Get); + infoReq->setPath("/info"); + + auto [result, fetchedInfoResp] = infoClient->sendRequest(infoReq); + if (result != drogon::ReqResult::Ok || !fetchedInfoResp) { + raise(fmt::format("Failed to fetch datasource info: [{}]", drogon::to_string_view(result))); + } + if ((int)fetchedInfoResp->statusCode() >= 300) { + raise(fmt::format("Failed to fetch datasource info: [{}]", (int)fetchedInfoResp->statusCode())); + } + info_ = DataSourceInfo::fromJson(nlohmann::json::parse(std::string(fetchedInfoResp->body()))); if (info_.nodeId_.empty()) { // Unique node IDs are required for the string pool offsets. raise( fmt::format("Remote data source is missing node ID! Source info: {}", - fetchedInfoJson->body)); + std::string(fetchedInfoResp->body()))); } // Create as many clients as parallel requests are allowed. - for (auto i = 0; i < std::max(info_.maxParallelJobs_, 1); ++i) - httpClients_.emplace_back(host, port); + const auto clientCount = (std::max)(info_.maxParallelJobs_, 1); + httpClients_.reserve(clientCount); + for (auto i = 0; i < clientCount; ++i) { + httpClients_.emplace_back(drogon::HttpClient::newHttpClient(hostString, httpClientLoop_->getLoop())); + } } +RemoteDataSource::~RemoteDataSource() = default; + DataSourceInfo RemoteDataSource::info() { return info_; @@ -54,30 +76,26 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn auto& client = httpClients_[(nextClient_++) % httpClients_.size()]; // Send a GET tile request. - auto tileResponse = client.Get(fmt::format( + auto tileReq = drogon::HttpRequest::newHttpRequest(); + tileReq->setMethod(drogon::Get); + tileReq->setPath(fmt::format( "/tile?layer={}&tileId={}&stringPoolOffset={}", k.layerId_, k.tileId_.value_, cachedStringPoolOffset(info.nodeId_, cache))); + auto [resultCode, tileResponse] = client->sendRequest(tileReq); // Check that the response is OK. - if (!tileResponse || tileResponse->status >= 300) { + if (resultCode != drogon::ReqResult::Ok || !tileResponse || (int)tileResponse->statusCode() >= 300) { // Forward to base class get(). This will instantiate a // default TileLayer and call fill(). In our implementation // of fill, we set an error. - if (tileResponse) { - if (tileResponse->has_header("HTTPLIB_ERROR")) { - error_ = tileResponse->get_header_value("HTTPLIB_ERROR"); - } - else if (tileResponse->has_header("EXCEPTION_WHAT")) { - error_ = tileResponse->get_header_value("EXCEPTION_WHAT"); - } - else { - error_ = fmt::format("Code {}", tileResponse->status); - } - } - else { + if (resultCode != drogon::ReqResult::Ok) { + error_ = drogon::to_string(resultCode); + } else if (tileResponse) { + error_ = fmt::format("Code {}", (int)tileResponse->statusCode()); + } else { error_ = "No remote response."; } @@ -92,7 +110,7 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn [&](auto&& mapId, auto&& layerId) { return info.getLayer(std::string(layerId)); }, [&](auto&& tile) { result = tile; }, cache); - reader.read(tileResponse->body); + reader.read(std::string(tileResponse->body())); return result; } @@ -102,12 +120,15 @@ std::vector RemoteDataSource::locate(const LocateRequest& req) // Round-robin usage of http clients to facilitate parallel requests. auto& client = httpClients_[(nextClient_++) % httpClients_.size()]; - // Send a GET tile request. - auto locateResponse = client.Post( - fmt::format("/locate"), req.serialize().dump(), "application/json"); + auto locateReq = drogon::HttpRequest::newHttpRequest(); + locateReq->setMethod(drogon::Post); + locateReq->setPath("/locate"); + locateReq->setContentTypeCode(drogon::CT_APPLICATION_JSON); + locateReq->setBody(req.serialize().dump()); + auto [resultCode, locateResponse] = client->sendRequest(locateReq); // Check that the response is OK. - if (!locateResponse || locateResponse->status >= 300) { + if (resultCode != drogon::ReqResult::Ok || !locateResponse || (int)locateResponse->statusCode() >= 300) { // Forward to base class get(). This will instantiate a // default TileFeatureLayer and call fill(). In our implementation // of fill, we set an error. @@ -116,7 +137,7 @@ std::vector RemoteDataSource::locate(const LocateRequest& req) } // Check the response body for expected content. - auto responseJson = nlohmann::json::parse(locateResponse->body); + auto responseJson = nlohmann::json::parse(std::string(locateResponse->body())); if (responseJson.is_null()) { return {}; } diff --git a/libs/http-datasource/src/datasource-server.cpp b/libs/http-datasource/src/datasource-server.cpp index a8435058..30f83794 100644 --- a/libs/http-datasource/src/datasource-server.cpp +++ b/libs/http-datasource/src/datasource-server.cpp @@ -4,7 +4,8 @@ #include "mapget/model/info.h" #include "mapget/model/stream.h" -#include +#include +#include #include #include @@ -60,98 +61,107 @@ DataSourceServer& DataSourceServer::onLocateRequest( DataSourceInfo const& DataSourceServer::info() { return impl_->info_; } -void DataSourceServer::setup(uWS::App& app) +void DataSourceServer::setup(drogon::HttpAppFramework& app) { - app.get("/tile", [this](auto* res, auto* req) { - try { - auto layerIdParam = req->getQuery("layer"); - auto tileIdParam = req->getQuery("tileId"); - - if (layerIdParam.empty() || tileIdParam.empty()) { - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Missing query parameter: layer and/or tileId"); - return; - } + app.registerHandler( + "/tile", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) + { + try { + auto const& layerIdParam = req->getParameter("layer"); + auto const& tileIdParam = req->getParameter("tileId"); + + if (layerIdParam.empty() || tileIdParam.empty()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Missing query parameter: layer and/or tileId"); + callback(resp); + return; + } - auto layer = impl_->info_.getLayer(std::string(layerIdParam)); + auto layer = impl_->info_.getLayer(layerIdParam); + TileId tileId{std::stoull(tileIdParam)}; - TileId tileId{std::stoull(std::string(tileIdParam))}; + auto stringPoolOffsetParam = (simfil::StringId)0; + auto const& stringPoolOffsetStr = req->getParameter("stringPoolOffset"); + if (!stringPoolOffsetStr.empty()) { + stringPoolOffsetParam = (simfil::StringId)std::stoul(stringPoolOffsetStr); + } - auto stringPoolOffsetParam = (simfil::StringId)0; - auto stringPoolOffsetStr = req->getQuery("stringPoolOffset"); - if (!stringPoolOffsetStr.empty()) { - stringPoolOffsetParam = (simfil::StringId)std::stoul(std::string(stringPoolOffsetStr)); - } + std::string responseType = "binary"; + auto const& responseTypeStr = req->getParameter("responseType"); + if (!responseTypeStr.empty()) + responseType = responseTypeStr; + + auto tileLayer = [&]() -> std::shared_ptr + { + switch (layer->type_) { + case mapget::LayerType::Features: { + auto tileFeatureLayer = std::make_shared( + tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); + impl_->tileFeatureCallback_(tileFeatureLayer); + return tileFeatureLayer; + } + case mapget::LayerType::SourceData: { + auto tileSourceLayer = std::make_shared( + tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); + impl_->tileSourceDataCallback_(tileSourceLayer); + return tileSourceLayer; + } + default: + throw std::runtime_error(fmt::format("Unsupported layer type {}", (int)layer->type_)); + } + }(); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + + if (responseType == "binary") { + std::string content; + TileLayerStream::StringPoolOffsetMap stringPoolOffsets{{impl_->info_.nodeId_, stringPoolOffsetParam}}; + TileLayerStream::Writer layerWriter{ + [&](std::string bytes, TileLayerStream::MessageType) { content.append(bytes); }, + stringPoolOffsets}; + layerWriter.write(tileLayer); + + resp->setContentTypeString("application/binary"); + resp->setBody(std::move(content)); + } else { + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(tileLayer->toJson().dump()); + } - std::string responseType = "binary"; - auto responseTypeStr = req->getQuery("responseType"); - if (!responseTypeStr.empty()) { - responseType = std::string(responseTypeStr); + callback(resp); } - - auto tileLayer = [&]() -> std::shared_ptr { - switch (layer->type_) { - case mapget::LayerType::Features: { - auto tileFeatureLayer = std::make_shared( - tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); - impl_->tileFeatureCallback_(tileFeatureLayer); - return tileFeatureLayer; - } - case mapget::LayerType::SourceData: { - auto tileSourceLayer = std::make_shared( - tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); - impl_->tileSourceDataCallback_(tileSourceLayer); - return tileSourceLayer; - } - default: - throw std::runtime_error(fmt::format("Unsupported layer type {}", (int)layer->type_)); - } - }(); - - if (responseType == "binary") { - std::string content; - TileLayerStream::StringPoolOffsetMap stringPoolOffsets{ - {impl_->info_.nodeId_, stringPoolOffsetParam}}; - TileLayerStream::Writer layerWriter{ - [&](std::string bytes, TileLayerStream::MessageType) { content.append(bytes); }, - stringPoolOffsets}; - layerWriter.write(tileLayer); - - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/binary"); - res->end(content); - } else { - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(tileLayer->toJson().dump()); + catch (std::exception const& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error: ") + e.what()); + callback(resp); } - } - catch (std::exception const& e) { - res->writeStatus("500 Internal Server Error"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Error: ") + e.what()); - } - }); - - app.get("/info", [this](auto* res, auto* /*req*/) { - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(impl_->info_.toJson().dump()); - }); - - app.post("/locate", [this](auto* res, auto* /*req*/) { - auto aborted = std::make_shared(false); - res->onAborted([aborted]() { *aborted = true; }); - - res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { - if (*aborted) - return; - body.append(chunk.data(), chunk.size()); - if (!last) - return; + }, + {drogon::Get}); + + app.registerHandler( + "/info", + [this](const drogon::HttpRequestPtr&, std::function&& callback) + { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(impl_->info_.toJson().dump()); + callback(resp); + }, + {drogon::Get}); + + app.registerHandler( + "/locate", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) + { try { - LocateRequest parsedReq(nlohmann::json::parse(body)); + LocateRequest parsedReq(nlohmann::json::parse(std::string(req->body()))); auto responseJson = nlohmann::json::array(); if (impl_->locateCallback_) { @@ -160,21 +170,21 @@ void DataSourceServer::setup(uWS::App& app) } } - if (*aborted) - return; - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(responseJson.dump()); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(responseJson.dump()); + callback(resp); } catch (std::exception const& e) { - if (*aborted) - return; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Invalid request: ") + e.what()); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid request: ") + e.what()); + callback(resp); } - }); - }); + }, + {drogon::Post}); } } // namespace mapget diff --git a/libs/http-datasource/src/http-server.cpp b/libs/http-datasource/src/http-server.cpp index cdceab2c..f7106509 100644 --- a/libs/http-datasource/src/http-server.cpp +++ b/libs/http-datasource/src/http-server.cpp @@ -1,19 +1,17 @@ #include "mapget/detail/http-server.h" #include "mapget/log.h" -#include -#include +#include +#include #include #include +#include #include #include #include -#include #include #include -#include -#include #include #include #include @@ -24,20 +22,28 @@ namespace mapget { -// initialize the atomic activeHttpServer with nullptr +// Used by waitForSignal() so the signal handler knows what to stop. static std::atomic activeHttpServer = nullptr; +// Drogon uses a singleton app instance; running multiple independent servers +// in-process is not supported. +static std::atomic activeDrogonServer = nullptr; + namespace { + struct MountPoint { std::string urlPrefix; std::filesystem::path fsRoot; }; -[[nodiscard]] bool startsWith(std::string_view s, std::string_view prefix) +[[nodiscard]] bool looksLikeWindowsDrivePath(std::string_view s) { - return s.size() >= prefix.size() && s.substr(0, prefix.size()) == prefix; + if (s.size() < 3) + return false; + const unsigned char drive = static_cast(s[0]); + return std::isalpha(drive) && s[1] == ':' && (s[2] == '\\' || s[2] == '/'); } [[nodiscard]] std::string normalizeUrlPrefix(std::string prefix) @@ -51,75 +57,6 @@ struct MountPoint return prefix; } -[[nodiscard]] std::string_view guessMimeType(std::filesystem::path const& filePath) -{ - auto ext = filePath.extension().string(); - std::ranges::transform(ext, ext.begin(), [](unsigned char c) { return (char)std::tolower(c); }); - - if (ext == ".html" || ext == ".htm") - return "text/html"; - if (ext == ".css") - return "text/css"; - if (ext == ".js") - return "application/javascript"; - if (ext == ".json") - return "application/json"; - if (ext == ".svg") - return "image/svg+xml"; - if (ext == ".png") - return "image/png"; - if (ext == ".jpg" || ext == ".jpeg") - return "image/jpeg"; - if (ext == ".ico") - return "image/x-icon"; - if (ext == ".woff2") - return "font/woff2"; - if (ext == ".woff") - return "font/woff"; - if (ext == ".ttf") - return "font/ttf"; - if (ext == ".txt") - return "text/plain"; - - return "application/octet-stream"; -} - -[[nodiscard]] std::optional resolveStaticFile( - std::vector const& mounts, - std::string_view urlPath) -{ - if (mounts.empty()) - return std::nullopt; - if (!startsWith(urlPath, "/")) - return std::nullopt; - - // Longest-prefix match. - MountPoint const* best = nullptr; - for (auto const& m : mounts) { - if (startsWith(urlPath, m.urlPrefix) && (!best || m.urlPrefix.size() > best->urlPrefix.size())) - best = &m; - } - if (!best) - return std::nullopt; - - std::string_view remainder = urlPath.substr(best->urlPrefix.size()); - if (!remainder.empty() && remainder.front() == '/') - remainder.remove_prefix(1); - - std::filesystem::path relativePath = std::filesystem::path(std::string(remainder)).lexically_normal(); - if (relativePath.empty() || urlPath.back() == '/') - relativePath /= "index.html"; - - // Basic path traversal protection: reject any ".." segments. - for (auto const& part : relativePath) { - if (part == "..") - return std::nullopt; - } - - std::filesystem::path candidate = (best->fsRoot / relativePath).lexically_normal(); - return candidate; -} - } // namespace struct HttpServer::Impl @@ -134,20 +71,14 @@ struct HttpServer::Impl uint16_t port_ = 0; bool printPortToStdout_ = false; + bool startedOnce_ = false; std::mutex mountsMutex_; std::vector mounts_; - uWS::Loop* loop_ = nullptr; - us_listen_socket_t* listenSocket_ = nullptr; - static void handleSignal(int) { - // Temporarily holds the current active HttpServer auto* expected = activeHttpServer.load(); - - // Stop the active instance when a signal is received. - // We use compare_exchange_strong to make the operation atomic. if (activeHttpServer.compare_exchange_strong(expected, nullptr)) { if (expected) { expected->stop(); @@ -168,14 +99,21 @@ HttpServer::HttpServer() : impl_(new Impl()) {} HttpServer::~HttpServer() { - if (isRunning()) - stop(); + stop(); } void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t waitMs) { if (impl_->running_ || impl_->serverThread_.joinable()) raise("HttpServer is already running"); + if (impl_->startedOnce_) + raise("HttpServer cannot be restarted in-process (Drogon singleton)"); + + HttpServer* expected = nullptr; + if (!activeDrogonServer.compare_exchange_strong(expected, this)) + raise("Only one HttpServer can run per process (Drogon singleton)"); + + impl_->startedOnce_ = true; // Reset start state. { @@ -188,12 +126,9 @@ void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t wa [this, interfaceAddr, port] { try { - uWS::App app; + auto& app = drogon::app(); - // Allow derived class to set up the server - setup(app); - - // Copy mounts to avoid locking in the hot path. + // Copy mounts to avoid locking after the server thread starts. std::vector mountsCopy; { std::lock_guard lock(impl_->mountsMutex_); @@ -201,60 +136,36 @@ void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t wa } if (!mountsCopy.empty()) { - app.get( - "/*", - [mounts = std::move(mountsCopy)](auto* res, auto* req) mutable - { - auto urlPath = req->getUrl(); - auto candidate = resolveStaticFile(mounts, urlPath); - if (!candidate || !std::filesystem::exists(*candidate) || - !std::filesystem::is_regular_file(*candidate)) { - res->writeStatus("404 Not Found"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Not found"); - return; - } - - std::ifstream ifs(*candidate, std::ios::binary); - if (!ifs) { - res->writeStatus("500 Internal Server Error"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Failed to open file"); - return; - } - - std::string content; - ifs.seekg(0, std::ios::end); - content.resize(static_cast(ifs.tellg())); - ifs.seekg(0, std::ios::beg); - if (!content.empty()) { - ifs.read(content.data(), static_cast(content.size())); - } - - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", guessMimeType(*candidate)); - res->end(content); - }); + std::sort( + mountsCopy.begin(), + mountsCopy.end(), + [](MountPoint const& a, MountPoint const& b) { return a.urlPrefix.size() > b.urlPrefix.size(); }); + + // Using empty document root makes addALocation's "alias" parameter + // work with absolute Windows paths (e.g. "C:/path"). + app.setDocumentRoot(""); + + for (auto const& m : mountsCopy) { + app.addALocation(m.urlPrefix, "", m.fsRoot.generic_string()); + } } - app.listen( - interfaceAddr, - port, - [this, interfaceAddr, port](us_listen_socket_t* listenSocket) - { - if (!listenSocket) { - impl_->notifyStart( - fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); - return; - } + // Allow derived class to set up the server. + setup(app); - impl_->listenSocket_ = listenSocket; - impl_->loop_ = uWS::Loop::get(); + app.addListener(interfaceAddr, port); - // Determine actual port (port may be 0 for ephemeral). - impl_->port_ = static_cast( - us_socket_local_port(0, reinterpret_cast(listenSocket))); + app.registerBeginningAdvice([this]() { + // Beginning advice runs before listeners start. Post the actual + // startup notification to run after startListening() completed. + drogon::app().getLoop()->queueInLoop([this]() { + auto listeners = drogon::app().getListeners(); + if (listeners.empty()) { + impl_->notifyStart("HttpServer started without listeners"); + return; + } + impl_->port_ = listeners.front().toPort(); impl_->running_ = true; impl_->notifyStart(); @@ -263,14 +174,7 @@ void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t wa else log().info("====== Running on port {} ======", impl_->port_); }); - - // If listen failed, exit without running the loop. - if (!impl_->running_) { - if (!impl_->startNotified_) { - impl_->notifyStart(fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); - } - return; - } + }); app.run(); } @@ -279,8 +183,9 @@ void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t wa } impl_->running_ = false; - impl_->listenSocket_ = nullptr; - impl_->loop_ = nullptr; + + HttpServer* expected = this; + (void)activeDrogonServer.compare_exchange_strong(expected, nullptr); }); std::unique_lock lk(impl_->startMutex_); @@ -305,10 +210,8 @@ void HttpServer::stop() if (!impl_->serverThread_.joinable()) return; - if (impl_->loop_ && impl_->listenSocket_) { - auto* loop = impl_->loop_; - auto* listenSocket = impl_->listenSocket_; - loop->defer([listenSocket]() { us_listen_socket_close(0, listenSocket); }); + if (drogon::app().isRunning()) { + drogon::app().quit(); } if (impl_->serverThread_.get_id() != std::this_thread::get_id()) @@ -322,14 +225,11 @@ uint16_t HttpServer::port() const void HttpServer::waitForSignal() { - // So the signal handler knows what to call activeHttpServer = this; - // Set the signal handler for SIGINT and SIGTERM. std::signal(SIGINT, Impl::handleSignal); std::signal(SIGTERM, Impl::handleSignal); - // Wait for the signal handler to stop us, or the server to shut down on its own. while (isRunning()) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); } @@ -339,26 +239,29 @@ void HttpServer::waitForSignal() bool HttpServer::mountFileSystem(std::string const& pathFromTo) { - using namespace std::ranges; - auto parts = pathFromTo | views::split(':') | - views::transform([](auto&& s) { return std::string(&*s.begin(), distance(s)); }); - auto partsVec = std::vector(parts.begin(), parts.end()); - std::string urlPrefix; - std::filesystem::path fsRoot; - if (partsVec.size() == 1) { + std::string fsRootStr; + + const auto firstColon = pathFromTo.find(':'); + if (firstColon == std::string::npos || looksLikeWindowsDrivePath(pathFromTo)) { urlPrefix = "/"; - fsRoot = partsVec[0]; - } else if (partsVec.size() == 2) { - urlPrefix = partsVec[0]; - fsRoot = partsVec[1]; + fsRootStr = pathFromTo; } else { - return false; + urlPrefix = pathFromTo.substr(0, firstColon); + fsRootStr = pathFromTo.substr(firstColon + 1); + if (fsRootStr.empty()) + return false; } urlPrefix = normalizeUrlPrefix(std::move(urlPrefix)); - if (!std::filesystem::exists(fsRoot) || !std::filesystem::is_directory(fsRoot)) + std::filesystem::path fsRoot(fsRootStr); + std::error_code ec; + fsRoot = std::filesystem::absolute(fsRoot, ec); + if (ec) + return false; + + if (!std::filesystem::exists(fsRoot, ec) || ec || !std::filesystem::is_directory(fsRoot, ec) || ec) return false; std::lock_guard lock(impl_->mountsMutex_); diff --git a/libs/http-service/CMakeLists.txt b/libs/http-service/CMakeLists.txt index 23c2f847..b9e8799f 100644 --- a/libs/http-service/CMakeLists.txt +++ b/libs/http-service/CMakeLists.txt @@ -17,8 +17,7 @@ target_include_directories(mapget-http-service target_link_libraries(mapget-http-service PUBLIC - httplib::httplib - uWebSockets + drogon yaml-cpp CLI11::CLI11 nlohmann_json_schema_validator diff --git a/libs/http-service/include/mapget/http-service/http-service.h b/libs/http-service/include/mapget/http-service/http-service.h index 1e87225d..61ea4c8e 100644 --- a/libs/http-service/include/mapget/http-service/http-service.h +++ b/libs/http-service/include/mapget/http-service/http-service.h @@ -55,7 +55,7 @@ class HttpService : public HttpServer, public Service ~HttpService() override; protected: - void setup(uWS::App& app) override; + void setup(drogon::HttpAppFramework& app) override; private: struct Impl; diff --git a/libs/http-service/src/http-client.cpp b/libs/http-service/src/http-client.cpp index 95ebdbea..ea6bb04e 100644 --- a/libs/http-service/src/http-client.cpp +++ b/libs/http-service/src/http-client.cpp @@ -1,51 +1,72 @@ #include "http-client.h" -#include "httplib.h" + #include "mapget/log.h" +#include +#include +#include + +#include + +#include "fmt/format.h" + namespace mapget { +namespace +{ + +void applyHeaders(drogon::HttpRequestPtr const& req, AuthHeaders const& headers) +{ + for (auto const& [k, v] : headers) { + req->addHeader(k, v); + } +} + +} // namespace + struct HttpClient::Impl { - httplib::Client client_; + std::unique_ptr loopThread_; + drogon::HttpClientPtr client_; std::unordered_map sources_; std::shared_ptr stringPoolProvider_; - httplib::Headers headers_; + AuthHeaders headers_; - Impl(std::string const& host, uint16_t port, AuthHeaders headers, bool enableCompression) : - client_(host, port), - headers_() + Impl(std::string const& host, uint16_t port, AuthHeaders headers, bool enableCompression) : headers_(std::move(headers)) { - for (auto const& [k, v] : headers) { - headers_.emplace(k, v); - } - // Add Accept-Encoding header if compression is enabled and not already present - if (enableCompression) { - bool hasAcceptEncoding = false; - for (const auto& [key, value] : headers_) { - if (key == "Accept-Encoding") { - hasAcceptEncoding = true; - break; - } - } - if (!hasAcceptEncoding) { - headers_.emplace("Accept-Encoding", "gzip"); - } + if (enableCompression && !(headers_.contains("Accept-Encoding") || headers_.contains("accept-encoding"))) { + headers_.emplace("Accept-Encoding", "gzip"); } - + + loopThread_ = std::make_unique("MapgetHttpClient"); + loopThread_->run(); + + const auto hostString = fmt::format("http://{}:{}/", host, port); + client_ = drogon::HttpClient::newHttpClient(hostString, loopThread_->getLoop()); + stringPoolProvider_ = std::make_shared(); - client_.set_keep_alive(false); - auto sourcesJson = client_.Get("/sources", headers_); - if (!sourcesJson || sourcesJson->status != 200) - raise( - fmt::format("Failed to fetch sources: [{}]", sourcesJson->status)); - for (auto const& info : nlohmann::json::parse(sourcesJson->body)) { + + // Fetch data sources (/sources). + auto req = drogon::HttpRequest::newHttpRequest(); + req->setMethod(drogon::Get); + req->setPath("/sources"); + applyHeaders(req, headers_); + + auto [result, resp] = client_->sendRequest(req); + if (result != drogon::ReqResult::Ok || !resp) { + raise(fmt::format("Failed to fetch sources: [{}]", drogon::to_string_view(result))); + } + if (resp->statusCode() != drogon::k200OK) { + raise(fmt::format("Failed to fetch sources: [{}]", (int)resp->statusCode())); + } + + for (auto const& info : nlohmann::json::parse(std::string(resp->body()))) { auto parsedInfo = DataSourceInfo::fromJson(info); sources_.emplace(parsedInfo.mapId_, parsedInfo); } } - [[nodiscard]] std::shared_ptr - resolve(std::string_view const& map, std::string_view const& layer) const + [[nodiscard]] std::shared_ptr resolve(std::string_view const& map, std::string_view const& layer) const { auto mapIt = sources_.find(std::string(map)); if (mapIt == sources_.end()) @@ -54,8 +75,10 @@ struct HttpClient::Impl { } }; -HttpClient::HttpClient(const std::string& host, uint16_t port, AuthHeaders headers, bool enableCompression) : impl_( - std::make_unique(host, port, std::move(headers), enableCompression)) {} +HttpClient::HttpClient(const std::string& host, uint16_t port, AuthHeaders headers, bool enableCompression) + : impl_(std::make_unique(host, port, std::move(headers), enableCompression)) +{ +} HttpClient::~HttpClient() = default; @@ -76,43 +99,41 @@ LayerTilesRequest::Ptr HttpClient::request(const LayerTilesRequest::Ptr& request } auto reader = std::make_unique( - [this](auto&& mapId, auto&& layerId){return impl_->resolve(mapId, layerId);}, + [this](auto&& mapId, auto&& layerId) { return impl_->resolve(mapId, layerId); }, [request](auto&& result) { request->notifyResult(result); }, impl_->stringPoolProvider_); using namespace nlohmann; - // TODO: Currently, cpp-httplib client-POST does not support async responses. - // Those are only supported by GET. So, currently, this HttpClient - // does not profit from the streaming response. However, erdblick is - // is fully able to process async responses as it uses the browser fetch()-API. - auto tileResponse = impl_->client_.Post( - "/tiles", - impl_->headers_, - json::object({ - {"requests", json::array({request->toJson()})}, - {"stringPoolOffsets", reader->stringPoolCache()->stringPoolOffsets()} - }).dump(), - "application/json"); - - if (tileResponse) { - if (tileResponse->status == 200) { - reader->read(tileResponse->body); - } - else if (tileResponse->status == 400) { + auto body = json::object({ + {"requests", json::array({request->toJson()})}, + {"stringPoolOffsets", reader->stringPoolCache()->stringPoolOffsets()}, + }).dump(); + + auto httpReq = drogon::HttpRequest::newHttpRequest(); + httpReq->setMethod(drogon::Post); + httpReq->setPath("/tiles"); + httpReq->setContentTypeCode(drogon::CT_APPLICATION_JSON); + httpReq->setBody(std::move(body)); + applyHeaders(httpReq, impl_->headers_); + + auto [result, resp] = impl_->client_->sendRequest(httpReq); + if (result == drogon::ReqResult::Ok && resp) { + if (resp->statusCode() == drogon::k200OK) { + reader->read(std::string(resp->body())); + } else if (resp->statusCode() == drogon::k400BadRequest) { request->setStatus(RequestStatus::NoDataSource); - } - else if (tileResponse->status == 403) { + } else if (resp->statusCode() == drogon::k403Forbidden) { request->setStatus(RequestStatus::Unauthorized); + } else { + request->setStatus(RequestStatus::Aborted); } - // TODO if multiple LayerTileRequests are ever sent by this client, - // additionally handle RequestStatus::Aborted. - } - else { + } else { request->setStatus(RequestStatus::Aborted); } return request; } -} +} // namespace mapget + diff --git a/libs/http-service/src/http-service.cpp b/libs/http-service/src/http-service.cpp index 9f09586f..1a4249ea 100644 --- a/libs/http-service/src/http-service.cpp +++ b/libs/http-service/src/http-service.cpp @@ -4,7 +4,9 @@ #include "mapget/log.h" #include "mapget/service/config.h" -#include +#include +#include +#include #include #include @@ -90,11 +92,11 @@ class GzipCompressor z_stream strm_{}; }; -[[nodiscard]] AuthHeaders authHeadersFromRequest(uWS::HttpRequest* req) +[[nodiscard]] AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) { AuthHeaders headers; - for (auto const& [k, v] : *req) { - headers.emplace(std::string(k), std::string(v)); + for (auto const& [k, v] : req->headers()) { + headers.emplace(k, v); } return headers; } @@ -147,8 +149,7 @@ struct HttpService::Impl static constexpr auto jsonlMimeType = "application/jsonl"; static constexpr auto anyMimeType = "*/*"; - explicit TilesStreamState(Impl const& impl, uWS::HttpResponse* res, uWS::Loop* loop) - : impl_(impl), res_(res), loop_(loop) + explicit TilesStreamState(Impl const& impl, trantor::EventLoop* loop) : impl_(impl), loop_(loop) { static std::atomic_uint64_t nextRequestId; requestId_ = nextRequestId++; @@ -156,6 +157,20 @@ struct HttpService::Impl [this](auto&& msg, auto&& /*msgType*/) { appendOutgoingUnlocked(msg); }, stringOffsets_); } + void attachStream(drogon::ResponseStreamPtr stream) + { + { + std::lock_guard lock(mutex_); + if (aborted_ || responseEnded_) { + if (stream) + stream->close(); + return; + } + stream_ = std::move(stream); + } + scheduleDrain(); + } + void parseRequestFromJson(nlohmann::json const& requestJson) { std::string mapId = requestJson["mapId"]; @@ -200,6 +215,15 @@ struct HttpService::Impl impl_.self_.abort(req); } } + drogon::ResponseStreamPtr stream; + { + std::lock_guard lock(mutex_); + if (responseEnded_.exchange(true)) + return; + stream = std::move(stream_); + } + if (stream) + stream->close(); } void addResult(TileLayer::Ptr const& result) @@ -251,26 +275,32 @@ struct HttpService::Impl return; auto weak = weak_from_this(); - loop_->defer([weak = std::move(weak)]() mutable { + loop_->queueInLoop([weak = std::move(weak)]() mutable { if (auto self = weak.lock()) { self->drainOnLoop(); } }); } - void drainOnLoop() - { - drainScheduled_ = false; - if (aborted_ || responseEnded_) - return; - - constexpr size_t maxChunk = 64 * 1024; + void drainOnLoop() + { + drainScheduled_ = false; + if (aborted_ || responseEnded_) + return; + + constexpr size_t maxChunk = 64 * 1024; + + for (;;) { + std::string chunk; + bool done = false; + bool needAbort = false; + bool scheduleAgain = false; + drogon::ResponseStreamPtr streamToClose; + { + std::lock_guard lock(mutex_); + if (!stream_) + return; - for (;;) { - std::string chunk; - bool done = false; - { - std::lock_guard lock(mutex_); if (!pending_.empty()) { size_t n = std::min(pending_.size(), maxChunk); chunk.assign(pending_.data(), n); @@ -283,25 +313,36 @@ struct HttpService::Impl } done = allDone_; } - } - if (!chunk.empty()) { - bool ok = res_->write(chunk); - if (!ok) { - // Backpressure: resume in onWritable. - return; - } - continue; + if (!chunk.empty()) { + if (!stream_->send(chunk)) { + needAbort = true; + } else if (!pending_.empty() || allDone_) { + // Keep draining until we sent everything and closed the stream. + scheduleAgain = true; + } + } else if (done) { + responseEnded_ = true; + streamToClose = std::move(stream_); + } + } + + if (needAbort) { + onAborted(); + return; } - if (done) { - responseEnded_ = true; - res_->end(); - impl_.tryMemoryTrim(trimResponseType_); - } - return; - } - } + if (done) { + if (streamToClose) + streamToClose->close(); + impl_.tryMemoryTrim(trimResponseType_); + return; + } + if (scheduleAgain) + scheduleDrain(); + return; + } + } void appendOutgoingUnlocked(std::string_view bytes) { @@ -316,8 +357,7 @@ struct HttpService::Impl } Impl const& impl_; - uWS::HttpResponse* res_; - uWS::Loop* loop_; + trantor::EventLoop* loop_; std::mutex mutex_; uint64_t requestId_ = 0; @@ -326,6 +366,7 @@ struct HttpService::Impl ResponseType trimResponseType_ = ResponseType::Binary; std::string pending_; + drogon::ResponseStreamPtr stream_; std::unique_ptr writer_; std::vector requests_; TileLayerStream::StringPoolOffsetMap stringOffsets_; @@ -366,169 +407,154 @@ struct HttpService::Impl } } - void handleTilesRequest(uWS::HttpResponse* res, uWS::HttpRequest* req) const + void handleTilesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const { - auto* loop = uWS::Loop::get(); - auto state = std::make_shared(*this, res, loop); + auto state = std::make_shared(*this, drogon::app().getLoop()); - std::string accept = std::string(req->getHeader("accept")); - std::string acceptEncoding = std::string(req->getHeader("accept-encoding")); + const std::string accept = req->getHeader("accept"); + const std::string acceptEncoding = req->getHeader("accept-encoding"); auto clientHeaders = authHeadersFromRequest(req); - res->onAborted([state]() { state->onAborted(); }); - - res->onData([this, - res, - state, - clientHeaders = std::move(clientHeaders), - accept = std::move(accept), - acceptEncoding = std::move(acceptEncoding), - body = std::string()](std::string_view chunk, bool last) mutable { - if (state->aborted_ || state->responseEnded_) - return; - - body.append(chunk.data(), chunk.size()); - if (!last) - return; - - nlohmann::json j; - try { - j = nlohmann::json::parse(body); - } - catch (const std::exception& e) { - state->responseEnded_ = true; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Invalid JSON: ") + e.what()); - return; - } - - auto requestsIt = j.find("requests"); - if (requestsIt == j.end() || !requestsIt->is_array()) { - state->responseEnded_ = true; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Missing or invalid 'requests' array"); - return; - } - - log().info("Processing tiles request {}", state->requestId_); - for (auto& requestJson : *requestsIt) { - state->parseRequestFromJson(requestJson); - } + nlohmann::json j; + try { + j = nlohmann::json::parse(std::string(req->body())); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + return; + } - if (j.contains("stringPoolOffsets")) { - for (auto& item : j["stringPoolOffsets"].items()) { - state->stringOffsets_[item.key()] = item.value().get(); - } - } + auto requestsIt = j.find("requests"); + if (requestsIt == j.end() || !requestsIt->is_array()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Missing or invalid 'requests' array"); + callback(resp); + return; + } - std::string acceptError; - if (!state->setResponseTypeFromAccept(accept, acceptError)) { - state->responseEnded_ = true; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end(acceptError); - return; - } + log().info("Processing tiles request {}", state->requestId_); + for (auto& requestJson : *requestsIt) { + state->parseRequestFromJson(requestJson); + } - const bool gzip = containsGzip(acceptEncoding); - if (gzip) { - state->enableGzip(); + if (j.contains("stringPoolOffsets")) { + for (auto& item : j["stringPoolOffsets"].items()) { + state->stringOffsets_[item.key()] = item.value().get(); } + } - for (auto& request : state->requests_) { - request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); - request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); - request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; - } + std::string acceptError; + if (!state->setResponseTypeFromAccept(accept, acceptError)) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::move(acceptError)); + callback(resp); + return; + } - auto canProcess = self_.request(state->requests_, clientHeaders); - if (!canProcess) { - state->responseEnded_ = true; - std::vector> requestStatuses{}; - bool anyUnauthorized = false; - for (auto const& r : state->requests_) { - auto status = r->getStatus(); - requestStatuses.emplace_back(static_cast>(status)); - anyUnauthorized |= (status == RequestStatus::Unauthorized); - } - res->writeStatus(anyUnauthorized ? "403 Forbidden" : "400 Bad Request"); - res->writeHeader("Content-Type", "application/json"); - res->end(nlohmann::json::object({{"status", requestStatuses}}).dump()); - return; - } + const bool gzip = containsGzip(acceptEncoding); + if (gzip) { + state->enableGzip(); + } - if (j.contains("clientId")) { - abortRequestsForClientId(j["clientId"].get(), state); - } + for (auto& request : state->requests_) { + request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); + request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); + request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; + } - if (gzip) { - res->writeHeader("Content-Encoding", "gzip"); - } + const auto canProcess = self_.request(state->requests_, clientHeaders); + if (!canProcess) { + std::vector> requestStatuses{}; + bool anyUnauthorized = false; + for (auto const& r : state->requests_) { + auto status = r->getStatus(); + requestStatuses.emplace_back(static_cast>(status)); + anyUnauthorized |= (status == RequestStatus::Unauthorized); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(anyUnauthorized ? drogon::k403Forbidden : drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(nlohmann::json::object({{"status", requestStatuses}}).dump()); + callback(resp); + return; + } - res->writeHeader("Content-Type", state->responseType_); - res->onWritable([state](uintmax_t) { - state->drainOnLoop(); - return !state->responseEnded_.load(); - }); + if (j.contains("clientId")) { + abortRequestsForClientId(j["clientId"].get(), state); + } - state->scheduleDrain(); - }); + auto resp = drogon::HttpResponse::newAsyncStreamResponse( + [state](drogon::ResponseStreamPtr stream) { state->attachStream(std::move(stream)); }, + true); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeString(state->responseType_); + if (gzip) { + resp->addHeader("Content-Encoding", "gzip"); + } + callback(resp); } - void handleAbortRequest(uWS::HttpResponse* res) const + void handleAbortRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const { - auto aborted = std::make_shared(false); - res->onAborted([aborted]() { *aborted = true; }); - - res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { - if (*aborted) - return; - body.append(chunk.data(), chunk.size()); - if (!last) + try { + auto j = nlohmann::json::parse(std::string(req->body())); + if (j.contains("clientId")) { + abortRequestsForClientId(j["clientId"].get()); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("OK"); + callback(resp); return; - - try { - auto j = nlohmann::json::parse(body); - if (j.contains("clientId")) { - abortRequestsForClientId(j["clientId"].get()); - if (*aborted) - return; - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "text/plain"); - res->end("OK"); - return; - } - - if (*aborted) - return; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Missing clientId"); - } - catch (const std::exception& e) { - if (*aborted) - return; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Invalid JSON: ") + e.what()); } - }); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Missing clientId"); + callback(resp); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + } } - void handleSourcesRequest(uWS::HttpResponse* res, uWS::HttpRequest* req) const + void handleSourcesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const { auto sourcesInfo = nlohmann::json::array(); for (auto& source : self_.info(authHeadersFromRequest(req))) { sourcesInfo.push_back(source.toJson()); } - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(sourcesInfo.dump()); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(sourcesInfo.dump()); + callback(resp); } - void handleStatusRequest(uWS::HttpResponse* res) const + void handleStatusRequest( + const drogon::HttpRequestPtr&, + std::function&& callback) const { auto serviceStats = self_.getStatistics(); auto cacheStats = self_.cache()->getStatistics(); @@ -542,92 +568,93 @@ struct HttpService::Impl oss << "
" << cacheStats.dump(4) << "
"; oss << ""; - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "text/html"); - res->end(oss.str()); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_HTML); + resp->setBody(oss.str()); + callback(resp); } - void handleLocateRequest(uWS::HttpResponse* res) const + void handleLocateRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const { - auto aborted = std::make_shared(false); - res->onAborted([aborted]() { *aborted = true; }); - - res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { - if (*aborted) - return; - body.append(chunk.data(), chunk.size()); - if (!last) - return; - - try { - nlohmann::json j = nlohmann::json::parse(body); - auto requestsJson = j["requests"]; - auto allResponsesJson = nlohmann::json::array(); - - for (auto const& locateReqJson : requestsJson) { - LocateRequest locateReq{locateReqJson}; - auto responsesJson = nlohmann::json::array(); - for (auto const& resp : self_.locate(locateReq)) - responsesJson.emplace_back(resp.serialize()); - allResponsesJson.emplace_back(responsesJson); - } - - if (*aborted) - return; - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); - } - catch (const std::exception& e) { - if (*aborted) - return; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Invalid JSON: ") + e.what()); - } - }); + try { + nlohmann::json j = nlohmann::json::parse(std::string(req->body())); + auto requestsJson = j["requests"]; + auto allResponsesJson = nlohmann::json::array(); + + for (auto const& locateReqJson : requestsJson) { + LocateRequest locateReq{locateReqJson}; + auto responsesJson = nlohmann::json::array(); + for (auto const& resp : self_.locate(locateReq)) + responsesJson.emplace_back(resp.serialize()); + allResponsesJson.emplace_back(responsesJson); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); + callback(resp); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + } } - static bool openConfigFile(std::ifstream& configFile, uWS::HttpResponse* res) + static drogon::HttpResponsePtr openConfigFile(std::ifstream& configFile) { auto configFilePath = DataSourceConfigService::get().getConfigFilePath(); if (!configFilePath.has_value()) { - res->writeStatus("404 Not Found"); - res->writeHeader("Content-Type", "text/plain"); - res->end("The config file path is not set. Check the server configuration."); - return false; + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k404NotFound); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The config file path is not set. Check the server configuration."); + return resp; } std::filesystem::path path = *configFilePath; if (!std::filesystem::exists(path)) { - res->writeStatus("404 Not Found"); - res->writeHeader("Content-Type", "text/plain"); - res->end("The server does not have a config file."); - return false; + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k404NotFound); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The server does not have a config file."); + return resp; } configFile.open(*configFilePath); if (!configFile) { - res->writeStatus("500 Internal Server Error"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Failed to open config file."); - return false; + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Failed to open config file."); + return resp; } - return true; + return nullptr; } - static void handleGetConfigRequest(uWS::HttpResponse* res) + static void handleGetConfigRequest( + const drogon::HttpRequestPtr&, + std::function&& callback) { if (!isGetConfigEndpointEnabled()) { - res->writeStatus("403 Forbidden"); - res->writeHeader("Content-Type", "text/plain"); - res->end("The GET /config endpoint is disabled by the server administrator."); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k403Forbidden); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The GET /config endpoint is disabled by the server administrator."); + callback(resp); return; } std::ifstream configFile; - if (!openConfigFile(configFile, res)) { + if (auto errorResp = openConfigFile(configFile)) { + callback(errorResp); return; } @@ -647,149 +674,145 @@ struct HttpService::Impl combinedJson["model"] = jsonConfig; combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(combinedJson.dump(2)); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(combinedJson.dump(2)); + callback(resp); } catch (const std::exception& e) { - res->writeStatus("500 Internal Server Error"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Error processing config file: ") + e.what()); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error processing config file: ") + e.what()); + callback(resp); } } - void handlePostConfigRequest(uWS::HttpResponse* res) const + void handlePostConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const { if (!isPostConfigEndpointEnabled()) { - res->writeStatus("403 Forbidden"); - res->writeHeader("Content-Type", "text/plain"); - res->end("The POST /config endpoint is not enabled by the server administrator."); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k403Forbidden); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The POST /config endpoint is not enabled by the server administrator."); + callback(resp); return; } struct ConfigUpdateState : std::enable_shared_from_this { - uWS::HttpResponse* res = nullptr; - uWS::Loop* loop = nullptr; - std::atomic_bool aborted{false}; + trantor::EventLoop* loop = nullptr; std::atomic_bool done{false}; std::atomic_bool wroteConfig{false}; std::unique_ptr subscription; - std::string body; + std::function callback; }; - auto state = std::make_shared(); - state->res = res; - state->loop = uWS::Loop::get(); - - res->onAborted([state]() { - state->aborted = true; - state->done = true; - state->subscription.reset(); - }); - - res->onData([state](std::string_view chunk, bool last) mutable { - if (state->aborted) - return; - state->body.append(chunk.data(), chunk.size()); - if (!last) - return; + std::ifstream configFile; + if (auto errorResp = openConfigFile(configFile)) { + callback(errorResp); + return; + } - std::ifstream configFile; - if (!Impl::openConfigFile(configFile, state->res)) { - state->done = true; - return; - } + nlohmann::json jsonConfig; + try { + jsonConfig = nlohmann::json::parse(std::string(req->body())); + } + catch (const nlohmann::json::parse_error& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON format: ") + e.what()); + callback(resp); + return; + } - nlohmann::json jsonConfig; - try { - jsonConfig = nlohmann::json::parse(state->body); - } - catch (const nlohmann::json::parse_error& e) { - state->res->writeStatus("400 Bad Request"); - state->res->writeHeader("Content-Type", "text/plain"); - state->res->end(std::string("Invalid JSON format: ") + e.what()); - state->done = true; - return; - } + try { + DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Validation failed: ") + e.what()); + callback(resp); + return; + } - try { - DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); - } - catch (const std::exception& e) { - state->res->writeStatus("500 Internal Server Error"); - state->res->writeHeader("Content-Type", "text/plain"); - state->res->end(std::string("Validation failed: ") + e.what()); - state->done = true; - return; - } + auto yamlConfig = YAML::Load(configFile); + std::unordered_map maskedSecrets; + yamlToJson(yamlConfig, true, &maskedSecrets); - auto yamlConfig = YAML::Load(configFile); - std::unordered_map maskedSecrets; - yamlToJson(yamlConfig, true, &maskedSecrets); + for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + if (jsonConfig.contains(key)) + yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); + } - for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { - if (jsonConfig.contains(key)) - yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); - } + auto state = std::make_shared(); + state->loop = drogon::app().getLoop(); + state->callback = std::move(callback); - // Subscribe before writing; ignore any callbacks that happen before we write. - state->subscription = DataSourceConfigService::get().subscribe( - [state](std::vector const&) mutable { - if (!state->wroteConfig) { - return; - } - if (state->done.exchange(true) || state->aborted) - return; - state->loop->defer([state]() mutable { - if (state->aborted) - return; - state->res->writeStatus("200 OK"); - state->res->writeHeader("Content-Type", "text/plain"); - state->res->end("Configuration updated and applied successfully."); - state->subscription.reset(); - }); - }, - [state](std::string const& error) mutable { - if (!state->wroteConfig) { - return; - } - if (state->done.exchange(true) || state->aborted) - return; - state->loop->defer([state, error]() mutable { - if (state->aborted) - return; - state->res->writeStatus("500 Internal Server Error"); - state->res->writeHeader("Content-Type", "text/plain"); - state->res->end(std::string("Error applying the configuration: ") + error); - state->subscription.reset(); - }); + // Subscribe before writing; ignore any callbacks that happen before we write. + state->subscription = DataSourceConfigService::get().subscribe( + [state](std::vector const&) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Configuration updated and applied successfully."); + state->callback(resp); + state->subscription.reset(); }); + }, + [state](std::string const& error) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state, error]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error applying the configuration: ") + error); + state->callback(resp); + state->subscription.reset(); + }); + }); - configFile.close(); - log().trace("Writing new config."); - state->wroteConfig = true; - std::ofstream newConfigFile(*DataSourceConfigService::get().getConfigFilePath()); + configFile.close(); + log().trace("Writing new config."); + state->wroteConfig = true; + if (auto configFilePath = DataSourceConfigService::get().getConfigFilePath()) { + std::ofstream newConfigFile(*configFilePath); newConfigFile << yamlConfig; newConfigFile.close(); + } - // Timeout fail-safe (rare endpoint; ok to spawn a thread). - std::thread([weak = state->weak_from_this()]() { - std::this_thread::sleep_for(std::chrono::seconds(60)); - if (auto state = weak.lock()) { - if (state->done.exchange(true) || state->aborted) - return; - state->loop->defer([state]() mutable { - if (state->aborted) - return; - state->res->writeStatus("500 Internal Server Error"); - state->res->writeHeader("Content-Type", "text/plain"); - state->res->end("Timeout while waiting for config to update."); - state->subscription.reset(); - }); - } - }).detach(); - }); + // Timeout fail-safe (rare endpoint; ok to spawn a thread). + std::thread([weak = state->weak_from_this()]() { + std::this_thread::sleep_for(std::chrono::seconds(60)); + if (auto state = weak.lock()) { + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Timeout while waiting for config to update."); + state->callback(resp); + state->subscription.reset(); + }); + } + }).detach(); } }; @@ -800,15 +823,62 @@ HttpService::HttpService(Cache::Ptr cache, const HttpServiceConfig& config) HttpService::~HttpService() = default; -void HttpService::setup(uWS::App& app) +void HttpService::setup(drogon::HttpAppFramework& app) { - app.post("/tiles", [this](auto* res, auto* req) { impl_->handleTilesRequest(res, req); }); - app.post("/abort", [this](auto* res, auto* /*req*/) { impl_->handleAbortRequest(res); }); - app.get("/sources", [this](auto* res, auto* req) { impl_->handleSourcesRequest(res, req); }); - app.get("/status", [this](auto* res, auto* /*req*/) { impl_->handleStatusRequest(res); }); - app.post("/locate", [this](auto* res, auto* /*req*/) { impl_->handleLocateRequest(res); }); - app.get("/config", [](auto* res, auto* /*req*/) { Impl::handleGetConfigRequest(res); }); - app.post("/config", [this](auto* res, auto* /*req*/) { impl_->handlePostConfigRequest(res); }); + app.registerHandler( + "/tiles", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleTilesRequest(req, std::move(callback)); + }, + {drogon::Post}); + + app.registerHandler( + "/abort", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleAbortRequest(req, std::move(callback)); + }, + {drogon::Post}); + + app.registerHandler( + "/sources", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleSourcesRequest(req, std::move(callback)); + }, + {drogon::Get}); + + app.registerHandler( + "/status", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleStatusRequest(req, std::move(callback)); + }, + {drogon::Get}); + + app.registerHandler( + "/locate", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleLocateRequest(req, std::move(callback)); + }, + {drogon::Post}); + + app.registerHandler( + "/config", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + if (req->method() == drogon::Get) { + Impl::handleGetConfigRequest(req, std::move(callback)); + return; + } + if (req->method() == drogon::Post) { + impl_->handlePostConfigRequest(req, std::move(callback)); + return; + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k405MethodNotAllowed); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Method not allowed"); + callback(resp); + }, + {drogon::Get, drogon::Post}); } } // namespace mapget diff --git a/libs/pymapget/CMakeLists.txt b/libs/pymapget/CMakeLists.txt index 8d9a2016..fb7b8066 100644 --- a/libs/pymapget/CMakeLists.txt +++ b/libs/pymapget/CMakeLists.txt @@ -28,17 +28,7 @@ target_compile_features(pymapget INTERFACE cxx_std_17) -FetchContent_GetProperties(cpp-httplib) -if (MSVC AND CPP-HTTPLIB_POPULATED) - # Required because cpp-httplib speaks https via OpenSSL. - # Only needed if httplib came via FetchContent. - set(DEPLOY_FILES - "${OPENSSL_INCLUDE_DIR}/../libcrypto-1_1-x64.dll" - "${OPENSSL_INCLUDE_DIR}/../libssl-1_1-x64.dll" - "${CMAKE_CURRENT_LIST_DIR}/__main__.py") -else() - set(DEPLOY_FILES "${CMAKE_CURRENT_LIST_DIR}/__main__.py") -endif() +set(DEPLOY_FILES "${CMAKE_CURRENT_LIST_DIR}/__main__.py") add_wheel(pymapget NAME mapget diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 8a2deff4..af35f5cb 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -1,6 +1,7 @@ project(test.mapget.unit CXX) add_executable(test.mapget + test-main.cpp test-model.cpp test-model-geometry.cpp test-simfil-geometry.cpp @@ -16,6 +17,9 @@ add_executable(test.mapget add_executable(test.mapget.filelog test-file-logging.cpp) +add_executable(test.mapget.datasource-server + test-datasource-server.cpp) + target_link_libraries(test.mapget PUBLIC mapget-log @@ -23,7 +27,16 @@ target_link_libraries(test.mapget mapget-http-datasource mapget-http-service geojsonsource - Catch2::Catch2WithMain) + Catch2::Catch2) + +target_link_libraries(test.mapget.datasource-server + PUBLIC + mapget-log + mapget-http-datasource) + +target_compile_definitions(test.mapget + PRIVATE + MAPGET_TEST_DATASOURCE_SERVER_EXE=\"$\") target_link_libraries(test.mapget.filelog PUBLIC diff --git a/test/unit/test-datasource-server.cpp b/test/unit/test-datasource-server.cpp new file mode 100644 index 00000000..e5ebfc4f --- /dev/null +++ b/test/unit/test-datasource-server.cpp @@ -0,0 +1,70 @@ +#include "mapget/http-datasource/datasource-server.h" +#include "mapget/log.h" + +#include "nlohmann/json.hpp" + +using namespace mapget; +using namespace nlohmann; + +int main() +{ + setLogLevel("trace", log()); + + auto info = DataSourceInfo::fromJson(R"( + { + "nodeId": "test-datasource", + "mapId": "Tropico", + "layers": { + "WayLayer": { + "featureTypes": + [ + { + "name": "Way", + "uniqueIdCompositions": + [ + [ + { + "partId": "areaId", + "description": "String which identifies the map area.", + "datatype": "STR" + }, + { + "partId": "wayId", + "description": "Globally Unique 32b integer.", + "datatype": "U32" + } + ] + ] + } + ] + }, + "SourceData-WayLayer": { + "type": "SourceData" + } + } + } + )"_json); + + DataSourceServer ds(info); + ds.onTileFeatureRequest( + [&](const auto& tile) + { + auto f = tile->newFeature("Way", {{"areaId", "Area42"}, {"wayId", 0}}); + auto g = f->geom()->newGeometry(GeomType::Line); + g->append({42., 11}); + g->append({42., 12}); + }); + ds.onTileSourceDataRequest([&](const auto&) {}); + ds.onLocateRequest( + [&](LocateRequest const& request) -> std::vector + { + LocateResponse response(request); + response.tileKey_.layerId_ = "WayLayer"; + response.tileKey_.tileId_.value_ = 1; + return {response}; + }); + + ds.go("127.0.0.1", 0, 5000); + ds.waitForSignal(); + return 0; +} diff --git a/test/unit/test-http-datasource.cpp b/test/unit/test-http-datasource.cpp index 8f58c08b..a39f63c6 100644 --- a/test/unit/test-http-datasource.cpp +++ b/test/unit/test-http-datasource.cpp @@ -1,34 +1,167 @@ #include + +#include +#include #include +#include +#include +#include +#include +#include +#include #include -#include +#include + #ifndef _WIN32 -#include #include +#include #endif -#include "httplib.h" -#include "mapget/log.h" -#include "nlohmann/json.hpp" -#include "utility.h" +#include +#include +#include + +#include "process.hpp" + #include "mapget/http-datasource/datasource-client.h" -#include "mapget/http-datasource/datasource-server.h" +#include "mapget/http-service/cli.h" #include "mapget/http-service/http-client.h" #include "mapget/http-service/http-service.h" +#include "mapget/log.h" +#include "mapget/model/info.h" #include "mapget/model/stream.h" #include "mapget/service/config.h" -#include "mapget/http-service/cli.h" + +#include "nlohmann/json.hpp" + +#include "test-http-service-fixture.h" +#include "utility.h" using namespace mapget; namespace fs = std::filesystem; -TEST_CASE("HttpDataSource", "[HttpDataSource]") +namespace { - setLogLevel("trace", log()); - // Create DataSourceInfo. - auto info = DataSourceInfo::fromJson(R"( +class SyncHttpClient +{ +public: + SyncHttpClient(std::string host, uint16_t port) + { + loopThread_ = std::make_unique("MapgetTestHttpClient"); + loopThread_->run(); + + client_ = drogon::HttpClient::newHttpClient( + fmt::format("http://{}:{}/", host, port), + loopThread_->getLoop()); + } + + std::pair get(std::string path) + { + auto req = drogon::HttpRequest::newHttpRequest(); + req->setMethod(drogon::Get); + req->setPath(std::move(path)); + return client_->sendRequest(req); + } + + std::pair postJson(std::string path, std::string body) + { + auto req = drogon::HttpRequest::newHttpRequest(); + req->setMethod(drogon::Post); + req->setPath(std::move(path)); + req->setContentTypeCode(drogon::CT_APPLICATION_JSON); + req->setBody(std::move(body)); + return client_->sendRequest(req); + } + +private: + std::unique_ptr loopThread_; + drogon::HttpClientPtr client_; +}; + +class ChildProcessWithPort +{ +public: + explicit ChildProcessWithPort(std::string exePath) + { + auto stderrCallback = [](const char* bytes, size_t n) { + auto output = std::string(bytes, n); + output.erase(output.find_last_not_of(" \n\r\t") + 1); + if (!output.empty()) + std::cerr << output << std::endl; + }; + + auto stdoutCallback = [this](const char* bytes, size_t n) { + std::lock_guard lock(mutex_); + stdoutBuffer_.append(bytes, n); + + for (;;) { + auto nl = stdoutBuffer_.find_first_of("\r\n"); + if (nl == std::string::npos) + break; + + auto line = stdoutBuffer_.substr(0, nl); + stdoutBuffer_.erase(0, nl + 1); + line.erase(line.find_last_not_of(" \n\r\t") + 1); + + if (!portReady_) { + std::regex portRegex(R"(Running on port (\d+))"); + std::smatch matches; + if (std::regex_search(line, matches, portRegex) && matches.size() > 1) { + port_ = static_cast(std::stoi(matches.str(1))); + portReady_ = true; + cv_.notify_all(); + } + } + } + }; + + process_ = std::make_unique( + fmt::format("\"{}\"", exePath), + "", + stdoutCallback, + stderrCallback, + true); + + std::unique_lock lock(mutex_); +#if defined(NDEBUG) + if (!cv_.wait_for(lock, std::chrono::seconds(10), [this] { return portReady_; })) { + raise("Timeout waiting for the child process to start listening."); + } +#else + log().warn("Using Debug build: will wait forever!"); + cv_.wait(lock, [this] { return portReady_; }); +#endif + } + + ~ChildProcessWithPort() { + if (process_) { + process_->kill(true); + process_->get_exit_status(); + } + } + + [[nodiscard]] uint16_t port() const + { + return port_; + } + +private: + std::unique_ptr process_; + mutable std::mutex mutex_; + std::condition_variable cv_; + std::string stdoutBuffer_; + uint16_t port_ = 0; + bool portReady_ = false; +}; + +nlohmann::json testDataSourceInfoJson() +{ + using nlohmann::json; + return json::parse(R"( + { + "nodeId": "test-datasource", "mapId": "Tropico", "layers": { "WayLayer": { @@ -59,69 +192,41 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") } } } - )"_json); - - // Initialize a DataSource. - DataSourceServer ds(info); - std::atomic_uint32_t dataSourceFeatureRequestCount = 0; - std::atomic_uint32_t dataSourceSourceDataRequestCount = 0; - ds.onTileFeatureRequest( - [&](const auto& tile) - { - auto f = tile->newFeature("Way", {{"areaId", "Area42"}, {"wayId", 0}}); - auto g = f->geom()->newGeometry(GeomType::Line); - g->append({42., 11}); - g->append({42., 12}); - ++dataSourceFeatureRequestCount; - }); - ds.onTileSourceDataRequest( - [&](const auto& tile) { - ++dataSourceSourceDataRequestCount; - }); - ds.onLocateRequest( - [&](LocateRequest const& request) -> std::vector - { - REQUIRE(request.mapId_ == "Tropico"); - REQUIRE(request.typeId_ == "Way"); - REQUIRE(request.featureId_ == KeyValuePairs{{"wayId", 0}}); + )"); +} - LocateResponse response(request); - response.tileKey_.layerId_ = "WayLayer"; - response.tileKey_.tileId_.value_ = 1; - return {response}; - }); +} // namespace - // Launch the DataSource on a separate thread. - ds.go(); +TEST_CASE("HttpDataSource", "[HttpDataSource]") +{ + setLogLevel("trace", log()); + + // Start datasource server in a separate process (Drogon is singleton). + ChildProcessWithPort dsProc(MAPGET_TEST_DATASOURCE_SERVER_EXE); + + // Expected datasource info. + auto info = DataSourceInfo::fromJson(testDataSourceInfoJson()); - // Ensure the DataSource is running. - REQUIRE(ds.isRunning() == true); + SyncHttpClient dsClient("127.0.0.1", dsProc.port()); - SECTION("Fetch /info") + // Fetch /info { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); + auto [result, resp] = dsClient.get("/info"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); - // Send a GET info request. - auto fetchedInfoJson = cli.Get("/info"); - auto fetchedInfo = - DataSourceInfo::fromJson(nlohmann::json::parse(fetchedInfoJson->body)); + auto fetchedInfo = DataSourceInfo::fromJson(nlohmann::json::parse(std::string(resp->body()))); REQUIRE(fetchedInfo.toJson() == info.toJson()); } - SECTION("Fetch /tile") + // Fetch /tile { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); + auto [result, resp] = dsClient.get("/tile?layer=WayLayer&tileId=1"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); - // Send a GET tile request. - auto tileResponse = cli.Get("/tile?layer=WayLayer&tileId=1"); - - // Check that the response is OK. - REQUIRE(tileResponse != nullptr); - REQUIRE(tileResponse->status == 200); - - // Check the response body for expected content. auto receivedTileCount = 0; TileLayerStream::Reader reader( [&](auto&& mapId, auto&& layerId) @@ -133,24 +238,18 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(tile->id().layer_ == LayerType::Features); receivedTileCount++; }); - reader.read(tileResponse->body); + reader.read(std::string(resp->body())); REQUIRE(receivedTileCount == 1); } - SECTION("Fetch /tile SourceData") + // Fetch /tile SourceData { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); - - // Send a GET tile request - auto tileResponse = cli.Get("/tile?layer=SourceData-WayLayer&tileId=1"); + auto [result, resp] = dsClient.get("/tile?layer=SourceData-WayLayer&tileId=1"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); - // Check that the response is OK. - REQUIRE(tileResponse != nullptr); - REQUIRE(tileResponse->status == 200); - - // Check the response body for expected content. auto receivedTileCount = 0; TileLayerStream::Reader reader( [&](auto&& mapId, auto&& layerId) @@ -162,57 +261,49 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(tile->id().layer_ == LayerType::SourceData); receivedTileCount++; }); - reader.read(tileResponse->body); + reader.read(std::string(resp->body())); REQUIRE(receivedTileCount == 1); } - SECTION("Fetch /locate") + // Fetch /locate { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); - - // Send a POST locate request. - auto response = cli.Post("/locate", R"({ - "mapId": "Tropico", - "typeId": "Way", - "featureId": ["wayId", 0] - })", "application/json"); - - // Check that the response is OK. - REQUIRE(response != nullptr); - REQUIRE(response->status == 200); - - // Check the response body for expected content. - LocateResponse responseParsed(nlohmann::json::parse(response->body)[0]); + auto [result, resp] = dsClient.postJson( + "/locate", + R"({ + "mapId": "Tropico", + "typeId": "Way", + "featureId": ["wayId", 0] + })"); + + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); + + LocateResponse responseParsed(nlohmann::json::parse(std::string(resp->body()))[0]); REQUIRE(responseParsed.tileKey_.mapId_ == "Tropico"); REQUIRE(responseParsed.tileKey_.layer_ == LayerType::Features); REQUIRE(responseParsed.tileKey_.layerId_ == "WayLayer"); REQUIRE(responseParsed.tileKey_.tileId_.value_ == 1); } - SECTION("Query mapget HTTP service") + // Query mapget HTTP service (in-process, started once for entire test binary) { + auto& service = test::httpService(); + auto remoteDataSource = std::make_shared("127.0.0.1", dsProc.port()); + service.add(remoteDataSource); + auto countReceivedTiles = [](auto& client, auto mapId, auto layerId, auto tiles) { auto tileCount = 0; - auto request = std::make_shared(mapId, layerId, tiles); - request->onFeatureLayer([&](auto&& tile) { tileCount++; }); - //request->onSourceDataLayer([&](auto&& tile) { tileCount++; }); - + request->onFeatureLayer([&](auto&&) { tileCount++; }); client.request(request)->wait(); return std::make_tuple(request, tileCount); }; - HttpService service; - auto remoteDataSource = std::make_shared("localhost", ds.port()); - service.add(remoteDataSource); - - service.go(); - - SECTION("Query through mapget HTTP service") + // Query through mapget HTTP service { - HttpClient client("localhost", service.port()); + HttpClient client("127.0.0.1", service.port()); auto [request, receivedTileCount] = countReceivedTiles( client, @@ -221,54 +312,47 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") std::vector{{1234, 5678, 9112, 1234}}); REQUIRE(receivedTileCount == 4); - // One tile requested twice, so the cache was used. - REQUIRE(dataSourceFeatureRequestCount == 3); + REQUIRE(request->getStatus() == RequestStatus::Success); } - SECTION("Trigger 400 responses") + // Trigger 400 responses { - HttpClient client("localhost", service.port()); + HttpClient client("127.0.0.1", service.port()); { - auto [request, receivedTileCount] = countReceivedTiles( - client, - "UnknownMap", - "WayLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(client, "UnknownMap", "WayLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::NoDataSource); REQUIRE(receivedTileCount == 0); } { - auto [request, receivedTileCount] = countReceivedTiles( - client, - "Tropico", - "UnknownLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(client, "Tropico", "UnknownLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::NoDataSource); REQUIRE(receivedTileCount == 0); } } - SECTION("Run /locate through service") + // Run /locate through service { - httplib::Client client("localhost", service.port()); - - // Send a POST locate request. - auto response = client.Post("/locate", R"({ - "requests": [{ - "mapId": "Tropico", - "typeId": "Way", - "featureId": ["wayId", 0] - }] - })", "application/json"); - - // Check that the response is OK. - REQUIRE(response != nullptr); - REQUIRE(response->status == 200); - - // Check the response body for expected content. - auto responseJsonLists = nlohmann::json::parse(response->body)["responses"]; + SyncHttpClient serviceClient("127.0.0.1", service.port()); + + auto [result, resp] = serviceClient.postJson( + "/locate", + R"({ + "requests": [{ + "mapId": "Tropico", + "typeId": "Way", + "featureId": ["wayId", 0] + }] + })"); + + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); + + auto responseJsonLists = nlohmann::json::parse(std::string(resp->body()))["responses"]; REQUIRE(responseJsonLists.size() == 1); auto responseJsonList = responseJsonLists[0]; REQUIRE(responseJsonList.size() == 1); @@ -279,79 +363,52 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(responseParsed.tileKey_.tileId_.value_ == 1); } - SECTION("Test auth header requirement") + // Test auth header requirement { - remoteDataSource->requireAuthHeaderRegexMatchOption( - "X-USER-ROLE", - std::regex("\\bTropico-Viewer\\b")); + remoteDataSource->requireAuthHeaderRegexMatchOption("X-USER-ROLE", std::regex("\\bTropico-Viewer\\b")); - HttpClient badClient("localhost", service.port()); - HttpClient goodClient("localhost", service.port(), {{"X-USER-ROLE", "Tropico-Viewer"}}); + HttpClient badClient("127.0.0.1", service.port()); + HttpClient goodClient("127.0.0.1", service.port(), {{"X-USER-ROLE", "Tropico-Viewer"}}); - // Check sources REQUIRE(badClient.sources().empty()); REQUIRE(goodClient.sources().size() == 1); - // Try to load tiles with bad client { - auto [request, receivedTileCount] = countReceivedTiles( - badClient, - "Tropico", - "WayLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(badClient, "Tropico", "WayLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::Unauthorized); REQUIRE(receivedTileCount == 0); } - // Try to load tiles with good client { - auto [request, receivedTileCount] = countReceivedTiles( - goodClient, - "Tropico", - "WayLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(goodClient, "Tropico", "WayLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::Success); REQUIRE(receivedTileCount == 1); } } - service.stop(); - REQUIRE(service.isRunning() == false); - } - - SECTION("Wait for data source") - { - auto waitThread = std::thread([&] { ds.waitForSignal(); }); - ds.stop(); - waitThread.join(); - REQUIRE(ds.isRunning() == false); + service.remove(remoteDataSource); } - - ds.stop(); - REQUIRE(ds.isRunning() == false); } TEST_CASE("Configuration Endpoint Tests", "[Configuration]") { + auto& service = test::httpService(); + REQUIRE(service.isRunning() == true); + + SyncHttpClient cli("127.0.0.1", service.port()); + auto tempDir = fs::temp_directory_path() / test::generateTimestampedDirectoryName("mapget_test_http_config"); fs::create_directory(tempDir); auto tempConfigPath = tempDir / "temp_config.yaml"; - // Setting up the server and client. - HttpService service; - service.go(); - REQUIRE(service.isRunning() == true); - httplib::Client cli("localhost", service.port()); - // Set up the config file. DataSourceConfigService::get().reset(); struct SchemaPatchGuard { - ~SchemaPatchGuard() { - DataSourceConfigService::get().setDataSourceConfigSchemaPatch(nlohmann::json::object()); - } + ~SchemaPatchGuard() { DataSourceConfigService::get().setDataSourceConfigSchemaPatch(nlohmann::json::object()); } } schemaPatchGuard; - // Emulate the CLI-provided config-schema patch so http-settings participates in the auto schema. auto schemaPatch = nlohmann::json::parse(R"( { "properties": { @@ -364,110 +421,127 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") )"); DataSourceConfigService::get().setDataSourceConfigSchemaPatch(schemaPatch); - SECTION("Get Configuration - Config File Not Found") { + SECTION("Get Configuration - Config File Not Found") + { DataSourceConfigService::get().loadConfig(tempConfigPath.string()); - auto res = cli.Get("/config"); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 404); - REQUIRE(res->body == "The server does not have a config file."); + REQUIRE(res->statusCode() == drogon::k404NotFound); + REQUIRE(std::string(res->body()) == "The server does not have a config file."); } // Create config file for tests that need it { std::ofstream configFile(tempConfigPath); - configFile << "sources: []\nhttp-settings: [{'password': 'hunter2'}]"; // Update http-settings to an array. + configFile << "sources: []\nhttp-settings: [{'password': 'hunter2'}]"; configFile.flush(); configFile.close(); - - // Ensure file is synced to disk - #ifndef _WIN32 + +#ifndef _WIN32 int fd = open(tempConfigPath.c_str(), O_RDONLY); if (fd != -1) { fsync(fd); close(fd); } - #endif +#endif std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - - // Load the config after file is created + DataSourceConfigService::get().loadConfig(tempConfigPath.string()); - - // Give the config watcher time to detect the file std::this_thread::sleep_for(std::chrono::milliseconds(500)); - SECTION("Get Configuration - Not allowed") { + SECTION("Get Configuration - Not allowed") + { setGetConfigEndpointEnabled(false); - auto res = cli.Get("/config"); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 403); + REQUIRE(res->statusCode() == drogon::k403Forbidden); } - SECTION("Get Configuration - No Config File Path Set") { + SECTION("Get Configuration - No Config File Path Set") + { setGetConfigEndpointEnabled(true); - DataSourceConfigService::get().loadConfig(""); // Simulate no config path set. - auto res = cli.Get("/config"); + DataSourceConfigService::get().loadConfig(""); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 404); - REQUIRE(res->body == "The config file path is not set. Check the server configuration."); + REQUIRE(res->statusCode() == drogon::k404NotFound); + REQUIRE(std::string(res->body()) == + "The config file path is not set. Check the server configuration."); } - SECTION("Get Configuration - Success") { - auto res = cli.Get("/config"); + SECTION("Get Configuration - Success") + { + setGetConfigEndpointEnabled(true); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 200); - REQUIRE(res->body.find("sources") != std::string::npos); - REQUIRE(res->body.find("http-settings") != std::string::npos); - - // Ensure that the password is masked as SHA256. - REQUIRE(res->body.find("hunter2") == std::string::npos); - REQUIRE(res->body.find("MASKED:0:f52fbd32b2b3b86ff88ef6c490628285f482af15ddcb29541f94bcf526a3f6c7") != std::string::npos); + REQUIRE(res->statusCode() == drogon::k200OK); + + auto body = std::string(res->body()); + REQUIRE(body.find("sources") != std::string::npos); + REQUIRE(body.find("http-settings") != std::string::npos); + REQUIRE(body.find("hunter2") == std::string::npos); + REQUIRE( + body.find("MASKED:0:f52fbd32b2b3b86ff88ef6c490628285f482af15ddcb29541f94bcf526a3f6c7") != + std::string::npos); } - SECTION("Post Configuration - Not Enabled") { + SECTION("Post Configuration - Not Enabled") + { setPostConfigEndpointEnabled(false); - auto res = cli.Post("/config", "", "application/json"); + auto [result, res] = cli.postJson("/config", ""); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 403); + REQUIRE(res->statusCode() == drogon::k403Forbidden); } - SECTION("Post Configuration - Invalid JSON Format") { + SECTION("Post Configuration - Invalid JSON Format") + { setPostConfigEndpointEnabled(true); - std::string invalidJson = "this is not valid json"; - auto res = cli.Post("/config", invalidJson, "application/json"); + auto [result, res] = cli.postJson("/config", "this is not valid json"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 400); - REQUIRE(res->body.find("Invalid JSON format") != std::string::npos); + REQUIRE(res->statusCode() == drogon::k400BadRequest); + REQUIRE(std::string(res->body()).find("Invalid JSON format") != std::string::npos); } - SECTION("Post Configuration - Missing Sources") { - std::string newConfig = R"({"http-settings": []})"; - auto res = cli.Post("/config", newConfig, "application/json"); + SECTION("Post Configuration - Missing Sources") + { + setPostConfigEndpointEnabled(true); + auto [result, res] = cli.postJson("/config", R"({"http-settings": []})"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 500); - REQUIRE(res->body.starts_with("Validation failed")); + REQUIRE(res->statusCode() == drogon::k500InternalServerError); + REQUIRE(std::string(res->body()).starts_with("Validation failed")); } - SECTION("Post Configuration - Missing Http Settings") { - std::string newConfig = R"({"sources": []})"; - auto res = cli.Post("/config", newConfig, "application/json"); + SECTION("Post Configuration - Missing Http Settings") + { + setPostConfigEndpointEnabled(true); + auto [result, res] = cli.postJson("/config", R"({"sources": []})"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 500); - REQUIRE(res->body.starts_with("Validation failed")); + REQUIRE(res->statusCode() == drogon::k500InternalServerError); + REQUIRE(std::string(res->body()).starts_with("Validation failed")); } - SECTION("Post Configuration - Valid JSON Config") { + SECTION("Post Configuration - Valid JSON Config") + { + setPostConfigEndpointEnabled(true); std::string newConfig = R"({ "sources": [{"type": "TestDataSource"}], "http-settings": [{"scope": "https://example.com", "password": "MASKED:0:f52fbd32b2b3b86ff88ef6c490628285f482af15ddcb29541f94bcf526a3f6c7"}] })"; - log().set_level(spdlog::level::trace); - auto res = cli.Post("/config", newConfig, "application/json"); + + auto [result, res] = cli.postJson("/config", newConfig); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 200); - REQUIRE(res->body == "Configuration updated and applied successfully."); + REQUIRE(res->statusCode() == drogon::k200OK); + REQUIRE(std::string(res->body()) == "Configuration updated and applied successfully."); - // Check that the password SHA was re-substituted. std::ifstream config(*mapget::DataSourceConfigService::get().getConfigFilePath()); std::stringstream configContentStream; configContentStream << config.rdbuf(); @@ -475,9 +549,5 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") REQUIRE(configContent.find("hunter2") != std::string::npos); } - service.stop(); - REQUIRE(service.isRunning() == false); - - // Clean up the test configuration files. fs::remove(tempConfigPath); } diff --git a/test/unit/test-http-service-fixture.h b/test/unit/test-http-service-fixture.h new file mode 100644 index 00000000..bbb9b0c3 --- /dev/null +++ b/test/unit/test-http-service-fixture.h @@ -0,0 +1,17 @@ +#pragma once + +#include "mapget/http-service/http-service.h" + +namespace mapget::test +{ + +// Starts the HTTP service lazily (on first use) and keeps it alive for the +// lifetime of the test process. `shutdownHttpService()` stops the server and +// joins its server thread to avoid Drogon shutdown issues. +HttpService& httpService(); + +// Safe to call even if the service was never started. +void shutdownHttpService(); + +} // namespace mapget::test + diff --git a/test/unit/test-main.cpp b/test/unit/test-main.cpp new file mode 100644 index 00000000..eb67e32f --- /dev/null +++ b/test/unit/test-main.cpp @@ -0,0 +1,50 @@ +#define CATCH_CONFIG_RUNNER + +#include + +#include + +#include "mapget/log.h" +#include "test-http-service-fixture.h" + +namespace mapget::test +{ +namespace +{ + +std::mutex serviceMutex; +HttpService* servicePtr = nullptr; + +} // namespace + +HttpService& httpService() +{ + std::lock_guard lock(serviceMutex); + + if (!servicePtr) { + // Intentionally leaked to avoid destructor ordering issues at process shutdown. + servicePtr = new HttpService(); + servicePtr->go("127.0.0.1", 0, 5000); + } + + return *servicePtr; +} + +void shutdownHttpService() +{ + std::lock_guard lock(serviceMutex); + if (!servicePtr) + return; + + servicePtr->stop(); +} + +} // namespace mapget::test + +int main(int argc, char* argv[]) +{ + auto result = Catch::Session().run(argc, argv); + mapget::test::shutdownHttpService(); + return result; +} + From 0814ffabbd2c7233c8314ef040ccbed3822b0548 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Fri, 23 Jan 2026 19:26:40 +0100 Subject: [PATCH 07/14] Pick ports for integration tests dynamically. --- test/integration/CMakeLists.txt | 63 ++++++---- .../detect-ports-and-prepare-config-yaml.py | 111 ++++++++++++++++++ test/integration/run-with-ports.bash | 20 ++++ 3 files changed, 169 insertions(+), 25 deletions(-) create mode 100644 test/integration/detect-ports-and-prepare-config-yaml.py create mode 100644 test/integration/run-with-ports.bash diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 6ce5ba32..1af0caf9 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -1,9 +1,7 @@ project(test.mapget.integration CXX) -# TODO: Figure out a way to do this without assuming free ports. -set (MAPGET_SERVER_PORT 61852) -set (DATASOURCE_CPP_PORT 61853) -set (DATASOURCE_PY_PORT 61854) +set(MAPGET_PICK_PORTS_PY "${CMAKE_CURRENT_LIST_DIR}/detect-ports-and-prepare-config-yaml.py") +set(MAPGET_RUN_WITH_PORTS "${CMAKE_CURRENT_LIST_DIR}/run-with-ports.bash") add_wheel_test(test-local-example WORKING_DIRECTORY @@ -20,20 +18,23 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports at test runtime (reduces CI flakiness vs. hard-coded ports) + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-cli-cpp" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "./mapget --log-level trace serve -p ${MAPGET_SERVER_PORT} -d 127.0.0.1:${DATASOURCE_CPP_PORT} -d 127.0.0.1:${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./mapget --log-level trace serve -p $MAPGET_SERVER_PORT -d 127.0.0.1:$DATASOURCE_CPP_PORT -d 127.0.0.1:$DATASOURCE_PY_PORT" # Request from cpp datasource - -f "./mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m Tropico -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m Tropico -l WayLayer -t 12345" # Request from py datasource - -f "./mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m TestMap -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m TestMap -l WayLayer -t 12345" ) endif () @@ -42,11 +43,14 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports at test runtime (reduces CI flakiness vs. hard-coded ports) + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-cli-datasource-exe" + # Run service with auto-launched python datasource - -b "${CMAKE_CURRENT_LIST_DIR}/mapget-exec-datasource.bash ${MAPGET_SERVER_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-datasource-exe/ports.env ${CMAKE_CURRENT_LIST_DIR}/mapget-exec-datasource.bash $MAPGET_SERVER_PORT" # Request from py datasource - -f "./mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m TestMap -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-datasource-exe/ports.env ./mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m TestMap -l WayLayer -t 12345" ) endif () @@ -56,17 +60,20 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports at test runtime (reduces CI flakiness vs. hard-coded ports) + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-cli-python" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "python -m mapget --log-level trace serve -p ${MAPGET_SERVER_PORT} -d 127.0.0.1:${DATASOURCE_CPP_PORT} -d 127.0.0.1:${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env python -m mapget --log-level trace serve -p $MAPGET_SERVER_PORT -d 127.0.0.1:$DATASOURCE_CPP_PORT -d 127.0.0.1:$DATASOURCE_PY_PORT" # Request from py datasource - -f "python -m mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m TestMap -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env python -m mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m TestMap -l WayLayer -t 12345" ) endif() @@ -76,20 +83,23 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports + write config YAMLs at test runtime. + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-config-cpp" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-cpp/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-cpp/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "./mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-service.yaml serve" + -b "./mapget --config .integration/test-config-cpp/sample-service.yaml serve" # Request from py datasource - -f "./mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-second-datasource.yaml fetch" + -f "./mapget --config .integration/test-config-cpp/sample-second-datasource.yaml fetch" # Request from cpp datasource - -f "./mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-first-datasource.yaml fetch" + -f "./mapget --config .integration/test-config-cpp/sample-first-datasource.yaml fetch" ) endif () @@ -99,19 +109,22 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports + write config YAMLs at test runtime. + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-config-py" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-py/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-py/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "python -m mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-service.yaml serve" + -b "python -m mapget --config .integration/test-config-py/sample-service.yaml serve" # Request from py datasource - -f "python -m mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-second-datasource.yaml fetch" + -f "python -m mapget --config .integration/test-config-py/sample-second-datasource.yaml fetch" # Request from cpp datasource - -f "python -m mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-first-datasource.yaml fetch" + -f "python -m mapget --config .integration/test-config-py/sample-first-datasource.yaml fetch" ) endif () diff --git a/test/integration/detect-ports-and-prepare-config-yaml.py b/test/integration/detect-ports-and-prepare-config-yaml.py new file mode 100644 index 00000000..0360ccdd --- /dev/null +++ b/test/integration/detect-ports-and-prepare-config-yaml.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import re +import socket +from pathlib import Path + + +def _pick_free_tcp_ports(count: int) -> list[int]: + if count <= 0: + raise ValueError("count must be > 0") + + sockets: list[socket.socket] = [] + ports: list[int] = [] + try: + for _ in range(count): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("127.0.0.1", 0)) + sockets.append(s) + ports.append(int(s.getsockname()[1])) + finally: + for s in sockets: + try: + s.close() + except Exception: + pass + + if len(set(ports)) != len(ports): + raise RuntimeError(f"Port picker returned duplicates: {ports}") + + return ports + + +def _patch_sample_service_yaml(text: str, mapget_port: int, datasource_cpp_port: int, datasource_py_port: int) -> str: + text = re.sub( + r"(?m)^(\s*port:\s*)\d+(\s*)$", + rf"\g<1>{mapget_port}\g<2>", + text, + count=1, + ) + text = text.replace("127.0.0.1:61853", f"127.0.0.1:{datasource_cpp_port}") + text = text.replace("127.0.0.1:61854", f"127.0.0.1:{datasource_py_port}") + return text + + +def _patch_sample_fetch_yaml(text: str, mapget_port: int) -> str: + return re.sub( + r"(?m)^(\s*server:\s*127\.0\.0\.1:)\d+(\s*)$", + rf"\g<1>{mapget_port}\g<2>", + text, + count=1, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--out-dir", required=True, help="Output directory for generated files (created if needed).") + args = parser.parse_args() + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + # Pick ports at test runtime (reduces collision risk vs. configure-time selection). + mapget_port, datasource_cpp_port, datasource_py_port = _pick_free_tcp_ports(3) + + ports_env = out_dir / "ports.env" + ports_env.write_text( + "\n".join( + [ + f"export MAPGET_SERVER_PORT={mapget_port}", + f"export DATASOURCE_CPP_PORT={datasource_cpp_port}", + f"export DATASOURCE_PY_PORT={datasource_py_port}", + "", + ] + ), + encoding="utf-8", + newline="\n", + ) + + repo_root = Path(__file__).resolve().parents[2] + examples_config = repo_root / "examples" / "config" + + sample_service = (examples_config / "sample-service.yaml").read_text(encoding="utf-8") + (out_dir / "sample-service.yaml").write_text( + _patch_sample_service_yaml(sample_service, mapget_port, datasource_cpp_port, datasource_py_port), + encoding="utf-8", + newline="\n", + ) + + sample_first = (examples_config / "sample-first-datasource.yaml").read_text(encoding="utf-8") + (out_dir / "sample-first-datasource.yaml").write_text( + _patch_sample_fetch_yaml(sample_first, mapget_port), + encoding="utf-8", + newline="\n", + ) + + sample_second = (examples_config / "sample-second-datasource.yaml").read_text(encoding="utf-8") + (out_dir / "sample-second-datasource.yaml").write_text( + _patch_sample_fetch_yaml(sample_second, mapget_port), + encoding="utf-8", + newline="\n", + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/test/integration/run-with-ports.bash b/test/integration/run-with-ports.bash new file mode 100644 index 00000000..b0f17c54 --- /dev/null +++ b/test/integration/run-with-ports.bash @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -lt 2 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +ports_env="$1" +shift + +# shellcheck disable=SC1090 +source "$ports_env" + +# Note: The wheel test harness passes commands as a single string, so `$MAPGET_SERVER_PORT` +# etc. are not expanded. We intentionally use `eval` here so the variables sourced above +# are expanded before executing the command. +eval "exec $*" + From 52c5d1038cf6e355792ad578a00b1166b985ca10 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 26 Jan 2026 13:23:27 +0100 Subject: [PATCH 08/14] Fix tests and sonar findings. --- cmake/deps.cmake | 2 +- libs/http-datasource/src/datasource-server.cpp | 2 +- libs/http-datasource/src/http-server.cpp | 10 +++++++--- libs/http-service/src/http-service.cpp | 3 +++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 629fc978..323dc283 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -140,7 +140,7 @@ if (MAPGET_WITH_SERVICE OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) endif() if (MAPGET_WITH_WHEEL AND NOT TARGET python-cmake-wheel) - CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel@1.2.0") + CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel#80592e483cc2be044f64e35c4686a00a9126abd2@1.2.1") endif() if (MAPGET_ENABLE_TESTING) diff --git a/libs/http-datasource/src/datasource-server.cpp b/libs/http-datasource/src/datasource-server.cpp index 30f83794..98ea11b0 100644 --- a/libs/http-datasource/src/datasource-server.cpp +++ b/libs/http-datasource/src/datasource-server.cpp @@ -121,7 +121,7 @@ void DataSourceServer::setup(drogon::HttpAppFramework& app) std::string content; TileLayerStream::StringPoolOffsetMap stringPoolOffsets{{impl_->info_.nodeId_, stringPoolOffsetParam}}; TileLayerStream::Writer layerWriter{ - [&](std::string bytes, TileLayerStream::MessageType) { content.append(bytes); }, + [&](std::string const& bytes, TileLayerStream::MessageType) { content.append(bytes); }, stringPoolOffsets}; layerWriter.write(tileLayer); diff --git a/libs/http-datasource/src/http-server.cpp b/libs/http-datasource/src/http-server.cpp index f7106509..bd917360 100644 --- a/libs/http-datasource/src/http-server.cpp +++ b/libs/http-datasource/src/http-server.cpp @@ -261,11 +261,15 @@ bool HttpServer::mountFileSystem(std::string const& pathFromTo) if (ec) return false; - if (!std::filesystem::exists(fsRoot, ec) || ec || !std::filesystem::is_directory(fsRoot, ec) || ec) + auto exists = std::filesystem::exists(fsRoot, ec); + if (!exists || ec) + return false; + auto isDirectory = std::filesystem::is_directory(fsRoot, ec); + if (isDirectory || ec) return false; - std::lock_guard lock(impl_->mountsMutex_); - impl_->mounts_.push_back(MountPoint{std::move(urlPrefix), std::move(fsRoot)}); + std::scoped_lock lock(impl_->mountsMutex_); + impl_->mounts_.emplace_back(MountPoint{std::move(urlPrefix), std::move(fsRoot)}); return true; } diff --git a/libs/http-service/src/http-service.cpp b/libs/http-service/src/http-service.cpp index 1a4249ea..b1e531d5 100644 --- a/libs/http-service/src/http-service.cpp +++ b/libs/http-service/src/http-service.cpp @@ -59,6 +59,9 @@ class GzipCompressor ~GzipCompressor() { deflateEnd(&strm_); } + GzipCompressor(GzipCompressor const&) = delete; + GzipCompressor(GzipCompressor&&) = delete; + std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) { std::string result; From b6f5d6ba18b11d2d2dc75d2a0d9476fcfd4dcbea Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 26 Jan 2026 14:14:25 +0100 Subject: [PATCH 09/14] Re-Add TODO. --- libs/http-service/src/http-client.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/http-service/src/http-client.cpp b/libs/http-service/src/http-client.cpp index ea6bb04e..e618a649 100644 --- a/libs/http-service/src/http-client.cpp +++ b/libs/http-service/src/http-client.cpp @@ -120,6 +120,10 @@ LayerTilesRequest::Ptr HttpClient::request(const LayerTilesRequest::Ptr& request auto [result, resp] = impl_->client_->sendRequest(httpReq); if (result == drogon::ReqResult::Ok && resp) { if (resp->statusCode() == drogon::k200OK) { + // TODO: Support streamed/chunked tile responses. + // Drogon's `HttpClient` API only provides the full buffered body. + // True streaming would require a custom client built on + // `trantor::TcpClient` (still within the Drogon dependency). reader->read(std::string(resp->body())); } else if (resp->statusCode() == drogon::k400BadRequest) { request->setStatus(RequestStatus::NoDataSource); @@ -136,4 +140,3 @@ LayerTilesRequest::Ptr HttpClient::request(const LayerTilesRequest::Ptr& request } } // namespace mapget - From e6af301bc7b1fb0b465c5bfe8839915854320ee8 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 26 Jan 2026 18:31:35 +0100 Subject: [PATCH 10/14] Implement Websocket protocol for erdblick. --- docs/mapget-api.md | 36 +- docs/mapget-dev-guide.md | 13 +- docs/mapget-user-guide.md | 2 +- libs/http-service/CMakeLists.txt | 8 + libs/http-service/src/config-handler.cpp | 229 +++++ libs/http-service/src/http-service-impl.cpp | 39 + libs/http-service/src/http-service-impl.h | 72 ++ libs/http-service/src/http-service.cpp | 823 +----------------- libs/http-service/src/locate-handler.cpp | 43 + libs/http-service/src/sources-handler.cpp | 27 + libs/http-service/src/status-handler.cpp | 34 + libs/http-service/src/tiles-http-handler.cpp | 420 +++++++++ libs/http-service/src/tiles-ws-controller.cpp | 558 ++++++++++++ libs/http-service/src/tiles-ws-controller.h | 19 + libs/model/include/mapget/model/stream.h | 6 + libs/model/src/stream.cpp | 7 +- test/unit/test-http-datasource.cpp | 293 +++++++ 17 files changed, 1786 insertions(+), 843 deletions(-) create mode 100644 libs/http-service/src/config-handler.cpp create mode 100644 libs/http-service/src/http-service-impl.cpp create mode 100644 libs/http-service/src/http-service-impl.h create mode 100644 libs/http-service/src/locate-handler.cpp create mode 100644 libs/http-service/src/sources-handler.cpp create mode 100644 libs/http-service/src/status-handler.cpp create mode 100644 libs/http-service/src/tiles-http-handler.cpp create mode 100644 libs/http-service/src/tiles-ws-controller.cpp create mode 100644 libs/http-service/src/tiles-ws-controller.h diff --git a/docs/mapget-api.md b/docs/mapget-api.md index e4e7ce11..18dffb33 100644 --- a/docs/mapget-api.md +++ b/docs/mapget-api.md @@ -1,12 +1,12 @@ -# REST/GeoJSON API Guide +# HTTP / WebSocket API Guide -Mapget exposes a small HTTP API that lets clients discover datasources, stream tiles, abort long‑running requests, locate features by ID and inspect or update the running configuration. This guide describes the endpoints and their request and response formats. +Mapget exposes a small HTTP + WebSocket API that lets clients discover datasources, stream tiles, locate features by ID and inspect or update the running configuration. This guide describes the endpoints and their request and response formats. ## Base URL and formats The server started by `mapget serve` listens on the configured host and port (by default on all interfaces and an automatically chosen port). All endpoints are rooted at that host and port. -Requests that send JSON use `Content-Type: application/json`. Tile streaming supports two response encodings, selected via the `Accept` header: +Requests that send JSON use `Content-Type: application/json`. HTTP tile streaming supports two response encodings, selected via the `Accept` header: - `Accept: application/jsonl` returns a JSON‑Lines stream where each line is one JSON object. - `Accept: application/binary` returns a compact binary stream optimized for high-volume traffic. @@ -23,9 +23,9 @@ The binary format and the logical feature model are described in more detail in Each item contains map ID, available layers and basic metadata. This endpoint is typically used by frontends to discover which maps and layers can be requested via `/tiles`. -## `/tiles` – stream tiles +## `/tiles` – stream tiles (HTTP) -`POST /tiles` streams tiles for one or more map–layer combinations. It is the main data retrieval endpoint used by clients such as erdblick. +`POST /tiles` streams tiles for one or more map–layer combinations. - **Method:** `POST` - **Request body (JSON):** @@ -34,7 +34,6 @@ Each item contains map ID, available layers and basic metadata. This endpoint is - `layerId`: string, ID of the layer within that map. - `tileIds`: array of numeric tile IDs in mapget’s tiling scheme. - `stringPoolOffsets` (optional): dictionary from datasource node ID to last known string ID. Used by advanced clients to avoid receiving the same field names repeatedly in the binary stream. - - `clientId` (optional): arbitrary string identifying this client connection for abort handling. - **Response:** - `application/jsonl` if `Accept: application/jsonl` is sent. - `application/binary` if `Accept: application/binary` is sent, using the tile stream protocol. @@ -43,6 +42,21 @@ Tiles are streamed as they become available. In JSONL mode, each line is the JSO If `Accept-Encoding: gzip` is set, the server compresses responses where possible, which is especially useful for JSONL streams. +To cancel an in-flight HTTP stream, close the HTTP connection. + +## `/tiles` – stream tiles (WebSocket) + +`GET /tiles` supports WebSocket upgrades. This is the preferred tile streaming mode for interactive clients because it supports long-lived connections and request replacement without introducing an extra abort endpoint. + +- **Connect:** `ws://:/tiles` +- **Client → Server:** send one *text* message containing the same JSON body as for `POST /tiles` (`requests`, optional `stringPoolOffsets`). + - `stringPoolOffsets` is optional; the server remembers the latest offsets per WebSocket connection. Clients may re-send it to reset/resync offsets. +- **Server → Client:** sends only *binary* WebSocket messages. Each WebSocket message contains exactly one `TileLayerStream` VTLV frame. + - `StringPool`, `TileFeatureLayer`, `TileSourceDataLayer` are unchanged. + - `Status` frames contain UTF-8 JSON payload describing per-request `RequestStatus` transitions and a human-readable message. The final status frame has `"allDone": true`. + +To cancel, either send a new request message on the same connection (which replaces the current one) or close the WebSocket connection. + ### Why JSONL instead of JSON? JSON Lines is better suited to streaming large responses than a single JSON array. Clients can start processing the first tiles immediately, do not need to buffer the complete response in memory, and can naturally consume the stream with incremental parsers. @@ -87,16 +101,6 @@ Each line in the JSONL response is a GeoJSON-like FeatureCollection with additio The `error` object is only present if an error occurred while filling the tile. When present, the `features` array may be empty or contain partial data. -## `/abort` – cancel tile streaming - -`POST /abort` cancels a running `/tiles` request that was started with a matching `clientId`. It is useful when the viewport changes and the previous stream should be abandoned. - -- **Method:** `POST` -- **Request body (JSON):** `{ "clientId": "" }` -- **Response:** `text/plain` confirmation; a 400 status code if `clientId` is missing. - -Internally the service marks the matching tile requests as aborted and stops scheduling further work for them. - ### Curl Call Example For example, the following curl call could be used to stream GeoJSON feature objects diff --git a/docs/mapget-dev-guide.md b/docs/mapget-dev-guide.md index d7c9b905..3d95b117 100644 --- a/docs/mapget-dev-guide.md +++ b/docs/mapget-dev-guide.md @@ -150,7 +150,7 @@ sequenceDiagram participant Ds as DataSource participant Cache - Client->>Http: POST /tiles
requests, clientId + Client->>Http: POST /tiles
requests Http->>Service: request(requests, headers) Service->>Worker: enqueue jobs per datasource loop per tile @@ -169,30 +169,29 @@ sequenceDiagram Service-->>Http: request complete ``` -If a client supplies a `clientId` in the `/tiles` request, the HTTP layer uses it to track open requests and to implement `/abort`. +For interactive clients, tile streaming can also be done via WebSocket `GET /tiles`, where sending a new request message replaces the current in-flight request on that connection. ## HTTP service internals `mapget::HttpService` binds the core service to an HTTP server implementation. Its responsibilities are: -- map HTTP endpoints to service calls (`/sources`, `/tiles`, `/abort`, `/status`, `/locate`, `/config`), +- map HTTP/WebSocket endpoints to service calls (`/sources`, `/tiles`, `/status`, `/locate`, `/config`), - parse JSON requests and build `LayerTilesRequest` objects, - serialize tile responses as JSONL or binary streams, -- manage per‑client state such as `clientId` for abort handling, and - provide `/config` as a JSON view on the YAML config file. ### Tile streaming For `/tiles`, the HTTP layer: -- parses the JSON body to extract `requests`, `stringPoolOffsets` and an optional `clientId`, +- parses the JSON body to extract `requests` and optional `stringPoolOffsets`, - constructs one `LayerTilesRequest` per map–layer combination, - attaches callbacks that feed results into a shared `HttpTilesRequestState`, and - sends out each tile as soon as it is produced by the service. In JSONL mode the response is a sequence of newline‑separated JSON objects. In binary mode the HTTP layer uses `TileLayerStream::Writer` to serialize string pool updates and tile blobs. Binary responses can optionally be compressed using gzip if the client sends `Accept-Encoding: gzip`. -The `/abort` endpoint uses the `clientId` mechanism to cancel all open tile requests for a given client and to prevent further work from being scheduled for them. +WebSocket `/tiles` uses the same request JSON shape but responds with binary VTLV frames only, and includes `Status` frames (JSON payload) whenever a request’s `RequestStatus` changes. ### Configuration endpoints @@ -207,7 +206,7 @@ These endpoints are guarded by command‑line flags: `--no-get-config` disables The model library provides both the binary tile encoding and the simfil query integration: -- `TileLayerStream::Writer` and `TileLayerStream::Reader` handle versioned, type‑tagged messages for string pools and tile layers. Each message starts with a protocol version, a `MessageType` (string pool, feature tile, SourceData tile, end-of-stream), and a payload size. +- `TileLayerStream::Writer` and `TileLayerStream::Reader` handle versioned, type‑tagged messages for string pools and tile layers. Each message starts with a protocol version, a `MessageType` (string pool, feature tile, SourceData tile, status, end-of-stream), and a payload size. - `TileFeatureLayer` derives from `simfil::ModelPool` and exposes methods such as `evaluate(...)` and `complete(...)` to run simfil expressions and obtain completion candidates. String pools are streamed incrementally. The server keeps a `StringPoolOffsetMap` that tracks, for each ongoing tile request, the highest string ID known to a given client per datasource node id. When a tile is written, `TileLayerStream::Writer` compares that offset with the current `StringPool::highest()` value: diff --git a/docs/mapget-user-guide.md b/docs/mapget-user-guide.md index 978cf6c9..68964c29 100644 --- a/docs/mapget-user-guide.md +++ b/docs/mapget-user-guide.md @@ -10,7 +10,7 @@ The guide is split into several focused documents: - [**Setup Guide**](mapget-setup.md) explains how to install mapget via `pip`, how to build the native executable from source, and how to start a server or use the built‑in `fetch` client for quick experiments. - [**Configuration Guide**](mapget-config.md) documents the YAML configuration file used with `--config`, the supported datasource types (`DataSourceHost`, `DataSourceProcess`, `GridDataSource`, `GeoJsonFolder) and the optional `http-settings` section used by tools and UIs. -- [**REST API Guide**](mapget-api.md) describes the HTTP endpoints exposed by `mapget serve`, including `/sources`, `/tiles`, `/abort`, `/status`, `/locate` and `/config`, along with their request and response formats and example calls. +- [**HTTP / WebSocket API Guide**](mapget-api.md) describes the endpoints exposed by `mapget serve`, including `/sources`, `/tiles`, `/status`, `/locate` and `/config`, along with their request and response formats and example calls. - [**Caching Guide**](mapget-cache.md) covers the available cache modes (`memory`, `persistent`, `none`), explains how to configure cache size and location, and shows how to inspect cache statistics via the status endpoint. - [**Simfil Language Extensions**](mapget-simfil-extensions.md) introduces the feature model, tiling scheme, geometry and validity concepts, and the binary tile stream format. This chapter is especially relevant if you are writing datasources or low‑level clients. - [**Layered Data Model**](mapget-model.md) introduces the feature model, tiling scheme, geometry and validity concepts, and the binary tile stream format. This chapter is especially relevant if you are writing datasources or low‑level clients. diff --git a/libs/http-service/CMakeLists.txt b/libs/http-service/CMakeLists.txt index b9e8799f..99476b81 100644 --- a/libs/http-service/CMakeLists.txt +++ b/libs/http-service/CMakeLists.txt @@ -6,7 +6,15 @@ add_library(mapget-http-service STATIC include/mapget/http-service/cli.h src/http-service.cpp + src/http-service-impl.h + src/http-service-impl.cpp + src/config-handler.cpp src/http-client.cpp + src/locate-handler.cpp + src/sources-handler.cpp + src/status-handler.cpp + src/tiles-http-handler.cpp + src/tiles-ws-controller.cpp src/cli.cpp) target_include_directories(mapget-http-service diff --git a/libs/http-service/src/config-handler.cpp b/libs/http-service/src/config-handler.cpp new file mode 100644 index 00000000..82ac86e6 --- /dev/null +++ b/libs/http-service/src/config-handler.cpp @@ -0,0 +1,229 @@ +#include "http-service-impl.h" + +#include "cli.h" +#include "mapget/log.h" +#include "mapget/service/config.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "nlohmann/json.hpp" +#include "yaml-cpp/yaml.h" + +namespace mapget +{ + +drogon::HttpResponsePtr HttpService::Impl::openConfigFile(std::ifstream& configFile) +{ + auto configFilePath = DataSourceConfigService::get().getConfigFilePath(); + if (!configFilePath.has_value()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k404NotFound); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The config file path is not set. Check the server configuration."); + return resp; + } + + std::filesystem::path path = *configFilePath; + if (!std::filesystem::exists(path)) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k404NotFound); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The server does not have a config file."); + return resp; + } + + configFile.open(*configFilePath); + if (!configFile) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Failed to open config file."); + return resp; + } + + return nullptr; +} + +void HttpService::Impl::handleGetConfigRequest( + const drogon::HttpRequestPtr& /*req*/, + std::function&& callback) +{ + if (!isGetConfigEndpointEnabled()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k403Forbidden); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The GET /config endpoint is disabled by the server administrator."); + callback(resp); + return; + } + + std::ifstream configFile; + if (auto errorResp = openConfigFile(configFile)) { + callback(errorResp); + return; + } + + nlohmann::json jsonSchema = DataSourceConfigService::get().getDataSourceConfigSchema(); + + try { + YAML::Node configYaml = YAML::Load(configFile); + nlohmann::json jsonConfig; + std::unordered_map maskedSecretMap; + for (const auto& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + if (auto configYamlEntry = configYaml[key]) + jsonConfig[key] = yamlToJson(configYaml[key], true, &maskedSecretMap); + } + + nlohmann::json combinedJson; + combinedJson["schema"] = jsonSchema; + combinedJson["model"] = jsonConfig; + combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(combinedJson.dump(2)); + callback(resp); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error processing config file: ") + e.what()); + callback(resp); + } +} + +void HttpService::Impl::handlePostConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + if (!isPostConfigEndpointEnabled()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k403Forbidden); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The POST /config endpoint is not enabled by the server administrator."); + callback(resp); + return; + } + + struct ConfigUpdateState : std::enable_shared_from_this + { + trantor::EventLoop* loop = nullptr; + std::atomic_bool done{false}; + std::atomic_bool wroteConfig{false}; + std::unique_ptr subscription; + std::function callback; + }; + + std::ifstream configFile; + if (auto errorResp = openConfigFile(configFile)) { + callback(errorResp); + return; + } + + nlohmann::json jsonConfig; + try { + jsonConfig = nlohmann::json::parse(std::string(req->body())); + } + catch (const nlohmann::json::parse_error& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON format: ") + e.what()); + callback(resp); + return; + } + + try { + DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Validation failed: ") + e.what()); + callback(resp); + return; + } + + auto yamlConfig = YAML::Load(configFile); + std::unordered_map maskedSecrets; + yamlToJson(yamlConfig, true, &maskedSecrets); + + for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + if (jsonConfig.contains(key)) + yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); + } + + auto state = std::make_shared(); + state->loop = drogon::app().getLoop(); + state->callback = std::move(callback); + + state->subscription = DataSourceConfigService::get().subscribe( + [state](std::vector const&) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Configuration updated and applied successfully."); + state->callback(resp); + state->subscription.reset(); + }); + }, + [state](std::string const& error) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state, error]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error applying the configuration: ") + error); + state->callback(resp); + state->subscription.reset(); + }); + }); + + configFile.close(); + log().trace("Writing new config."); + state->wroteConfig = true; + if (auto configFilePath = DataSourceConfigService::get().getConfigFilePath()) { + std::ofstream newConfigFile(*configFilePath); + newConfigFile << yamlConfig; + newConfigFile.close(); + } + + std::thread([weak = state->weak_from_this()]() { + std::this_thread::sleep_for(std::chrono::seconds(60)); + if (auto state = weak.lock()) { + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Timeout while waiting for config to update."); + state->callback(resp); + state->subscription.reset(); + }); + } + }).detach(); +} + +} // namespace mapget + diff --git a/libs/http-service/src/http-service-impl.cpp b/libs/http-service/src/http-service-impl.cpp new file mode 100644 index 00000000..1f323b31 --- /dev/null +++ b/libs/http-service/src/http-service-impl.cpp @@ -0,0 +1,39 @@ +#include "http-service-impl.h" + +#include "mapget/log.h" + +#ifdef __linux__ +#include +#endif + +namespace mapget +{ + +HttpService::Impl::Impl(HttpService& self, const HttpServiceConfig& config) : self_(self), config_(config) {} + +void HttpService::Impl::tryMemoryTrim(ResponseType responseType) const +{ + uint64_t interval = + (responseType == ResponseType::Binary) ? config_.memoryTrimIntervalBinary : config_.memoryTrimIntervalJson; + + if (interval == 0) { + return; + } + + auto& counter = (responseType == ResponseType::Binary) ? binaryRequestCounter_ : jsonRequestCounter_; + auto count = counter.fetch_add(1, std::memory_order_relaxed); + if ((count % interval) != 0) { + return; + } + +#ifdef __linux__ +#ifndef NDEBUG + const char* typeStr = (responseType == ResponseType::Binary) ? "binary" : "JSON"; + log().debug("Trimming memory after {} {} requests (interval: {})", count, typeStr, interval); +#endif + malloc_trim(0); +#endif +} + +} // namespace mapget + diff --git a/libs/http-service/src/http-service-impl.h b/libs/http-service/src/http-service-impl.h new file mode 100644 index 00000000..2db77a6c --- /dev/null +++ b/libs/http-service/src/http-service-impl.h @@ -0,0 +1,72 @@ +#pragma once + +#include "http-service.h" + +#include +#include + +#include +#include +#include +#include + +namespace mapget +{ + +namespace detail +{ + +[[nodiscard]] inline AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) +{ + AuthHeaders headers; + for (auto const& [k, v] : req->headers()) { + headers.emplace(k, v); + } + return headers; +} + +} // namespace detail + +struct HttpService::Impl +{ + HttpService& self_; + HttpServiceConfig config_; + mutable std::atomic binaryRequestCounter_{0}; + mutable std::atomic jsonRequestCounter_{0}; + + explicit Impl(HttpService& self, const HttpServiceConfig& config); + + enum class ResponseType { Binary, Json }; + + void tryMemoryTrim(ResponseType responseType) const; + + struct TilesStreamState; + + void handleTilesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + void handleSourcesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + void handleStatusRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + void handleLocateRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + static drogon::HttpResponsePtr openConfigFile(std::ifstream& configFile); + + static void handleGetConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback); + + void handlePostConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; +}; + +} // namespace mapget diff --git a/libs/http-service/src/http-service.cpp b/libs/http-service/src/http-service.cpp index b1e531d5..4e377448 100644 --- a/libs/http-service/src/http-service.cpp +++ b/libs/http-service/src/http-service.cpp @@ -1,824 +1,16 @@ -#include "http-service.h" +#include "http-service-impl.h" -#include "cli.h" -#include "mapget/log.h" -#include "mapget/service/config.h" +#include "tiles-ws-controller.h" #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nlohmann/json-schema.hpp" -#include "nlohmann/json.hpp" -#include "yaml-cpp/yaml.h" - -#include - -#ifdef __linux__ -#include -#endif +#include namespace mapget { -namespace -{ - -/** - * Simple gzip compressor for streaming compression. - */ -class GzipCompressor -{ -public: - GzipCompressor() - { - strm_.zalloc = Z_NULL; - strm_.zfree = Z_NULL; - strm_.opaque = Z_NULL; - // 16+MAX_WBITS enables gzip format (not just deflate) - int ret = deflateInit2( - &strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY); - if (ret != Z_OK) { - throw std::runtime_error("Failed to initialize gzip compressor"); - } - } - - ~GzipCompressor() { deflateEnd(&strm_); } - - GzipCompressor(GzipCompressor const&) = delete; - GzipCompressor(GzipCompressor&&) = delete; - - std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) - { - std::string result; - if (size == 0 && flush_mode == Z_NO_FLUSH) { - return result; - } - - strm_.avail_in = static_cast(size); - strm_.next_in = reinterpret_cast(const_cast(data)); - - char outbuf[8192]; - do { - strm_.avail_out = sizeof(outbuf); - strm_.next_out = reinterpret_cast(outbuf); - - int ret = deflate(&strm_, flush_mode); - if (ret == Z_STREAM_ERROR) { - throw std::runtime_error("Gzip compression failed"); - } - - size_t have = sizeof(outbuf) - strm_.avail_out; - result.append(outbuf, have); - } while (strm_.avail_out == 0); - - return result; - } - - std::string finish() { return compress(nullptr, 0, Z_FINISH); } - -private: - z_stream strm_{}; -}; - -[[nodiscard]] AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) -{ - AuthHeaders headers; - for (auto const& [k, v] : req->headers()) { - headers.emplace(k, v); - } - return headers; -} - -[[nodiscard]] bool containsGzip(std::string_view acceptEncoding) -{ - return !acceptEncoding.empty() && acceptEncoding.find("gzip") != std::string_view::npos; -} - -} // namespace - -struct HttpService::Impl -{ - HttpService& self_; - HttpServiceConfig config_; - mutable std::atomic binaryRequestCounter_{0}; - mutable std::atomic jsonRequestCounter_{0}; - - explicit Impl(HttpService& self, const HttpServiceConfig& config) : self_(self), config_(config) {} - - enum class ResponseType { Binary, Json }; - - void tryMemoryTrim(ResponseType responseType) const - { - uint64_t interval = - (responseType == ResponseType::Binary) ? config_.memoryTrimIntervalBinary : config_.memoryTrimIntervalJson; - - if (interval == 0) { - return; - } - - auto& counter = (responseType == ResponseType::Binary) ? binaryRequestCounter_ : jsonRequestCounter_; - auto count = counter.fetch_add(1, std::memory_order_relaxed); - if ((count % interval) != 0) { - return; - } - -#ifdef __linux__ -#ifndef NDEBUG - const char* typeStr = (responseType == ResponseType::Binary) ? "binary" : "JSON"; - log().debug("Trimming memory after {} {} requests (interval: {})", count, typeStr, interval); -#endif - malloc_trim(0); -#endif - } - - struct TilesStreamState : std::enable_shared_from_this - { - static constexpr auto binaryMimeType = "application/binary"; - static constexpr auto jsonlMimeType = "application/jsonl"; - static constexpr auto anyMimeType = "*/*"; - - explicit TilesStreamState(Impl const& impl, trantor::EventLoop* loop) : impl_(impl), loop_(loop) - { - static std::atomic_uint64_t nextRequestId; - requestId_ = nextRequestId++; - writer_ = std::make_unique( - [this](auto&& msg, auto&& /*msgType*/) { appendOutgoingUnlocked(msg); }, stringOffsets_); - } - - void attachStream(drogon::ResponseStreamPtr stream) - { - { - std::lock_guard lock(mutex_); - if (aborted_ || responseEnded_) { - if (stream) - stream->close(); - return; - } - stream_ = std::move(stream); - } - scheduleDrain(); - } - - void parseRequestFromJson(nlohmann::json const& requestJson) - { - std::string mapId = requestJson["mapId"]; - std::string layerId = requestJson["layerId"]; - std::vector tileIds; - tileIds.reserve(requestJson["tileIds"].size()); - for (auto const& tid : requestJson["tileIds"].get>()) { - tileIds.emplace_back(tid); - } - requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); - } - - [[nodiscard]] bool setResponseTypeFromAccept(std::string_view acceptHeader, std::string& error) - { - responseType_ = std::string(acceptHeader); - if (responseType_.empty()) - responseType_ = anyMimeType; - if (responseType_ == anyMimeType) - responseType_ = binaryMimeType; - - if (responseType_ == binaryMimeType) { - trimResponseType_ = ResponseType::Binary; - return true; - } - if (responseType_ == jsonlMimeType) { - trimResponseType_ = ResponseType::Json; - return true; - } - - error = "Unknown Accept header value: " + responseType_; - return false; - } - - void enableGzip() { compressor_ = std::make_unique(); } - - void onAborted() - { - if (aborted_.exchange(true)) - return; - for (auto const& req : requests_) { - if (!req->isDone()) { - impl_.self_.abort(req); - } - } - drogon::ResponseStreamPtr stream; - { - std::lock_guard lock(mutex_); - if (responseEnded_.exchange(true)) - return; - stream = std::move(stream_); - } - if (stream) - stream->close(); - } - - void addResult(TileLayer::Ptr const& result) - { - { - std::lock_guard lock(mutex_); - if (aborted_) - return; - - log().debug("Response ready: {}", MapTileKey(*result).toString()); - if (responseType_ == binaryMimeType) { - writer_->write(result); - } else { - auto dumped = result->toJson().dump( - -1, ' ', false, nlohmann::json::error_handler_t::ignore); - appendOutgoingUnlocked(dumped); - appendOutgoingUnlocked("\n"); - } - } - scheduleDrain(); - } - - void onRequestDone() - { - { - std::lock_guard lock(mutex_); - if (aborted_) - return; - - bool allDoneNow = std::all_of( - requests_.begin(), requests_.end(), [](auto const& r) { return r->isDone(); }); - - if (allDoneNow && !allDone_) { - allDone_ = true; - if (responseType_ == binaryMimeType && !endOfStreamSent_) { - writer_->sendEndOfStream(); - endOfStreamSent_ = true; - } - } - } - scheduleDrain(); - } - - void scheduleDrain() - { - if (aborted_ || responseEnded_) - return; - if (drainScheduled_.exchange(true)) - return; - - auto weak = weak_from_this(); - loop_->queueInLoop([weak = std::move(weak)]() mutable { - if (auto self = weak.lock()) { - self->drainOnLoop(); - } - }); - } - - void drainOnLoop() - { - drainScheduled_ = false; - if (aborted_ || responseEnded_) - return; - - constexpr size_t maxChunk = 64 * 1024; - - for (;;) { - std::string chunk; - bool done = false; - bool needAbort = false; - bool scheduleAgain = false; - drogon::ResponseStreamPtr streamToClose; - { - std::lock_guard lock(mutex_); - if (!stream_) - return; - - if (!pending_.empty()) { - size_t n = std::min(pending_.size(), maxChunk); - chunk.assign(pending_.data(), n); - pending_.erase(0, n); - } else { - if (allDone_ && compressor_ && !compressionFinished_) { - pending_.append(compressor_->finish()); - compressionFinished_ = true; - continue; - } - done = allDone_; - } - - if (!chunk.empty()) { - if (!stream_->send(chunk)) { - needAbort = true; - } else if (!pending_.empty() || allDone_) { - // Keep draining until we sent everything and closed the stream. - scheduleAgain = true; - } - } else if (done) { - responseEnded_ = true; - streamToClose = std::move(stream_); - } - } - - if (needAbort) { - onAborted(); - return; - } - - if (done) { - if (streamToClose) - streamToClose->close(); - impl_.tryMemoryTrim(trimResponseType_); - return; - } - if (scheduleAgain) - scheduleDrain(); - return; - } - } - - void appendOutgoingUnlocked(std::string_view bytes) - { - if (bytes.empty()) - return; - - if (compressor_) { - pending_.append(compressor_->compress(bytes.data(), bytes.size())); - } else { - pending_.append(bytes); - } - } - - Impl const& impl_; - trantor::EventLoop* loop_; - - std::mutex mutex_; - uint64_t requestId_ = 0; - - std::string responseType_; - ResponseType trimResponseType_ = ResponseType::Binary; - - std::string pending_; - drogon::ResponseStreamPtr stream_; - std::unique_ptr writer_; - std::vector requests_; - TileLayerStream::StringPoolOffsetMap stringOffsets_; - - std::unique_ptr compressor_; - bool compressionFinished_ = false; - bool endOfStreamSent_ = false; - bool allDone_ = false; - - std::atomic_bool aborted_{false}; - std::atomic_bool drainScheduled_{false}; - std::atomic_bool responseEnded_{false}; - }; - - mutable std::mutex clientRequestMapMutex_; - mutable std::unordered_map> requestStatePerClientId_; - - void abortRequestsForClientId( - std::string const& clientId, - std::shared_ptr newState = nullptr) const - { - std::unique_lock clientRequestMapAccess(clientRequestMapMutex_); - auto clientRequestIt = requestStatePerClientId_.find(clientId); - if (clientRequestIt != requestStatePerClientId_.end()) { - bool anySoftAbort = false; - for (auto const& req : clientRequestIt->second->requests_) { - if (!req->isDone()) { - self_.abort(req); - anySoftAbort = true; - } - } - if (anySoftAbort) - log().warn("Soft-aborting tiles request {}", clientRequestIt->second->requestId_); - requestStatePerClientId_.erase(clientRequestIt); - } - if (newState) { - requestStatePerClientId_.emplace(clientId, std::move(newState)); - } - } - - void handleTilesRequest( - const drogon::HttpRequestPtr& req, - std::function&& callback) const - { - auto state = std::make_shared(*this, drogon::app().getLoop()); - - const std::string accept = req->getHeader("accept"); - const std::string acceptEncoding = req->getHeader("accept-encoding"); - auto clientHeaders = authHeadersFromRequest(req); - - nlohmann::json j; - try { - j = nlohmann::json::parse(std::string(req->body())); - } - catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Invalid JSON: ") + e.what()); - callback(resp); - return; - } - - auto requestsIt = j.find("requests"); - if (requestsIt == j.end() || !requestsIt->is_array()) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("Missing or invalid 'requests' array"); - callback(resp); - return; - } - - log().info("Processing tiles request {}", state->requestId_); - for (auto& requestJson : *requestsIt) { - state->parseRequestFromJson(requestJson); - } - - if (j.contains("stringPoolOffsets")) { - for (auto& item : j["stringPoolOffsets"].items()) { - state->stringOffsets_[item.key()] = item.value().get(); - } - } - - std::string acceptError; - if (!state->setResponseTypeFromAccept(accept, acceptError)) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::move(acceptError)); - callback(resp); - return; - } - - const bool gzip = containsGzip(acceptEncoding); - if (gzip) { - state->enableGzip(); - } - - for (auto& request : state->requests_) { - request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); - request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); - request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; - } - - const auto canProcess = self_.request(state->requests_, clientHeaders); - if (!canProcess) { - std::vector> requestStatuses{}; - bool anyUnauthorized = false; - for (auto const& r : state->requests_) { - auto status = r->getStatus(); - requestStatuses.emplace_back(static_cast>(status)); - anyUnauthorized |= (status == RequestStatus::Unauthorized); - } - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(anyUnauthorized ? drogon::k403Forbidden : drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); - resp->setBody(nlohmann::json::object({{"status", requestStatuses}}).dump()); - callback(resp); - return; - } - - if (j.contains("clientId")) { - abortRequestsForClientId(j["clientId"].get(), state); - } - - auto resp = drogon::HttpResponse::newAsyncStreamResponse( - [state](drogon::ResponseStreamPtr stream) { state->attachStream(std::move(stream)); }, - true); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeString(state->responseType_); - if (gzip) { - resp->addHeader("Content-Encoding", "gzip"); - } - callback(resp); - } - - void handleAbortRequest( - const drogon::HttpRequestPtr& req, - std::function&& callback) const - { - try { - auto j = nlohmann::json::parse(std::string(req->body())); - if (j.contains("clientId")) { - abortRequestsForClientId(j["clientId"].get()); - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("OK"); - callback(resp); - return; - } - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("Missing clientId"); - callback(resp); - } - catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Invalid JSON: ") + e.what()); - callback(resp); - } - } - - void handleSourcesRequest( - const drogon::HttpRequestPtr& req, - std::function&& callback) const - { - auto sourcesInfo = nlohmann::json::array(); - for (auto& source : self_.info(authHeadersFromRequest(req))) { - sourcesInfo.push_back(source.toJson()); - } - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); - resp->setBody(sourcesInfo.dump()); - callback(resp); - } - - void handleStatusRequest( - const drogon::HttpRequestPtr&, - std::function&& callback) const - { - auto serviceStats = self_.getStatistics(); - auto cacheStats = self_.cache()->getStatistics(); - - std::ostringstream oss; - oss << ""; - oss << "

Status Information

"; - oss << "

Service Statistics

"; - oss << "
" << serviceStats.dump(4) << "
"; - oss << "

Cache Statistics

"; - oss << "
" << cacheStats.dump(4) << "
"; - oss << ""; - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_TEXT_HTML); - resp->setBody(oss.str()); - callback(resp); - } - - void handleLocateRequest( - const drogon::HttpRequestPtr& req, - std::function&& callback) const - { - try { - nlohmann::json j = nlohmann::json::parse(std::string(req->body())); - auto requestsJson = j["requests"]; - auto allResponsesJson = nlohmann::json::array(); - - for (auto const& locateReqJson : requestsJson) { - LocateRequest locateReq{locateReqJson}; - auto responsesJson = nlohmann::json::array(); - for (auto const& resp : self_.locate(locateReq)) - responsesJson.emplace_back(resp.serialize()); - allResponsesJson.emplace_back(responsesJson); - } - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); - resp->setBody(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); - callback(resp); - } - catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Invalid JSON: ") + e.what()); - callback(resp); - } - } - - static drogon::HttpResponsePtr openConfigFile(std::ifstream& configFile) - { - auto configFilePath = DataSourceConfigService::get().getConfigFilePath(); - if (!configFilePath.has_value()) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k404NotFound); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("The config file path is not set. Check the server configuration."); - return resp; - } - - std::filesystem::path path = *configFilePath; - if (!std::filesystem::exists(path)) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k404NotFound); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("The server does not have a config file."); - return resp; - } - - configFile.open(*configFilePath); - if (!configFile) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("Failed to open config file."); - return resp; - } - - return nullptr; - } - - static void handleGetConfigRequest( - const drogon::HttpRequestPtr&, - std::function&& callback) - { - if (!isGetConfigEndpointEnabled()) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k403Forbidden); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("The GET /config endpoint is disabled by the server administrator."); - callback(resp); - return; - } - - std::ifstream configFile; - if (auto errorResp = openConfigFile(configFile)) { - callback(errorResp); - return; - } - - nlohmann::json jsonSchema = DataSourceConfigService::get().getDataSourceConfigSchema(); - - try { - YAML::Node configYaml = YAML::Load(configFile); - nlohmann::json jsonConfig; - std::unordered_map maskedSecretMap; - for (const auto& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { - if (auto configYamlEntry = configYaml[key]) - jsonConfig[key] = yamlToJson(configYaml[key], true, &maskedSecretMap); - } - - nlohmann::json combinedJson; - combinedJson["schema"] = jsonSchema; - combinedJson["model"] = jsonConfig; - combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); - resp->setBody(combinedJson.dump(2)); - callback(resp); - } - catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Error processing config file: ") + e.what()); - callback(resp); - } - } - - void handlePostConfigRequest( - const drogon::HttpRequestPtr& req, - std::function&& callback) const - { - if (!isPostConfigEndpointEnabled()) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k403Forbidden); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("The POST /config endpoint is not enabled by the server administrator."); - callback(resp); - return; - } - - struct ConfigUpdateState : std::enable_shared_from_this - { - trantor::EventLoop* loop = nullptr; - std::atomic_bool done{false}; - std::atomic_bool wroteConfig{false}; - std::unique_ptr subscription; - std::function callback; - }; - - std::ifstream configFile; - if (auto errorResp = openConfigFile(configFile)) { - callback(errorResp); - return; - } - - nlohmann::json jsonConfig; - try { - jsonConfig = nlohmann::json::parse(std::string(req->body())); - } - catch (const nlohmann::json::parse_error& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Invalid JSON format: ") + e.what()); - callback(resp); - return; - } - - try { - DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); - } - catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Validation failed: ") + e.what()); - callback(resp); - return; - } - - auto yamlConfig = YAML::Load(configFile); - std::unordered_map maskedSecrets; - yamlToJson(yamlConfig, true, &maskedSecrets); - - for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { - if (jsonConfig.contains(key)) - yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); - } - - auto state = std::make_shared(); - state->loop = drogon::app().getLoop(); - state->callback = std::move(callback); - - // Subscribe before writing; ignore any callbacks that happen before we write. - state->subscription = DataSourceConfigService::get().subscribe( - [state](std::vector const&) mutable { - if (!state->wroteConfig) { - return; - } - if (state->done.exchange(true)) - return; - state->loop->queueInLoop([state]() mutable { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("Configuration updated and applied successfully."); - state->callback(resp); - state->subscription.reset(); - }); - }, - [state](std::string const& error) mutable { - if (!state->wroteConfig) { - return; - } - if (state->done.exchange(true)) - return; - state->loop->queueInLoop([state, error]() mutable { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Error applying the configuration: ") + error); - state->callback(resp); - state->subscription.reset(); - }); - }); - - configFile.close(); - log().trace("Writing new config."); - state->wroteConfig = true; - if (auto configFilePath = DataSourceConfigService::get().getConfigFilePath()) { - std::ofstream newConfigFile(*configFilePath); - newConfigFile << yamlConfig; - newConfigFile.close(); - } - - // Timeout fail-safe (rare endpoint; ok to spawn a thread). - std::thread([weak = state->weak_from_this()]() { - std::this_thread::sleep_for(std::chrono::seconds(60)); - if (auto state = weak.lock()) { - if (state->done.exchange(true)) - return; - state->loop->queueInLoop([state]() mutable { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("Timeout while waiting for config to update."); - state->callback(resp); - state->subscription.reset(); - }); - } - }).detach(); - } -}; - HttpService::HttpService(Cache::Ptr cache, const HttpServiceConfig& config) : Service(std::move(cache), config.watchConfig, config.defaultTtl), impl_(std::make_unique(*this, config)) { @@ -828,6 +20,8 @@ HttpService::~HttpService() = default; void HttpService::setup(drogon::HttpAppFramework& app) { + detail::registerTilesWebSocketController(app, *this); + app.registerHandler( "/tiles", [this](const drogon::HttpRequestPtr& req, std::function&& callback) { @@ -835,13 +29,6 @@ void HttpService::setup(drogon::HttpAppFramework& app) }, {drogon::Post}); - app.registerHandler( - "/abort", - [this](const drogon::HttpRequestPtr& req, std::function&& callback) { - impl_->handleAbortRequest(req, std::move(callback)); - }, - {drogon::Post}); - app.registerHandler( "/sources", [this](const drogon::HttpRequestPtr& req, std::function&& callback) { diff --git a/libs/http-service/src/locate-handler.cpp b/libs/http-service/src/locate-handler.cpp new file mode 100644 index 00000000..81ad9ea5 --- /dev/null +++ b/libs/http-service/src/locate-handler.cpp @@ -0,0 +1,43 @@ +#include "http-service-impl.h" + +#include + +#include "nlohmann/json.hpp" + +namespace mapget +{ + +void HttpService::Impl::handleLocateRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + try { + nlohmann::json j = nlohmann::json::parse(std::string(req->body())); + auto requestsJson = j["requests"]; + auto allResponsesJson = nlohmann::json::array(); + + for (auto const& locateReqJson : requestsJson) { + LocateRequest locateReq{locateReqJson}; + auto responsesJson = nlohmann::json::array(); + for (auto const& resp : self_.locate(locateReq)) + responsesJson.emplace_back(resp.serialize()); + allResponsesJson.emplace_back(responsesJson); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); + callback(resp); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + } +} + +} // namespace mapget + diff --git a/libs/http-service/src/sources-handler.cpp b/libs/http-service/src/sources-handler.cpp new file mode 100644 index 00000000..d085ac3f --- /dev/null +++ b/libs/http-service/src/sources-handler.cpp @@ -0,0 +1,27 @@ +#include "http-service-impl.h" + +#include + +#include "nlohmann/json.hpp" + +namespace mapget +{ + +void HttpService::Impl::handleSourcesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + auto sourcesInfo = nlohmann::json::array(); + for (auto& source : self_.info(detail::authHeadersFromRequest(req))) { + sourcesInfo.push_back(source.toJson()); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(sourcesInfo.dump()); + callback(resp); +} + +} // namespace mapget + diff --git a/libs/http-service/src/status-handler.cpp b/libs/http-service/src/status-handler.cpp new file mode 100644 index 00000000..b0a91216 --- /dev/null +++ b/libs/http-service/src/status-handler.cpp @@ -0,0 +1,34 @@ +#include "http-service-impl.h" + +#include + +#include + +namespace mapget +{ + +void HttpService::Impl::handleStatusRequest( + const drogon::HttpRequestPtr& /*req*/, + std::function&& callback) const +{ + auto serviceStats = self_.getStatistics(); + auto cacheStats = self_.cache()->getStatistics(); + + std::ostringstream oss; + oss << ""; + oss << "

Status Information

"; + oss << "

Service Statistics

"; + oss << "
" << serviceStats.dump(4) << "
"; + oss << "

Cache Statistics

"; + oss << "
" << cacheStats.dump(4) << "
"; + oss << ""; + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_HTML); + resp->setBody(oss.str()); + callback(resp); +} + +} // namespace mapget + diff --git a/libs/http-service/src/tiles-http-handler.cpp b/libs/http-service/src/tiles-http-handler.cpp new file mode 100644 index 00000000..3edaa1ad --- /dev/null +++ b/libs/http-service/src/tiles-http-handler.cpp @@ -0,0 +1,420 @@ +#include "http-service-impl.h" + +#include "mapget/log.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nlohmann/json.hpp" + +#include + +namespace mapget +{ +namespace +{ + +class GzipCompressor +{ +public: + GzipCompressor() + { + strm_.zalloc = Z_NULL; + strm_.zfree = Z_NULL; + strm_.opaque = Z_NULL; + // 16+MAX_WBITS enables gzip format (not just deflate) + int ret = deflateInit2( + &strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { + throw std::runtime_error("Failed to initialize gzip compressor"); + } + } + + ~GzipCompressor() { deflateEnd(&strm_); } + + GzipCompressor(GzipCompressor const&) = delete; + GzipCompressor(GzipCompressor&&) = delete; + + std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) + { + std::string result; + if (size == 0 && flush_mode == Z_NO_FLUSH) { + return result; + } + + strm_.avail_in = static_cast(size); + strm_.next_in = reinterpret_cast(const_cast(data)); + + char outbuf[8192]; + do { + strm_.avail_out = sizeof(outbuf); + strm_.next_out = reinterpret_cast(outbuf); + + int ret = deflate(&strm_, flush_mode); + if (ret == Z_STREAM_ERROR) { + throw std::runtime_error("Gzip compression failed"); + } + + size_t have = sizeof(outbuf) - strm_.avail_out; + result.append(outbuf, have); + } while (strm_.avail_out == 0); + + return result; + } + + std::string finish() { return compress(nullptr, 0, Z_FINISH); } + +private: + z_stream strm_{}; +}; + +[[nodiscard]] bool containsGzip(std::string_view acceptEncoding) +{ + return !acceptEncoding.empty() && acceptEncoding.find("gzip") != std::string_view::npos; +} + +} // namespace + +struct HttpService::Impl::TilesStreamState : std::enable_shared_from_this +{ + static constexpr auto binaryMimeType = "application/binary"; + static constexpr auto jsonlMimeType = "application/jsonl"; + static constexpr auto anyMimeType = "*/*"; + + explicit TilesStreamState(Impl const& impl, trantor::EventLoop* loop) : impl_(impl), loop_(loop) + { + static std::atomic_uint64_t nextRequestId; + requestId_ = nextRequestId++; + writer_ = std::make_unique( + [this](auto&& msg, auto&& /*msgType*/) { appendOutgoingUnlocked(msg); }, stringOffsets_); + } + + void attachStream(drogon::ResponseStreamPtr stream) + { + { + std::lock_guard lock(mutex_); + if (aborted_ || responseEnded_) { + if (stream) + stream->close(); + return; + } + stream_ = std::move(stream); + } + scheduleDrain(); + } + + void parseRequestFromJson(nlohmann::json const& requestJson) + { + std::string mapId = requestJson["mapId"]; + std::string layerId = requestJson["layerId"]; + std::vector tileIds; + tileIds.reserve(requestJson["tileIds"].size()); + for (auto const& tid : requestJson["tileIds"].get>()) { + tileIds.emplace_back(tid); + } + requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); + } + + [[nodiscard]] bool setResponseTypeFromAccept(std::string_view acceptHeader, std::string& error) + { + responseType_ = std::string(acceptHeader); + if (responseType_.empty()) + responseType_ = anyMimeType; + if (responseType_ == anyMimeType) + responseType_ = binaryMimeType; + + if (responseType_ == binaryMimeType) { + trimResponseType_ = HttpService::Impl::ResponseType::Binary; + return true; + } + if (responseType_ == jsonlMimeType) { + trimResponseType_ = HttpService::Impl::ResponseType::Json; + return true; + } + + error = "Unknown Accept header value: " + responseType_; + return false; + } + + void enableGzip() { compressor_ = std::make_unique(); } + + void onAborted() + { + if (aborted_.exchange(true)) + return; + for (auto const& req : requests_) { + if (!req->isDone()) { + impl_.self_.abort(req); + } + } + drogon::ResponseStreamPtr stream; + { + std::lock_guard lock(mutex_); + if (responseEnded_.exchange(true)) + return; + stream = std::move(stream_); + } + if (stream) + stream->close(); + } + + void addResult(TileLayer::Ptr const& result) + { + { + std::lock_guard lock(mutex_); + if (aborted_) + return; + + log().debug("Response ready: {}", MapTileKey(*result).toString()); + if (responseType_ == binaryMimeType) { + writer_->write(result); + } else { + auto dumped = result->toJson().dump(-1, ' ', false, nlohmann::json::error_handler_t::ignore); + appendOutgoingUnlocked(dumped); + appendOutgoingUnlocked("\n"); + } + } + scheduleDrain(); + } + + void onRequestDone() + { + { + std::lock_guard lock(mutex_); + if (aborted_) + return; + + bool allDoneNow = + std::all_of(requests_.begin(), requests_.end(), [](auto const& r) { return r->isDone(); }); + + if (allDoneNow && !allDone_) { + allDone_ = true; + if (responseType_ == binaryMimeType && !endOfStreamSent_) { + writer_->sendEndOfStream(); + endOfStreamSent_ = true; + } + } + } + scheduleDrain(); + } + + void scheduleDrain() + { + if (aborted_ || responseEnded_) + return; + if (drainScheduled_.exchange(true)) + return; + + auto weak = weak_from_this(); + loop_->queueInLoop([weak = std::move(weak)]() mutable { + if (auto self = weak.lock()) { + self->drainOnLoop(); + } + }); + } + + void drainOnLoop() + { + drainScheduled_ = false; + if (aborted_ || responseEnded_) + return; + + constexpr size_t maxChunk = 64 * 1024; + + for (;;) { + std::string chunk; + bool done = false; + bool needAbort = false; + bool scheduleAgain = false; + drogon::ResponseStreamPtr streamToClose; + { + std::lock_guard lock(mutex_); + if (!stream_) + return; + + if (!pending_.empty()) { + size_t n = std::min(pending_.size(), maxChunk); + chunk.assign(pending_.data(), n); + pending_.erase(0, n); + } else { + if (allDone_ && compressor_ && !compressionFinished_) { + pending_.append(compressor_->finish()); + compressionFinished_ = true; + continue; + } + done = allDone_; + } + + if (!chunk.empty()) { + if (!stream_->send(chunk)) { + needAbort = true; + } else if (!pending_.empty() || allDone_) { + scheduleAgain = true; + } + } else if (done) { + responseEnded_ = true; + streamToClose = std::move(stream_); + } + } + + if (needAbort) { + onAborted(); + return; + } + + if (done) { + if (streamToClose) + streamToClose->close(); + impl_.tryMemoryTrim(trimResponseType_); + return; + } + if (scheduleAgain) + scheduleDrain(); + return; + } + } + + void appendOutgoingUnlocked(std::string_view bytes) + { + if (bytes.empty()) + return; + + if (compressor_) { + pending_.append(compressor_->compress(bytes.data(), bytes.size())); + } else { + pending_.append(bytes); + } + } + + Impl const& impl_; + trantor::EventLoop* loop_; + + std::mutex mutex_; + uint64_t requestId_ = 0; + + std::string responseType_; + HttpService::Impl::ResponseType trimResponseType_ = HttpService::Impl::ResponseType::Binary; + + std::string pending_; + drogon::ResponseStreamPtr stream_; + std::unique_ptr writer_; + std::vector requests_; + TileLayerStream::StringPoolOffsetMap stringOffsets_; + + std::unique_ptr compressor_; + bool compressionFinished_ = false; + bool endOfStreamSent_ = false; + bool allDone_ = false; + + std::atomic_bool aborted_{false}; + std::atomic_bool drainScheduled_{false}; + std::atomic_bool responseEnded_{false}; +}; + +void HttpService::Impl::handleTilesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + auto state = std::make_shared(*this, drogon::app().getLoop()); + + const std::string accept = req->getHeader("accept"); + const std::string acceptEncoding = req->getHeader("accept-encoding"); + auto clientHeaders = detail::authHeadersFromRequest(req); + + nlohmann::json j; + try { + j = nlohmann::json::parse(std::string(req->body())); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + return; + } + + auto requestsIt = j.find("requests"); + if (requestsIt == j.end() || !requestsIt->is_array()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Missing or invalid 'requests' array"); + callback(resp); + return; + } + + log().info("Processing tiles request {}", state->requestId_); + for (auto& requestJson : *requestsIt) { + state->parseRequestFromJson(requestJson); + } + + if (j.contains("stringPoolOffsets")) { + for (auto& item : j["stringPoolOffsets"].items()) { + state->stringOffsets_[item.key()] = item.value().get(); + } + } + + std::string acceptError; + if (!state->setResponseTypeFromAccept(accept, acceptError)) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::move(acceptError)); + callback(resp); + return; + } + + const bool gzip = containsGzip(acceptEncoding); + if (gzip) { + state->enableGzip(); + } + + for (auto& request : state->requests_) { + request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); + request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); + request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; + } + + const auto canProcess = self_.request(state->requests_, clientHeaders); + if (!canProcess) { + std::vector> requestStatuses{}; + bool anyUnauthorized = false; + for (auto const& r : state->requests_) { + auto status = r->getStatus(); + requestStatuses.emplace_back(static_cast>(status)); + anyUnauthorized |= (status == RequestStatus::Unauthorized); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(anyUnauthorized ? drogon::k403Forbidden : drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(nlohmann::json::object({{"status", requestStatuses}}).dump()); + callback(resp); + return; + } + + auto resp = drogon::HttpResponse::newAsyncStreamResponse( + [state](drogon::ResponseStreamPtr stream) { state->attachStream(std::move(stream)); }, + true); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeString(state->responseType_); + if (gzip) { + resp->addHeader("Content-Encoding", "gzip"); + } + callback(resp); +} + +} // namespace mapget diff --git a/libs/http-service/src/tiles-ws-controller.cpp b/libs/http-service/src/tiles-ws-controller.cpp new file mode 100644 index 00000000..82fefac9 --- /dev/null +++ b/libs/http-service/src/tiles-ws-controller.cpp @@ -0,0 +1,558 @@ +#include "tiles-ws-controller.h" + +#include "mapget/http-service/http-service.h" + +#include "mapget/log.h" +#include "mapget/model/stream.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "nlohmann/json.hpp" + +namespace mapget::detail +{ +namespace +{ + +[[nodiscard]] AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) +{ + AuthHeaders headers; + for (auto const& [k, v] : req->headers()) { + headers.emplace(k, v); + } + return headers; +} + +[[nodiscard]] std::string_view requestStatusToString(RequestStatus s) +{ + switch (s) { + case RequestStatus::Open: + return "Open"; + case RequestStatus::Success: + return "Success"; + case RequestStatus::NoDataSource: + return "NoDataSource"; + case RequestStatus::Unauthorized: + return "Unauthorized"; + case RequestStatus::Aborted: + return "Aborted"; + } + return "Unknown"; +} + +[[nodiscard]] std::string encodeStreamMessage(TileLayerStream::MessageType type, std::string_view payload) +{ + std::ostringstream headerStream; + bitsery::Serializer s(headerStream); + s.object(TileLayerStream::CurrentProtocolVersion); + s.value1b(type); + s.value4b(static_cast(payload.size())); + + auto message = headerStream.str(); + message.append(payload); + return message; +} + +struct WsConnectionState +{ + AuthHeaders authHeaders; + TileLayerStream::StringPoolOffsetMap stringPoolOffsets; + std::shared_ptr session; +}; + +class TilesWsSession : public std::enable_shared_from_this +{ +public: + TilesWsSession( + HttpService& service, + std::weak_ptr conn, + std::weak_ptr connState, + AuthHeaders authHeaders, + TileLayerStream::StringPoolOffsetMap initialOffsets) + : service_(service), + loop_(drogon::app().getLoop()), + conn_(std::move(conn)), + connState_(std::move(connState)), + authHeaders_(std::move(authHeaders)), + offsets_(std::move(initialOffsets)), + writer_( + std::make_unique( + [this](std::string msg, TileLayerStream::MessageType type) { onWriterMessage(std::move(msg), type); }, + offsets_)) + { + } + + ~TilesWsSession() + { + // Best-effort cleanup: abort any in-flight requests if the session is destroyed. + cancelNoStatus(); + } + + TilesWsSession(TilesWsSession const&) = delete; + TilesWsSession& operator=(TilesWsSession const&) = delete; + + void start(const nlohmann::json& j) + { + auto requestsIt = j.find("requests"); + if (requestsIt == j.end() || !requestsIt->is_array()) { + queueStatusMessage("Missing or invalid 'requests' array"); + scheduleDrain(); + return; + } + + try { + requests_.clear(); + requests_.reserve(requestsIt->size()); + requestStatuses_.clear(); + requestStatuses_.reserve(requestsIt->size()); + + for (auto const& requestJson : *requestsIt) { + const std::string mapId = requestJson.at("mapId").get(); + const std::string layerId = requestJson.at("layerId").get(); + const auto& tileIdsJson = requestJson.at("tileIds"); + if (!tileIdsJson.is_array()) { + throw std::runtime_error("tileIds must be an array"); + } + + std::vector tileIds; + tileIds.reserve(tileIdsJson.size()); + for (auto const& tid : tileIdsJson) { + tileIds.emplace_back(tid.get()); + } + + requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); + requestStatuses_.push_back(RequestStatus::Open); + } + } + catch (const std::exception& e) { + queueStatusMessage(fmt::format("Invalid request JSON: {}", e.what())); + scheduleDrain(); + return; + } + + // Hook request callbacks before calling service_.request so early + // failures (NoDataSource/Unauthorized) still produce status updates. + const auto weak = weak_from_this(); + for (size_t i = 0; i < requests_.size(); ++i) { + auto& req = requests_[i]; + req->onFeatureLayer([weak](auto&& layer) { + if (auto self = weak.lock()) { + self->onTileLayer(std::move(layer)); + } + }); + req->onSourceDataLayer([weak](auto&& layer) { + if (auto self = weak.lock()) { + self->onTileLayer(std::move(layer)); + } + }); + req->onDone_ = [weak, i](RequestStatus status) { + if (auto self = weak.lock()) { + self->onRequestDone(i, status); + } + }; + } + + // Start processing (may synchronously set request statuses). + (void)service_.request(requests_, authHeaders_); + + { + std::lock_guard lock(mutex_); + statusEmissionEnabled_ = true; + } + queueStatusMessage({}); + scheduleDrain(); + } + + void cancel(std::string reason) + { + cancelled_ = true; + + // Stop sending any queued tile frames from this session. + { + std::lock_guard lock(mutex_); + outgoing_.clear(); + } + + // Abort in-flight requests (best-effort). + for (auto const& r : requests_) { + if (!r || r->isDone()) + continue; + service_.abort(r); + } + + // Refresh locally cached statuses after aborting. + { + std::lock_guard lock(mutex_); + for (size_t i = 0; i < requests_.size() && i < requestStatuses_.size(); ++i) { + if (requests_[i]) { + requestStatuses_[i] = requests_[i]->getStatus(); + } + } + } + + queueStatusMessage(std::move(reason)); + scheduleDrain(); + } + +private: + struct OutgoingFrame + { + std::string bytes; + std::optional> stringPoolCommit; + }; + + struct WriterMessage + { + std::string bytes; + TileLayerStream::MessageType type{TileLayerStream::MessageType::None}; + }; + + void cancelNoStatus() + { + if (cancelled_.exchange(true)) + return; + + // Ensure we stop emitting any further frames. + { + std::lock_guard lock(mutex_); + outgoing_.clear(); + } + + for (auto const& r : requests_) { + if (!r || r->isDone()) + continue; + service_.abort(r); + } + } + + void onWriterMessage(std::string msg, TileLayerStream::MessageType type) + { + // Writer messages are only generated from within onTileLayer under mutex_. + if (!currentWriteBatch_) { + raise("TilesWsSession writer callback used out-of-band"); + } + currentWriteBatch_->push_back(WriterMessage{std::move(msg), type}); + } + + void onTileLayer(TileLayer::Ptr layer) + { + if (cancelled_) + return; + if (!layer) + return; + + std::vector batch; + std::optional> stringPoolCommit; + + { + std::lock_guard lock(mutex_); + if (cancelled_) + return; + + currentWriteBatch_ = &batch; + writer_->write(layer); + currentWriteBatch_ = nullptr; + + // If a StringPool message was generated, the writer updates offsets_ + // to the new highest string ID for this node after emitting it. + const auto nodeId = layer->nodeId(); + const auto it = offsets_.find(nodeId); + if (it != offsets_.end()) { + const auto newOffset = it->second; + for (auto const& m : batch) { + if (m.type == TileLayerStream::MessageType::StringPool) { + stringPoolCommit = std::make_pair(nodeId, newOffset); + break; + } + } + } + + for (auto& m : batch) { + OutgoingFrame frame; + frame.bytes = std::move(m.bytes); + if (m.type == TileLayerStream::MessageType::StringPool) { + frame.stringPoolCommit = stringPoolCommit; + } + outgoing_.push_back(std::move(frame)); + } + } + + scheduleDrain(); + } + + void onRequestDone(size_t requestIndex, RequestStatus status) + { + if (cancelled_) + return; + + bool shouldEmit = false; + { + std::lock_guard lock(mutex_); + if (cancelled_) + return; + if (requestIndex >= requestStatuses_.size()) + return; + if (requestStatuses_[requestIndex] == status) + return; + requestStatuses_[requestIndex] = status; + shouldEmit = statusEmissionEnabled_; + } + + if (shouldEmit) { + queueStatusMessage({}); + scheduleDrain(); + } + } + + void queueStatusMessage(std::string message) + { + OutgoingFrame frame; + frame.bytes = encodeStreamMessage(TileLayerStream::MessageType::Status, buildStatusPayload(std::move(message))); + { + std::lock_guard lock(mutex_); + outgoing_.push_back(std::move(frame)); + } + } + + [[nodiscard]] std::string buildStatusPayload(std::string message) + { + nlohmann::json requestsJson = nlohmann::json::array(); + bool allDone = true; + + { + std::lock_guard lock(mutex_); + for (size_t i = 0; i < requests_.size(); ++i) { + const auto status = (i < requestStatuses_.size()) ? requestStatuses_[i] : RequestStatus::Open; + allDone &= (status != RequestStatus::Open); + + nlohmann::json reqJson = nlohmann::json::object(); + reqJson["index"] = i; + if (i < requests_.size() && requests_[i]) { + reqJson["mapId"] = requests_[i]->mapId_; + reqJson["layerId"] = requests_[i]->layerId_; + } else { + reqJson["mapId"] = ""; + reqJson["layerId"] = ""; + } + reqJson["status"] = static_cast>(status); + reqJson["statusText"] = std::string(requestStatusToString(status)); + requestsJson.push_back(std::move(reqJson)); + } + } + + return nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", allDone}, + {"requests", std::move(requestsJson)}, + {"message", std::move(message)}, + }).dump(); + } + + void scheduleDrain() + { + if (drainScheduled_.exchange(true)) + return; + + auto weak = weak_from_this(); + loop_->queueInLoop([weak = std::move(weak)]() mutable { + if (auto self = weak.lock()) { + self->drainOnLoop(); + } + }); + } + + void drainOnLoop() + { + drainScheduled_ = false; + + auto conn = conn_.lock(); + if (!conn || conn->disconnected()) { + cancelNoStatus(); + return; + } + + constexpr size_t maxFramesPerDrain = 256; + for (size_t i = 0; i < maxFramesPerDrain; ++i) { + OutgoingFrame frame; + { + std::lock_guard lock(mutex_); + if (outgoing_.empty()) { + break; + } + frame = std::move(outgoing_.front()); + outgoing_.pop_front(); + } + + conn->send(frame.bytes, drogon::WebSocketMessageType::Binary); + if (frame.stringPoolCommit) { + if (auto state = connState_.lock()) { + state->stringPoolOffsets[frame.stringPoolCommit->first] = frame.stringPoolCommit->second; + } + } + } + + { + std::lock_guard lock(mutex_); + if (outgoing_.empty()) + return; + } + scheduleDrain(); + } + + HttpService& service_; + trantor::EventLoop* loop_; + std::weak_ptr conn_; + std::weak_ptr connState_; + + AuthHeaders authHeaders_; + + std::mutex mutex_; + std::deque outgoing_; + + std::vector requests_; + std::vector requestStatuses_; + bool statusEmissionEnabled_ = false; + + TileLayerStream::StringPoolOffsetMap offsets_; + std::unique_ptr writer_; + std::vector* currentWriteBatch_ = nullptr; + + std::atomic_bool drainScheduled_{false}; + std::atomic_bool cancelled_{false}; +}; + +class TilesWebSocketController final : public drogon::WebSocketController +{ +public: + explicit TilesWebSocketController(HttpService& service) : service_(service) {} + + void handleNewConnection(const drogon::HttpRequestPtr& req, const drogon::WebSocketConnectionPtr& conn) override + { + auto state = std::make_shared(); + state->authHeaders = authHeadersFromRequest(req); + conn->setContext(std::move(state)); + } + + void handleNewMessage( + const drogon::WebSocketConnectionPtr& conn, + std::string&& message, + const drogon::WebSocketMessageType& type) override + { + auto state = conn->getContext(); + if (!state) { + state = std::make_shared(); + conn->setContext(state); + } + + if (type != drogon::WebSocketMessageType::Text) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", "Expected a text message containing JSON."}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + + nlohmann::json j; + try { + j = nlohmann::json::parse(message); + } + catch (const std::exception& e) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", fmt::format("Invalid JSON: {}", e.what())}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + + // Patch per-connection string pool offsets if supplied. + if (j.contains("stringPoolOffsets")) { + if (!j["stringPoolOffsets"].is_object()) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", "stringPoolOffsets must be an object."}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + try { + for (auto const& item : j["stringPoolOffsets"].items()) { + state->stringPoolOffsets[item.key()] = item.value().get(); + } + } + catch (const std::exception& e) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", fmt::format("Invalid stringPoolOffsets: {}", e.what())}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + } + + if (state->session) { + state->session->cancel("Replaced by a new /tiles WebSocket request."); + state->session.reset(); + } + + state->session = std::make_shared( + service_, + conn, + state, + state->authHeaders, + state->stringPoolOffsets); + state->session->start(j); + } + + void handleConnectionClosed(const drogon::WebSocketConnectionPtr& conn) override + { + if (auto state = conn->getContext()) { + if (state->session) { + state->session->cancel("WebSocket connection closed."); + } + } + } + + WS_PATH_LIST_BEGIN + WS_PATH_ADD("/tiles", drogon::Get); + WS_PATH_LIST_END + +private: + HttpService& service_; +}; + +} // namespace + +void registerTilesWebSocketController(drogon::HttpAppFramework& app, HttpService& service) +{ + app.registerController(std::make_shared(service)); +} + +} // namespace mapget::detail diff --git a/libs/http-service/src/tiles-ws-controller.h b/libs/http-service/src/tiles-ws-controller.h new file mode 100644 index 00000000..4acaed45 --- /dev/null +++ b/libs/http-service/src/tiles-ws-controller.h @@ -0,0 +1,19 @@ +#pragma once + +namespace drogon +{ +class HttpAppFramework; +} + +namespace mapget +{ +class HttpService; +} + +namespace mapget::detail +{ + +void registerTilesWebSocketController(drogon::HttpAppFramework& app, HttpService& service); + +} // namespace mapget::detail + diff --git a/libs/model/include/mapget/model/stream.h b/libs/model/include/mapget/model/stream.h index 4568e117..5308ba4f 100644 --- a/libs/model/include/mapget/model/stream.h +++ b/libs/model/include/mapget/model/stream.h @@ -29,6 +29,12 @@ class TileLayerStream StringPool = 1, TileFeatureLayer = 2, TileSourceDataLayer = 3, + /** + * JSON-encoded status updates, e.g. for WebSocket /tiles. + * + * Payload: UTF-8 JSON bytes (not null-terminated). + */ + Status = 4, EndOfStream = 128 }; diff --git a/libs/model/src/stream.cpp b/libs/model/src/stream.cpp index f7d78c18..f59da85d 100644 --- a/libs/model/src/stream.cpp +++ b/libs/model/src/stream.cpp @@ -50,7 +50,6 @@ bool TileLayerStream::Reader::continueReading() } } - bitsery::Deserializer s(buffer_); auto numUnreadBytes = buffer_.tellp() - buffer_.tellg(); if (numUnreadBytes < nextValueSize_) return false; @@ -80,6 +79,12 @@ bool TileLayerStream::Reader::continueReading() std::string stringPoolNodeId = StringPool::readDataSourceNodeId(buffer_); stringPoolProvider_->getStringPool(stringPoolNodeId)->read(buffer_); } + else + { + // Skip unknown message types for forward compatibility (e.g. status + // messages on WebSocket streams). + buffer_.seekg(nextValueSize_, std::ios_base::cur); + } currentPhase_ = Phase::ReadHeader; return true; diff --git a/test/unit/test-http-datasource.cpp b/test/unit/test-http-datasource.cpp index a39f63c6..a2f12e2a 100644 --- a/test/unit/test-http-datasource.cpp +++ b/test/unit/test-http-datasource.cpp @@ -1,9 +1,11 @@ #include +#include #include #include #include #include +#include #include #include #include @@ -19,6 +21,7 @@ #include #include +#include #include #include "process.hpp" @@ -386,6 +389,296 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(request->getStatus() == RequestStatus::Success); REQUIRE(receivedTileCount == 1); } + + auto runWsTilesRequest = [&](bool sendAuthHeader, std::string requestJson) { + auto wsLoopThread = std::make_unique("MapgetTestWsClient"); + wsLoopThread->run(); + + auto wsClient = drogon::WebSocketClient::newWebSocketClient( + fmt::format("ws://127.0.0.1:{}", service.port()), + wsLoopThread->getLoop()); + + std::mutex mutex; + std::condition_variable cv; + std::optional lastStatus; + std::atomic_int receivedTileCount{0}; + std::string error; + + const auto dsInfo = remoteDataSource->info(); + const auto layerInfo = dsInfo.getLayer("WayLayer"); + REQUIRE(layerInfo != nullptr); + + TileLayerStream::Reader reader( + [&](auto&&, auto&&) { return layerInfo; }, + [&](auto&& tile) { + if (tile->id().layer_ != LayerType::Features) { + std::lock_guard lock(mutex); + error = "Unexpected tile layer type"; + } + receivedTileCount.fetch_add(1, std::memory_order_relaxed); + }); + + wsClient->setMessageHandler( + [&](std::string&& msg, + const drogon::WebSocketClientPtr&, + const drogon::WebSocketMessageType& msgType) { + if (msgType != drogon::WebSocketMessageType::Binary) { + return; + } + + TileLayerStream::MessageType type = TileLayerStream::MessageType::None; + uint32_t payloadSize = 0; + std::stringstream ss; + ss.write(msg.data(), static_cast(msg.size())); + if (!TileLayerStream::Reader::readMessageHeader(ss, type, payloadSize)) { + std::lock_guard lock(mutex); + error = "Failed to read stream message header"; + cv.notify_all(); + return; + } + + if (type == TileLayerStream::MessageType::Status) { + std::string payload(payloadSize, '\0'); + ss.read(payload.data(), static_cast(payloadSize)); + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(payload); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse status JSON: ") + e.what(); + cv.notify_all(); + return; + } + { + std::lock_guard lock(mutex); + lastStatus = std::move(parsed); + } + cv.notify_all(); + return; + } + + try { + reader.read(msg); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse tile stream: ") + e.what(); + cv.notify_all(); + } + }); + + auto connectReq = drogon::HttpRequest::newHttpRequest(); + connectReq->setMethod(drogon::Get); + connectReq->setPath("/tiles"); + if (sendAuthHeader) { + connectReq->addHeader("X-USER-ROLE", "Tropico-Viewer"); + } + + std::promise connectPromise; + auto connectFuture = connectPromise.get_future(); + wsClient->connectToServer( + connectReq, + [&connectPromise]( + drogon::ReqResult result, + const drogon::HttpResponsePtr&, + const drogon::WebSocketClientPtr&) { connectPromise.set_value(result); }); + + REQUIRE(connectFuture.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + REQUIRE(connectFuture.get() == drogon::ReqResult::Ok); + + auto conn = wsClient->getConnection(); + if (!conn || !conn->connected()) { + wsClient->stop(); + FAIL("WebSocket connection not established"); + } + + conn->send(requestJson, drogon::WebSocketMessageType::Text); + + { + std::unique_lock lock(mutex); + REQUIRE(cv.wait_for(lock, std::chrono::seconds(10), [&] { + return !error.empty() || + (lastStatus.has_value() && lastStatus->value("allDone", false)); + })); + if (!error.empty()) { + wsClient->stop(); + FAIL(error); + } + } + + wsClient->stop(); + + REQUIRE(lastStatus.has_value()); + return std::make_tuple(*lastStatus, receivedTileCount.load(std::memory_order_relaxed)); + }; + + // WebSocket tiles: unauthorized without auth header. + { + auto req = nlohmann::json::object({ + {"requests", nlohmann::json::array({nlohmann::json::object({ + {"mapId", "Tropico"}, + {"layerId", "WayLayer"}, + {"tileIds", nlohmann::json::array({1234})}, + })})}, + }).dump(); + + auto [status, wsTileCount] = runWsTilesRequest(false, req); + REQUIRE(wsTileCount == 0); + REQUIRE(status["requests"].size() == 1); + REQUIRE(status["requests"][0]["status"].get() == + static_cast(RequestStatus::Unauthorized)); + } + + // WebSocket tiles: invalid request stays on the same connection, then succeeds. + { + auto wsLoopThread = std::make_unique("MapgetTestWsClientReuse"); + wsLoopThread->run(); + + auto wsClient = drogon::WebSocketClient::newWebSocketClient( + fmt::format("ws://127.0.0.1:{}", service.port()), + wsLoopThread->getLoop()); + + std::mutex mutex; + std::condition_variable cv; + std::optional lastStatus; + std::atomic_int receivedTileCount{0}; + std::string error; + + const auto dsInfo = remoteDataSource->info(); + const auto layerInfo = dsInfo.getLayer("WayLayer"); + REQUIRE(layerInfo != nullptr); + + TileLayerStream::Reader reader( + [&](auto&&, auto&&) { return layerInfo; }, + [&](auto&&) { receivedTileCount.fetch_add(1, std::memory_order_relaxed); }); + + wsClient->setMessageHandler( + [&](std::string&& msg, + const drogon::WebSocketClientPtr&, + const drogon::WebSocketMessageType& msgType) { + if (msgType != drogon::WebSocketMessageType::Binary) { + return; + } + + TileLayerStream::MessageType type = TileLayerStream::MessageType::None; + uint32_t payloadSize = 0; + std::stringstream ss; + ss.write(msg.data(), static_cast(msg.size())); + if (!TileLayerStream::Reader::readMessageHeader(ss, type, payloadSize)) { + std::lock_guard lock(mutex); + error = "Failed to read stream message header"; + cv.notify_all(); + return; + } + + if (type == TileLayerStream::MessageType::Status) { + std::string payload(payloadSize, '\0'); + ss.read(payload.data(), static_cast(payloadSize)); + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(payload); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse status JSON: ") + e.what(); + cv.notify_all(); + return; + } + { + std::lock_guard lock(mutex); + lastStatus = std::move(parsed); + } + cv.notify_all(); + return; + } + + try { + reader.read(msg); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse tile stream: ") + e.what(); + cv.notify_all(); + } + }); + + auto connectReq = drogon::HttpRequest::newHttpRequest(); + connectReq->setMethod(drogon::Get); + connectReq->setPath("/tiles"); + connectReq->addHeader("X-USER-ROLE", "Tropico-Viewer"); + + std::promise connectPromise; + auto connectFuture = connectPromise.get_future(); + wsClient->connectToServer( + connectReq, + [&connectPromise]( + drogon::ReqResult result, + const drogon::HttpResponsePtr&, + const drogon::WebSocketClientPtr&) { connectPromise.set_value(result); }); + + REQUIRE(connectFuture.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + REQUIRE(connectFuture.get() == drogon::ReqResult::Ok); + + auto conn = wsClient->getConnection(); + if (!conn || !conn->connected()) { + wsClient->stop(); + FAIL("WebSocket connection not established"); + } + + // Invalid JSON: should yield a Status message but keep the socket open. + { + conn->send("{not json", drogon::WebSocketMessageType::Text); + std::unique_lock lock(mutex); + REQUIRE(cv.wait_for(lock, std::chrono::seconds(5), [&] { + return !error.empty() || + (lastStatus.has_value() && lastStatus->value("allDone", false)); + })); + if (!error.empty()) { + wsClient->stop(); + FAIL(error); + } + REQUIRE(lastStatus->value("message", "").find("Invalid JSON") != std::string::npos); + REQUIRE(conn->connected()); + } + + // Valid request should succeed afterwards. + { + { + std::lock_guard lock(mutex); + lastStatus.reset(); + } + receivedTileCount.store(0, std::memory_order_relaxed); + + auto req = nlohmann::json::object({ + {"requests", nlohmann::json::array({nlohmann::json::object({ + {"mapId", "Tropico"}, + {"layerId", "WayLayer"}, + {"tileIds", nlohmann::json::array({1234})}, + })})}, + }).dump(); + + conn->send(req, drogon::WebSocketMessageType::Text); + + std::unique_lock lock(mutex); + REQUIRE(cv.wait_for(lock, std::chrono::seconds(10), [&] { + return !error.empty() || + (lastStatus.has_value() && lastStatus->value("allDone", false)); + })); + if (!error.empty()) { + wsClient->stop(); + FAIL(error); + } + + REQUIRE(receivedTileCount.load(std::memory_order_relaxed) == 1); + REQUIRE(lastStatus->contains("requests")); + REQUIRE((*lastStatus)["requests"].size() == 1); + REQUIRE((*lastStatus)["requests"][0]["status"].get() == + static_cast(RequestStatus::Success)); + } + + wsClient->stop(); + } } service.remove(remoteDataSource); From 9d36ecf86728a0751b6ac1f3c7b4b9d6f55712f8 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 27 Jan 2026 14:21:41 +0100 Subject: [PATCH 11/14] Communicate Tile Load State. --- .../http-datasource/datasource-client.h | 12 ++++- .../http-datasource/src/datasource-client.cpp | 19 ++++++-- libs/http-datasource/src/http-server.cpp | 2 +- libs/http-service/src/cli.cpp | 2 +- libs/http-service/src/tiles-ws-controller.cpp | 46 +++++++++++++++++++ libs/model/include/mapget/model/layer.h | 17 +++++++ libs/model/include/mapget/model/stream.h | 8 +++- libs/model/src/layer.cpp | 12 +++++ .../include/mapget/service/datasource.h | 6 ++- libs/service/include/mapget/service/service.h | 8 ++++ libs/service/src/datasource.cpp | 12 ++++- libs/service/src/service.cpp | 17 ++++++- 12 files changed, 149 insertions(+), 12 deletions(-) diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h index 91038d73..5a00d000 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h @@ -47,7 +47,11 @@ class RemoteDataSource : public DataSource DataSourceInfo info() override; void fill(TileFeatureLayer::Ptr const& featureTile) override; void fill(TileSourceDataLayer::Ptr const& blobTile) override; - TileLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) override; + TileLayer::Ptr get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback = {}) override; std::vector locate(const mapget::LocateRequest &req) override; private: @@ -87,7 +91,11 @@ class RemoteDataSourceProcess : public DataSource DataSourceInfo info() override; void fill(TileFeatureLayer::Ptr const& featureTile) override; void fill(TileSourceDataLayer::Ptr const& sourceDataLayer) override; - TileLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) override; + TileLayer::Ptr get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback = {}) override; std::vector locate(const mapget::LocateRequest &req) override; private: diff --git a/libs/http-datasource/src/datasource-client.cpp b/libs/http-datasource/src/datasource-client.cpp index 128a5446..0a17088b 100644 --- a/libs/http-datasource/src/datasource-client.cpp +++ b/libs/http-datasource/src/datasource-client.cpp @@ -70,7 +70,11 @@ void RemoteDataSource::fill(const TileSourceDataLayer::Ptr& blobTile) } TileLayer::Ptr -RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceInfo& info) +RemoteDataSource::get( + const MapTileKey& k, + Cache::Ptr& cache, + const DataSourceInfo& info, + TileLayer::LoadStateCallback loadStateCallback) { // Round-robin usage of http clients to facilitate parallel requests. auto& client = httpClients_[(nextClient_++) % httpClients_.size()]; @@ -101,7 +105,7 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn // Use tile instantiation logic of the base class, // the error is then set in fill(). - return DataSource::get(k, cache, info); + return DataSource::get(k, cache, info, std::move(loadStateCallback)); } // Check the response body for expected content. @@ -112,6 +116,9 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn cache); reader.read(std::string(tileResponse->body())); + if (result && loadStateCallback) { + result->setLoadStateCallback(std::move(loadStateCallback)); + } return result; } @@ -243,11 +250,15 @@ void RemoteDataSourceProcess::fill(TileSourceDataLayer::Ptr const& sourceDataLay } TileLayer::Ptr -RemoteDataSourceProcess::get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) +RemoteDataSourceProcess::get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback) { if (!remoteSource_) raise("Remote data source is not initialized."); - return remoteSource_->get(k, cache, info); + return remoteSource_->get(k, cache, info, std::move(loadStateCallback)); } std::vector RemoteDataSourceProcess::locate(const LocateRequest& req) diff --git a/libs/http-datasource/src/http-server.cpp b/libs/http-datasource/src/http-server.cpp index bd917360..422ab1b6 100644 --- a/libs/http-datasource/src/http-server.cpp +++ b/libs/http-datasource/src/http-server.cpp @@ -265,7 +265,7 @@ bool HttpServer::mountFileSystem(std::string const& pathFromTo) if (!exists || ec) return false; auto isDirectory = std::filesystem::is_directory(fsRoot, ec); - if (isDirectory || ec) + if (!isDirectory || ec) return false; std::scoped_lock lock(impl_->mountsMutex_); diff --git a/libs/http-service/src/cli.cpp b/libs/http-service/src/cli.cpp index 7e51c540..f8ea8c6a 100644 --- a/libs/http-service/src/cli.cpp +++ b/libs/http-service/src/cli.cpp @@ -446,7 +446,7 @@ struct ServeCommand log().info("Webapp: {}", webapp_); if (!srv.mountFileSystem(webapp_)) { log().error(" ...failed to mount!"); - exit(1); + raise("Failed to mount webapp filesystem path."); } } diff --git a/libs/http-service/src/tiles-ws-controller.cpp b/libs/http-service/src/tiles-ws-controller.cpp index 82fefac9..6d7f5d23 100644 --- a/libs/http-service/src/tiles-ws-controller.cpp +++ b/libs/http-service/src/tiles-ws-controller.cpp @@ -58,6 +58,19 @@ namespace return "Unknown"; } +[[nodiscard]] std::string_view loadStateToString(TileLayer::LoadState s) +{ + switch (s) { + case TileLayer::LoadState::LoadingQueued: + return "LoadingQueued"; + case TileLayer::LoadState::BackendFetching: + return "BackendFetching"; + case TileLayer::LoadState::BackendConverting: + return "BackendConverting"; + } + return "Unknown"; +} + [[nodiscard]] std::string encodeStreamMessage(TileLayerStream::MessageType type, std::string_view payload) { std::ostringstream headerStream; @@ -163,6 +176,11 @@ class TilesWsSession : public std::enable_shared_from_this self->onTileLayer(std::move(layer)); } }); + req->onLayerLoadStateChanged([weak](MapTileKey const& key, TileLayer::LoadState state) { + if (auto self = weak.lock()) { + self->onLoadStateChanged(key, state); + } + }); req->onDone_ = [weak, i](RequestStatus status) { if (auto self = weak.lock()) { self->onRequestDone(i, status); @@ -332,6 +350,22 @@ class TilesWsSession : public std::enable_shared_from_this } } + void onLoadStateChanged(MapTileKey const& key, TileLayer::LoadState state) + { + if (cancelled_) + return; + + OutgoingFrame frame; + frame.bytes = encodeStreamMessage( + TileLayerStream::MessageType::LoadStateChange, + buildLoadStatePayload(key, state)); + { + std::lock_guard lock(mutex_); + outgoing_.push_back(std::move(frame)); + } + scheduleDrain(); + } + [[nodiscard]] std::string buildStatusPayload(std::string message) { nlohmann::json requestsJson = nlohmann::json::array(); @@ -366,6 +400,18 @@ class TilesWsSession : public std::enable_shared_from_this }).dump(); } + [[nodiscard]] std::string buildLoadStatePayload(MapTileKey const& key, TileLayer::LoadState state) + { + return nlohmann::json::object({ + {"type", "mapget.tiles.load-state"}, + {"mapId", key.mapId_}, + {"layerId", key.layerId_}, + {"tileId", key.tileId_.value_}, + {"state", static_cast(state)}, + {"stateText", std::string(loadStateToString(state))}, + }).dump(); + } + void scheduleDrain() { if (drainScheduled_.exchange(true)) diff --git a/libs/model/include/mapget/model/layer.h b/libs/model/include/mapget/model/layer.h index 5a2acfa5..0f383fa4 100644 --- a/libs/model/include/mapget/model/layer.h +++ b/libs/model/include/mapget/model/layer.h @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace simfil { struct StringPool; } @@ -85,6 +86,12 @@ class TileLayer { public: using Ptr = std::shared_ptr; + enum class LoadState : uint8_t { + LoadingQueued = 0, + BackendFetching = 1, + BackendConverting = 2 + }; + using LoadStateCallback = std::function; /** * Constructor that takes tileId_, nodeId_, mapId_, layerInfo_, @@ -192,6 +199,15 @@ class TileLayer virtual tl::expected write(std::ostream& outputStream); virtual nlohmann::json toJson() const; + /** + * Set a load-state callback. Used by the service to forward state changes. + * Not serialized with the tile. + */ + void setLoadStateCallback(LoadStateCallback cb); + + /** Emit a load-state change (if a callback is registered). */ + void setLoadState(LoadState state); + protected: Version mapVersion_{0, 0, 0}; TileId tileId_; @@ -204,6 +220,7 @@ class TileLayer std::optional ttl_; nlohmann::json info_; std::optional legalInfo_; // Copyright-related information + LoadStateCallback onLoadStateChanged_; }; } diff --git a/libs/model/include/mapget/model/stream.h b/libs/model/include/mapget/model/stream.h index 5308ba4f..3d9dae76 100644 --- a/libs/model/include/mapget/model/stream.h +++ b/libs/model/include/mapget/model/stream.h @@ -35,13 +35,19 @@ class TileLayerStream * Payload: UTF-8 JSON bytes (not null-terminated). */ Status = 4, + /** + * JSON-encoded load-state updates for individual tiles. + * + * Payload: UTF-8 JSON bytes (not null-terminated). + */ + LoadStateChange = 5, EndOfStream = 128 }; struct StringPoolCache; /** Protocol Version which parsed blobs must be compatible with. */ - static constexpr Version CurrentProtocolVersion{0, 1, 1}; + static constexpr Version CurrentProtocolVersion{1, 0, 0}; /** Map to keep track of the highest sent string id per datasource node. */ using StringPoolOffsetMap = std::unordered_map; diff --git a/libs/model/src/layer.cpp b/libs/model/src/layer.cpp index be94dbac..199851c7 100644 --- a/libs/model/src/layer.cpp +++ b/libs/model/src/layer.cpp @@ -243,6 +243,18 @@ void TileLayer::setLegalInfo(const std::string& legalInfoString) legalInfo_ = legalInfoString; } +void TileLayer::setLoadStateCallback(LoadStateCallback cb) +{ + onLoadStateChanged_ = std::move(cb); +} + +void TileLayer::setLoadState(LoadState state) +{ + if (onLoadStateChanged_) { + onLoadStateChanged_(state); + } +} + tl::expected TileLayer::write(std::ostream& outputStream) { using namespace std::chrono; diff --git a/libs/service/include/mapget/service/datasource.h b/libs/service/include/mapget/service/datasource.h index a76c172e..dadf5ae7 100644 --- a/libs/service/include/mapget/service/datasource.h +++ b/libs/service/include/mapget/service/datasource.h @@ -58,7 +58,11 @@ class DataSource virtual std::vector locate(LocateRequest const& req); /** Called by mapget::Service worker. Dispatches to Cache or fill(...) on miss. */ - virtual TileLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info); + virtual TileLayer::Ptr get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback = {}); /** Add an authorization header-regex pair for this datasource. */ void requireAuthHeaderRegexMatchOption(std::string header, std::regex re); diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index 79c9689e..34789a5d 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -76,8 +76,15 @@ class LayerTilesRequest template LayerTilesRequest& onSourceDataLayer(Fun&& callback) { onSourceDataLayer_ = std::forward(callback); return *this; } + /** + * Callback for per-tile load-state changes. + */ + template + LayerTilesRequest& onLayerLoadStateChanged(Fun&& callback) { onLoadStateChanged_ = std::forward(callback); return *this; } + protected: virtual void notifyResult(TileLayer::Ptr); + void notifyLoadState(MapTileKey const& key, TileLayer::LoadState state); void setStatus(RequestStatus s); void notifyStatus(); nlohmann::json toJson(); @@ -88,6 +95,7 @@ class LayerTilesRequest */ std::function onFeatureLayer_; std::function onSourceDataLayer_; + std::function onLoadStateChanged_; // So the service can track which tileId index from tiles_ // is next in line to be processed. diff --git a/libs/service/src/datasource.cpp b/libs/service/src/datasource.cpp index 8074fcd4..3e7715fe 100644 --- a/libs/service/src/datasource.cpp +++ b/libs/service/src/datasource.cpp @@ -10,7 +10,11 @@ namespace mapget { -TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourceInfo const& info) +TileLayer::Ptr DataSource::get( + const MapTileKey& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback) { auto layerInfo = info.getLayer(k.layerId_); if (!layerInfo) @@ -27,6 +31,9 @@ TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourc info.mapId_, info.getLayer(k.layerId_), cache->getStringPool(info.nodeId_)); + if (loadStateCallback) { + tileFeatureLayer->setLoadStateCallback(loadStateCallback); + } fill(tileFeatureLayer); result = tileFeatureLayer; break; @@ -38,6 +45,9 @@ TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourc info.mapId_, info.getLayer(k.layerId_), cache->getStringPool(info.nodeId_)); + if (loadStateCallback) { + tileSourceDataLayer->setLoadStateCallback(loadStateCallback); + } fill(tileSourceDataLayer); result = tileSourceDataLayer; break; diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index 4abbf9bd..4c1c5678 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -62,6 +62,13 @@ void LayerTilesRequest::notifyResult(TileLayer::Ptr r) { } } +void LayerTilesRequest::notifyLoadState(MapTileKey const& key, TileLayer::LoadState state) +{ + if (onLoadStateChanged_) { + onLoadStateChanged_(key, state); + } +} + void LayerTilesRequest::setStatus(RequestStatus s) { { @@ -188,6 +195,7 @@ struct Service::Controller // Enter into the jobs-in-progress set. jobsInProgress_.insert(result->tileKey); + request->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); // Move this request to the end of the list, so others gain priority. requests_.splice(requests_.end(), requests_, reqIt); @@ -262,7 +270,14 @@ struct Service::Worker dataSource_->onCacheExpired(job.tileKey, *job.cacheExpiredAt); } - auto layer = dataSource_->get(job.tileKey, controller_.cache_, info_); + job.request->notifyLoadState(job.tileKey, TileLayer::LoadState::BackendFetching); + auto layer = dataSource_->get( + job.tileKey, + controller_.cache_, + info_, + [request = job.request, tileKey = job.tileKey](TileLayer::LoadState state) { + request->notifyLoadState(tileKey, state); + }); if (!layer) raise("DataSource::get() returned null."); From 87acbb451ab509820a3771b921e7b0e4d6220832 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 29 Jan 2026 17:19:49 +0100 Subject: [PATCH 12/14] Ensure that work on a tile always benefit all interested requests immediately. Bump protocol version. --- libs/model/include/mapget/model/stream.h | 16 +- libs/service/include/mapget/service/service.h | 4 + libs/service/src/service.cpp | 188 +++++++++++------- 3 files changed, 133 insertions(+), 75 deletions(-) diff --git a/libs/model/include/mapget/model/stream.h b/libs/model/include/mapget/model/stream.h index 3d9dae76..5eb0a5a5 100644 --- a/libs/model/include/mapget/model/stream.h +++ b/libs/model/include/mapget/model/stream.h @@ -46,8 +46,20 @@ class TileLayerStream struct StringPoolCache; - /** Protocol Version which parsed blobs must be compatible with. */ - static constexpr Version CurrentProtocolVersion{1, 0, 0}; + /** + * Protocol Version which parsed blobs must be compatible with. + * Version History: + * - Version 1.0: + * + Added TileFeatureLayer Message + * + Added StringPool Message + * + Added TileSourceDataLayer Message + * + Added EndOfStream Message + * - Version 1.1: + * + Added errorCode field to TileLayer + * + Added Status Message + * + Added LoadStateChange Message + */ + static constexpr Version CurrentProtocolVersion{1, 1, 0}; /** Map to keep track of the highest sent string id per datasource node. */ using StringPoolOffsetMap = std::unordered_map; diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index 34789a5d..c772dd33 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace mapget { @@ -101,6 +102,9 @@ class LayerTilesRequest // is next in line to be processed. size_t nextTileIndex_ = 0; + // Track which tiles still need to be scheduled/served for this request. + std::set tileIdsNotDone_; + // So the requester can track how many results have been received. size_t resultCount_ = 0; diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index 4c1c5678..d645691f 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -11,13 +11,11 @@ #include #include -#include #include #include #include #include #include -#include #include #include "simfil/types.h" @@ -33,6 +31,7 @@ LayerTilesRequest::LayerTilesRequest( layerId_(std::move(layerId)), tiles_(std::move(tiles)) { + tileIdsNotDone_.insert(tiles_.begin(), tiles_.end()); if (tiles_.empty()) { // An empty request is always set to success, but the client/service // is responsible for triggering notifyStatus() in that case. @@ -43,16 +42,16 @@ LayerTilesRequest::LayerTilesRequest( void LayerTilesRequest::notifyResult(TileLayer::Ptr r) { const auto type = r->layerInfo()->type_; switch (type) { - case mapget::LayerType::Features: + case LayerType::Features: if (onFeatureLayer_) - onFeatureLayer_(std::move(std::static_pointer_cast(r))); + onFeatureLayer_(std::move(std::static_pointer_cast(r))); break; - case mapget::LayerType::SourceData: + case LayerType::SourceData: if (onSourceDataLayer_) - onSourceDataLayer_(std::move(std::static_pointer_cast(r))); + onSourceDataLayer_(std::move(std::static_pointer_cast(r))); break; default: - mapget::log().error(fmt::format("Unhandled layer type {}, no matching callback!", static_cast(type))); + log().error(fmt::format("Unhandled layer type {}, no matching callback!", static_cast(type))); break; } @@ -121,13 +120,15 @@ bool LayerTilesRequest::isDone() struct Service::Controller { + virtual ~Controller() = default; + struct Job { MapTileKey tileKey; - LayerTilesRequest::Ptr request; + std::vector waitingRequests; std::optional cacheExpiredAt; }; - std::set jobsInProgress_; // Set of jobs currently in progress + std::map> jobsInProgress_; // Jobs currently in progress + interested requests Cache::Ptr cache_; // The cache for the service std::optional defaultTtl_; // Default TTL applied when datasource does not override std::list requests_; // List of requests currently being processed @@ -136,79 +137,101 @@ struct Service::Controller explicit Controller(Cache::Ptr cache, std::optional defaultTtl) : cache_(std::move(cache)), - defaultTtl_(std::move(defaultTtl)) + defaultTtl_(defaultTtl) { if (!cache_) raise("Cache must not be null!"); } - std::optional nextJob(DataSourceInfo const& i) + std::shared_ptr nextJob(DataSourceInfo const& i) { // Workers call the nextJob function when they are free. // Note: For thread safety, jobsMutex_ must be held // when calling this function. - std::optional result; + std::shared_ptr result; // Return next job, if available. - bool cachedTilesServed = false; + bool cachedTilesServedOrInProgressSkipped = false; + bool anyTasksRemaining = false; do { - cachedTilesServed = false; + cachedTilesServedOrInProgressSkipped = false; + anyTasksRemaining = false; for (auto reqIt = requests_.begin(); reqIt != requests_.end(); ++reqIt) { auto const& request = *reqIt; auto layerIt = i.layers_.find(request->layerId_); - // Are there tiles left to be processed in the request? - if (request->mapId_ == i.mapId_ && layerIt != i.layers_.end()) { - if (request->nextTileIndex_ >= request->tiles_.size()) { - continue; + // Does the Datasource Info (i) of the worker fit the request? + if (request->mapId_ != i.mapId_ || layerIt == i.layers_.end()) + continue; + + // Find the next pending tile in the request's ordered list. + TileId tileId{}; + bool foundTile = false; + while (request->nextTileIndex_ < request->tiles_.size()) { + tileId = request->tiles_[request->nextTileIndex_++]; + if (request->tileIdsNotDone_.find(tileId) != request->tileIdsNotDone_.end()) { + foundTile = true; + break; } + } + if (!foundTile) + continue; + anyTasksRemaining = true; + auto resultTileKey = MapTileKey(layerIt->second->type_, request->mapId_, request->layerId_, tileId); + + // Cache lookup. + auto cachedResult = cache_->getTileLayer(resultTileKey, i); + if (cachedResult.tile) { + request->tileIdsNotDone_.erase(tileId); + log().debug("Serving cached tile: {}", resultTileKey.toString()); + request->notifyResult(cachedResult.tile); + cachedTilesServedOrInProgressSkipped = true; + continue; + } + + // If another worker is working on this tile, ensure that this request gets it as well. + if (auto inProgress = jobsInProgress_.find(resultTileKey); + inProgress != jobsInProgress_.end()) { + // This tile is already being processed. Register interest so the result + // can satisfy multiple requests, and allow this request to advance. + log().debug("Joining tile with job in progress: {}", + resultTileKey.toString()); + request->tileIdsNotDone_.erase(tileId); + inProgress->second->waitingRequests.push_back(request); + cachedTilesServedOrInProgressSkipped = true; + continue; + } - // Create result wrapper object. - auto tileId = request->tiles_[request->nextTileIndex_++]; - result = Job{MapTileKey(), request, std::nullopt}; - result->tileKey.layer_ = layerIt->second->type_; - result->tileKey.mapId_ = request->mapId_; - result->tileKey.layerId_ = request->layerId_; - result->tileKey.tileId_ = tileId; - - // Cache lookup. - auto cachedResult = cache_->getTileLayer(result->tileKey, i); - if (cachedResult.tile) { - log().debug("Serving cached tile: {}", result->tileKey.toString()); - request->notifyResult(cachedResult.tile); - result.reset(); - cachedTilesServed = true; + // We found something to work on that is not cached and not in progress - + // enter it into the jobs-in-progress map with the requesting client. + request->tileIdsNotDone_.erase(tileId); + result = std::make_shared(Job{resultTileKey, {request}, cachedResult.expiredAt}); + // Proactively attach other requests that need this tile. + for (auto const& otherRequest : requests_) { + if (!otherRequest || otherRequest == request) continue; - } - result->cacheExpiredAt = cachedResult.expiredAt; - - if (jobsInProgress_.find(result->tileKey) != jobsInProgress_.end()) { - // Don't work on something that is already being worked on. - // Wait for the work to finish, then send the (hopefully cached) result. - log().debug("Delaying tile with job in progress: {}", - result->tileKey.toString()); - --request->nextTileIndex_; - result.reset(); + if (otherRequest->mapId_ != request->mapId_ || otherRequest->layerId_ != request->layerId_) continue; - } - - // Enter into the jobs-in-progress set. - jobsInProgress_.insert(result->tileKey); - request->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); + if (otherRequest->tileIdsNotDone_.erase(tileId) == 0) + continue; + result->waitingRequests.push_back(otherRequest); + otherRequest->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); + } + jobsInProgress_.emplace(result->tileKey, result); + request->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); - // Move this request to the end of the list, so others gain priority. - requests_.splice(requests_.end(), requests_, reqIt); + // Move this request to the end of the list, so others gain priority. + requests_.splice(requests_.end(), requests_, reqIt); - log().debug("Working on tile: {}", result->tileKey.toString()); - break; - } + log().debug("Working on tile: {}", result->tileKey.toString()); + break; } } - while (cachedTilesServed && !result); + while (cachedTilesServedOrInProgressSkipped && !result && anyTasksRemaining); // Clean up done requests. - requests_.remove_if([](auto&& r) {return r->nextTileIndex_ == r->tiles_.size(); }); + requests_.remove_if([](auto&& r) {return r->tileIdsNotDone_.empty(); }); return result; } @@ -239,7 +262,7 @@ struct Service::Worker bool work() { - std::optional nextJob; + std::shared_ptr nextJob; { std::unique_lock lock(controller_.jobsMutex_); @@ -255,7 +278,7 @@ struct Service::Worker return true; } nextJob = controller_.nextJob(info_); - return nextJob.has_value(); + return !!nextJob; }); } @@ -270,13 +293,26 @@ struct Service::Worker dataSource_->onCacheExpired(job.tileKey, *job.cacheExpiredAt); } - job.request->notifyLoadState(job.tileKey, TileLayer::LoadState::BackendFetching); + auto notifyWaitingRequests = [&](TileLayer::LoadState state) { + std::vector waiting; + { + std::unique_lock lock(controller_.jobsMutex_); + waiting = job.waitingRequests; + } + for (auto const& req : waiting) { + if (req) { + req->notifyLoadState(job.tileKey, state); + } + } + }; + + notifyWaitingRequests(TileLayer::LoadState::BackendFetching); auto layer = dataSource_->get( job.tileKey, controller_.cache_, info_, - [request = job.request, tileKey = job.tileKey](TileLayer::LoadState state) { - request->notifyLoadState(tileKey, state); + [¬ifyWaitingRequests](TileLayer::LoadState state) { + notifyWaitingRequests(state); }); if (!layer) raise("DataSource::get() returned null."); @@ -299,14 +335,20 @@ struct Service::Worker controller_.cache_->putTileLayer(layer); + std::vector notifyRequests; { std::unique_lock lock(controller_.jobsMutex_); controller_.jobsInProgress_.erase(job.tileKey); - job.request->notifyResult(layer); - // As we entered a tile into the cache, notify other workers - // that this tile can be served. - controller_.jobsAvailable_.notify_all(); + notifyRequests = job.waitingRequests; + } + for (auto const& req : notifyRequests) { + if (req) { + req->notifyResult(layer); + } } + // As we entered a tile into the cache, notify other workers + // that this tile can be served. + controller_.jobsAvailable_.notify_all(); } catch (std::exception& e) { log().error("Could not load tile {}: {}", @@ -331,7 +373,7 @@ struct Service::Impl : public Service::Controller Cache::Ptr cache, bool useDataSourceConfig, std::optional defaultTtl) - : Controller(std::move(cache), std::move(defaultTtl)) + : Controller(std::move(cache), defaultTtl) { if (!useDataSourceConfig) return; @@ -361,7 +403,7 @@ struct Service::Impl : public Service::Controller }); } - ~Impl() + ~Impl() override { // Ensure that no new datasources are added while we are cleaning up. configSubscription_.reset(); @@ -490,21 +532,21 @@ struct Service::Impl : public Service::Controller if (auxDataSource->info().mapId_ == baseTile->mapId()) { auto auxTile = [&]() -> TileFeatureLayer::Ptr { - auto auxTile = auxDataSource->get(baseTile->id(), cache_, auxDataSource->info()); - if (!auxTile) { + auto result = auxDataSource->get(baseTile->id(), cache_, auxDataSource->info()); + if (!result) { log().warn("auxDataSource returned null for {}", baseTile->id().toString()); return {}; } - if (auxTile->error()) { - log().warn("Error while fetching addon tile {}: {}", baseTile->id().toString(), *auxTile->error()); + if (result->error()) { + log().warn("Error while fetching addon tile {}: {}", baseTile->id().toString(), *result->error()); return {}; } - if (auxTile->layerInfo()->type_ != LayerType::Features) { + if (result->layerInfo()->type_ != LayerType::Features) { log().warn("Addon tile is not a feature layer"); return {}; } - return std::static_pointer_cast(auxTile); + return std::static_pointer_cast(result); }(); if (!auxTile) { @@ -578,7 +620,7 @@ struct Service::Impl : public Service::Controller }; Service::Service(Cache::Ptr cache, bool useDataSourceConfig, std::optional defaultTtl) - : impl_(std::make_unique(std::move(cache), useDataSourceConfig, std::move(defaultTtl))) + : impl_(std::make_unique(std::move(cache), useDataSourceConfig, defaultTtl)) { } From ceeb4aa3809bad4fdbe8c2a96b3623e47c63358d Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 29 Jan 2026 18:17:18 +0100 Subject: [PATCH 13/14] Ensure that nextJob does not lock until a to-be-aborted request has been fully served from cache. --- libs/service/include/mapget/service/service.h | 2 +- libs/service/src/service.cpp | 32 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index c772dd33..edebab07 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -111,7 +111,7 @@ class LayerTilesRequest // Mutex/condition variable for reading/setting request status. std::mutex statusMutex_; std::condition_variable statusConditionVariable_; - RequestStatus status_ = RequestStatus::Open; + std::atomic status_ = RequestStatus::Open; }; /** diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index d645691f..c8490b19 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -40,6 +40,10 @@ LayerTilesRequest::LayerTilesRequest( } void LayerTilesRequest::notifyResult(TileLayer::Ptr r) { + if (isDone()) { + return; + } + const auto type = r->layerInfo()->type_; switch (type) { case LayerType::Features: @@ -70,10 +74,7 @@ void LayerTilesRequest::notifyLoadState(MapTileKey const& key, TileLayer::LoadSt void LayerTilesRequest::setStatus(RequestStatus s) { - { - std::unique_lock statusLock(statusMutex_); - this->status_ = s; - } + this->status_ = s; notifyStatus(); } @@ -162,13 +163,15 @@ struct Service::Controller auto layerIt = i.layers_.find(request->layerId_); // Does the Datasource Info (i) of the worker fit the request? - if (request->mapId_ != i.mapId_ || layerIt == i.layers_.end()) + // Or is it done (/aborted) but not yet removed from requests? + if (request->mapId_ != i.mapId_ || layerIt == i.layers_.end() || request->isDone()) continue; // Find the next pending tile in the request's ordered list. TileId tileId{}; bool foundTile = false; while (request->nextTileIndex_ < request->tiles_.size()) { + // Skip over tiles which were meanwhile done by other workers. tileId = request->tiles_[request->nextTileIndex_++]; if (request->tileIdsNotDone_.find(tileId) != request->tileIdsNotDone_.end()) { foundTile = true; @@ -222,11 +225,17 @@ struct Service::Controller request->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); // Move this request to the end of the list, so others gain priority. + // It is ok to manipulate the list here, because we call `break` after the next line. requests_.splice(requests_.end(), requests_, reqIt); log().debug("Working on tile: {}", result->tileKey.toString()); break; } + + // Once unlock and re-lock before we make another sweep over the request list, + // so that it can be updated externally; clients might want to add/remove requests. + jobsMutex_.unlock(); + jobsMutex_.lock(); } while (cachedTilesServedOrInProgressSkipped && !result && anyTasksRemaining); @@ -506,12 +515,13 @@ struct Service::Impl : public Service::Controller void abortRequest(LayerTilesRequest::Ptr const& r) { - std::unique_lock lock(jobsMutex_); - // Remove the request from the list of requests. - auto numRemoved = requests_.remove_if([r](auto&& request) { return r == request; }); - // Clear its jobs to mark it as done. - if (numRemoved) { - r->setStatus(RequestStatus::Aborted); + // Mark the request as aborted. + r->setStatus(RequestStatus::Aborted); + + // Remove the request from the list of requests (needs lock). + { + std::unique_lock lock(jobsMutex_); + requests_.remove_if([r](auto&& request) { return r == request; }); } } From df7b042fe15436fe71978f01f523badecec22955 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 2 Feb 2026 19:30:46 +0100 Subject: [PATCH 14/14] Use more human-readable stat key names. --- libs/model/src/featurelayer.cpp | 2 +- libs/service/include/mapget/service/service.h | 4 +- libs/service/src/datasource.cpp | 2 +- libs/service/src/service.cpp | 39 ++++++++++--------- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/libs/model/src/featurelayer.cpp b/libs/model/src/featurelayer.cpp index cf891b66..e5cb6a56 100644 --- a/libs/model/src/featurelayer.cpp +++ b/libs/model/src/featurelayer.cpp @@ -336,7 +336,7 @@ simfil::model_ptr TileFeatureLayer::newFeature( // contains only references to feature nodes, in the order // of the feature node column. addRoot(simfil::ModelNode::Ptr(result)); - setInfo("num-features", numRoots()); + setInfo("Size/Features", numRoots()); return result; } diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index edebab07..cf1cb085 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -85,7 +85,7 @@ class LayerTilesRequest protected: virtual void notifyResult(TileLayer::Ptr); - void notifyLoadState(MapTileKey const& key, TileLayer::LoadState state); + void notifyLoadState(MapTileKey const& key, TileLayer::LoadState state) const; void setStatus(RequestStatus s); void notifyStatus(); nlohmann::json toJson(); @@ -103,7 +103,7 @@ class LayerTilesRequest size_t nextTileIndex_ = 0; // Track which tiles still need to be scheduled/served for this request. - std::set tileIdsNotDone_; + std::set tileIdsNotStarted_; // So the requester can track how many results have been received. size_t resultCount_ = 0; diff --git a/libs/service/src/datasource.cpp b/libs/service/src/datasource.cpp index 3e7715fe..319c77c0 100644 --- a/libs/service/src/datasource.cpp +++ b/libs/service/src/datasource.cpp @@ -59,7 +59,7 @@ TileLayer::Ptr DataSource::get( // Notify the tile how long it took to fill. if (result) { auto duration = std::chrono::steady_clock::now() - start; - result->setInfo("fill-time-ms", std::chrono::duration_cast(duration).count()); + result->setInfo("Load+Convert/Total#ms", std::chrono::duration_cast(duration).count()); } return result; } diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index c8490b19..aa066c65 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -31,7 +31,7 @@ LayerTilesRequest::LayerTilesRequest( layerId_(std::move(layerId)), tiles_(std::move(tiles)) { - tileIdsNotDone_.insert(tiles_.begin(), tiles_.end()); + tileIdsNotStarted_.insert(tiles_.begin(), tiles_.end()); if (tiles_.empty()) { // An empty request is always set to success, but the client/service // is responsible for triggering notifyStatus() in that case. @@ -65,8 +65,7 @@ void LayerTilesRequest::notifyResult(TileLayer::Ptr r) { } } -void LayerTilesRequest::notifyLoadState(MapTileKey const& key, TileLayer::LoadState state) -{ +void LayerTilesRequest::notifyLoadState(MapTileKey const& key, TileLayer::LoadState state) const { if (onLoadStateChanged_) { onLoadStateChanged_(key, state); } @@ -127,6 +126,7 @@ struct Service::Controller MapTileKey tileKey; std::vector waitingRequests; std::optional cacheExpiredAt; + TileLayer::LoadState loadStatus = TileLayer::LoadState::LoadingQueued; }; std::map> jobsInProgress_; // Jobs currently in progress + interested requests @@ -144,11 +144,12 @@ struct Service::Controller raise("Cache must not be null!"); } - std::shared_ptr nextJob(DataSourceInfo const& i) + std::shared_ptr nextJob(DataSourceInfo const& i, std::unique_lock& lock) { // Workers call the nextJob function when they are free. // Note: For thread safety, jobsMutex_ must be held - // when calling this function. + // when calling this function. The lock may be released/re-acquired + // between sweeps to allow external updates. std::shared_ptr result; @@ -173,7 +174,7 @@ struct Service::Controller while (request->nextTileIndex_ < request->tiles_.size()) { // Skip over tiles which were meanwhile done by other workers. tileId = request->tiles_[request->nextTileIndex_++]; - if (request->tileIdsNotDone_.find(tileId) != request->tileIdsNotDone_.end()) { + if (request->tileIdsNotStarted_.find(tileId) != request->tileIdsNotStarted_.end()) { foundTile = true; break; } @@ -186,7 +187,7 @@ struct Service::Controller // Cache lookup. auto cachedResult = cache_->getTileLayer(resultTileKey, i); if (cachedResult.tile) { - request->tileIdsNotDone_.erase(tileId); + request->tileIdsNotStarted_.erase(tileId); log().debug("Serving cached tile: {}", resultTileKey.toString()); request->notifyResult(cachedResult.tile); cachedTilesServedOrInProgressSkipped = true; @@ -200,7 +201,8 @@ struct Service::Controller // can satisfy multiple requests, and allow this request to advance. log().debug("Joining tile with job in progress: {}", resultTileKey.toString()); - request->tileIdsNotDone_.erase(tileId); + request->tileIdsNotStarted_.erase(tileId); + request->notifyLoadState(resultTileKey, inProgress->second->loadStatus); inProgress->second->waitingRequests.push_back(request); cachedTilesServedOrInProgressSkipped = true; continue; @@ -208,7 +210,7 @@ struct Service::Controller // We found something to work on that is not cached and not in progress - // enter it into the jobs-in-progress map with the requesting client. - request->tileIdsNotDone_.erase(tileId); + request->tileIdsNotStarted_.erase(tileId); result = std::make_shared(Job{resultTileKey, {request}, cachedResult.expiredAt}); // Proactively attach other requests that need this tile. for (auto const& otherRequest : requests_) { @@ -216,13 +218,11 @@ struct Service::Controller continue; if (otherRequest->mapId_ != request->mapId_ || otherRequest->layerId_ != request->layerId_) continue; - if (otherRequest->tileIdsNotDone_.erase(tileId) == 0) + if (otherRequest->tileIdsNotStarted_.erase(tileId) == 0) continue; result->waitingRequests.push_back(otherRequest); - otherRequest->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); } jobsInProgress_.emplace(result->tileKey, result); - request->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); // Move this request to the end of the list, so others gain priority. // It is ok to manipulate the list here, because we call `break` after the next line. @@ -232,15 +232,17 @@ struct Service::Controller break; } - // Once unlock and re-lock before we make another sweep over the request list, - // so that it can be updated externally; clients might want to add/remove requests. - jobsMutex_.unlock(); - jobsMutex_.lock(); + if (cachedTilesServedOrInProgressSkipped && !result && anyTasksRemaining) { + // Unlock and re-lock before we make another sweep over the request list, + // so that it can be updated externally; clients might want to add/remove requests. + lock.unlock(); + lock.lock(); + } } while (cachedTilesServedOrInProgressSkipped && !result && anyTasksRemaining); // Clean up done requests. - requests_.remove_if([](auto&& r) {return r->tileIdsNotDone_.empty(); }); + requests_.remove_if([](auto&& r) {return r->tileIdsNotStarted_.empty(); }); return result; } @@ -286,7 +288,7 @@ struct Service::Worker // is removed. All worker instances are expected to terminate. return true; } - nextJob = controller_.nextJob(info_); + nextJob = controller_.nextJob(info_, lock); return !!nextJob; }); } @@ -306,6 +308,7 @@ struct Service::Worker std::vector waiting; { std::unique_lock lock(controller_.jobsMutex_); + job.loadStatus = state; waiting = job.waitingRequests; } for (auto const& req : waiting) {