diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d73bb5b..f97065b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,10 @@ endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) +if (WIN32) + add_compile_definitions(NOMINMAX) +endif() + include(FetchContent) include(GNUInstallDirs) @@ -22,12 +26,21 @@ if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) set(MAPGET_WITH_HTTPLIB ON CACHE BOOL "Enable mapget-http-datasource and mapget-http-service libraries.") set(MAPGET_ENABLE_TESTING ON CACHE BOOL "Enable testing.") set(MAPGET_BUILD_EXAMPLES ON CACHE BOOL "Build examples.") + # Prevent CPM dependencies (e.g. Drogon/Trantor) from registering their own + # tests into this project's ctest run. + set(BUILD_TESTING OFF CACHE BOOL "Disable dependency tests" FORCE) endif() option(MAPGET_WITH_WHEEL "Enable mapget Python wheel (output to WHEEL_DEPLOY_DIRECTORY).") option(MAPGET_WITH_SERVICE "Enable mapget-service library. Requires threads.") option(MAPGET_WITH_HTTPLIB "Enable mapget-http-datasource and mapget-http-service libraries.") +if (MAPGET_ENABLE_TESTING) + # Enable testing before adding CPM dependencies so stale/third-party CTest + # files don't linger in the build tree. + enable_testing() +endif() + set(Python3_FIND_STRATEGY LOCATION) if (NOT MSVC) @@ -112,7 +125,6 @@ endif() # tests if (MAPGET_ENABLE_TESTING) - enable_testing() add_subdirectory(test/unit) if (MAPGET_WITH_WHEEL) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 7473f2b9..323dc283 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -15,7 +15,7 @@ CPMAddPackage( "EXPECTED_BUILD_TESTS OFF" "EXPECTED_BUILD_PACKAGE_DEB OFF") CPMAddPackage( - URI "gh:Klebert-Engineering/simfil@0.6.2" + URI "gh:Klebert-Engineering/simfil@0.6.3#v0.6.3" OPTIONS "SIMFIL_WITH_MODEL_JSON ON" "SIMFIL_SHARED OFF") @@ -25,6 +25,21 @@ CPMAddPackage( "BUILD_TESTING OFF") if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) + # OpenSSL's Configure script needs a "full" Perl distribution. Git for + # Windows ships a minimal perl that is missing required modules (e.g. + # Locale::Maketext::Simple), causing OpenSSL builds to fail. + if (WIN32) + if (NOT DEFINED PERL_EXECUTABLE OR PERL_EXECUTABLE MATCHES "[\\\\/]Git[\\\\/]usr[\\\\/]bin[\\\\/]perl\\.exe$") + find_program(_MAPGET_STRAWBERRY_PERL + NAMES perl.exe + PATHS "C:/Strawberry/perl/bin" + NO_DEFAULT_PATH) + if (_MAPGET_STRAWBERRY_PERL) + set(PERL_EXECUTABLE "${_MAPGET_STRAWBERRY_PERL}" CACHE FILEPATH "" FORCE) + endif() + endif() + endif() + set (OPENSSL_VERSION openssl-3.5.2) CPMAddPackage("gh:klebert-engineering/openssl-cmake@1.0.0") CPMAddPackage( @@ -40,19 +55,67 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) endif() CPMAddPackage( - URI "gh:yhirose/cpp-httplib@0.15.3" + NAME jsoncpp + GIT_REPOSITORY https://github.com/open-source-parsers/jsoncpp + GIT_TAG 1.9.5 + GIT_SHALLOW ON + OPTIONS + "JSONCPP_WITH_TESTS OFF" + "JSONCPP_WITH_POST_BUILD_UNITTEST OFF" + "JSONCPP_WITH_PKGCONFIG_SUPPORT OFF" + "JSONCPP_WITH_CMAKE_PACKAGE OFF" + "BUILD_SHARED_LIBS OFF" + "BUILD_STATIC_LIBS ON" + "BUILD_OBJECT_LIBS OFF") + # Help Drogon's FindJsoncpp.cmake locate jsoncpp when built via CPM. + set(JSONCPP_INCLUDE_DIRS "${jsoncpp_SOURCE_DIR}/include" CACHE PATH "" FORCE) + set(JSONCPP_LIBRARIES jsoncpp_static CACHE STRING "" FORCE) + # CPM generates a dummy package redirect config at + # `${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/jsoncpp-config.cmake`. Drogon uses + # `find_package(Jsoncpp)` (config-first), so make that redirect actually + # define the expected `Jsoncpp_lib` target. + if (DEFINED CMAKE_FIND_PACKAGE_REDIRECTS_DIR) + file(MAKE_DIRECTORY "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}") + file(WRITE "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/jsoncpp-extra.cmake" [=[ +if(NOT TARGET Jsoncpp_lib) + add_library(Jsoncpp_lib INTERFACE) + target_include_directories(Jsoncpp_lib INTERFACE "${JSONCPP_INCLUDE_DIRS}") + target_link_libraries(Jsoncpp_lib INTERFACE ${JSONCPP_LIBRARIES}) +endif() +]=]) + endif() + + # Drogon defines install(EXPORT ...) rules unconditionally, which fail when + # used as a subproject with CPM-provided dependencies (zlib/jsoncpp/etc). + # Since mapget only needs Drogon for building, temporarily suppress install + # rule generation while configuring Drogon. + set(_MAPGET_PREV_SKIP_INSTALL_RULES "${CMAKE_SKIP_INSTALL_RULES}") + if (DEFINED BUILD_TESTING) + set(_MAPGET_PREV_BUILD_TESTING "${BUILD_TESTING}") + endif() + set(CMAKE_SKIP_INSTALL_RULES ON) + set(BUILD_TESTING OFF) + + CPMAddPackage( + URI "gh:drogonframework/drogon@1.9.7" OPTIONS - "CPPHTTPLIB_USE_POLL ON" - "HTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN OFF" - "HTTPLIB_INSTALL OFF" - "HTTPLIB_USE_OPENSSL_IF_AVAILABLE OFF" - "HTTPLIB_USE_ZLIB_IF_AVAILABLE OFF") - # Manually enable openssl/zlib in httplib to avoid FindPackage calls. - target_compile_definitions(httplib INTERFACE - CPPHTTPLIB_OPENSSL_SUPPORT - CPPHTTPLIB_ZLIB_SUPPORT) - target_link_libraries(httplib INTERFACE - OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB) + "BUILD_CTL OFF" + "BUILD_EXAMPLES OFF" + "BUILD_ORM OFF" + "BUILD_BROTLI OFF" + "BUILD_YAML_CONFIG OFF" + "BUILD_SHARED_LIBS OFF" + "USE_SUBMODULE ON" + "USE_STATIC_LIBS_ONLY OFF" + "USE_POSTGRESQL OFF" + "USE_MYSQL OFF" + "USE_SQLITE3 OFF" + GIT_SUBMODULES "trantor") + + set(CMAKE_SKIP_INSTALL_RULES "${_MAPGET_PREV_SKIP_INSTALL_RULES}") + if (DEFINED _MAPGET_PREV_BUILD_TESTING) + set(BUILD_TESTING "${_MAPGET_PREV_BUILD_TESTING}") + endif() CPMAddPackage( URI "gh:jbeder/yaml-cpp#aa8d4e@0.8.0" # Use > 0.8.0 once available. @@ -64,6 +127,7 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) CPMAddPackage("gh:CLIUtils/CLI11@2.5.0") CPMAddPackage("gh:pboettch/json-schema-validator#2.3.0") CPMAddPackage("gh:okdshin/PicoSHA2@1.0.1") + endif () if (MAPGET_WITH_WHEEL AND NOT TARGET pybind11) @@ -76,7 +140,7 @@ if (MAPGET_WITH_SERVICE OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) endif() if (MAPGET_WITH_WHEEL AND NOT TARGET python-cmake-wheel) - CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel@1.1.0") + CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel#80592e483cc2be044f64e35c4686a00a9126abd2@1.2.1") endif() if (MAPGET_ENABLE_TESTING) diff --git a/docs/mapget-api.md b/docs/mapget-api.md index e4e7ce11..18dffb33 100644 --- a/docs/mapget-api.md +++ b/docs/mapget-api.md @@ -1,12 +1,12 @@ -# REST/GeoJSON API Guide +# HTTP / WebSocket API Guide -Mapget exposes a small HTTP API that lets clients discover datasources, stream tiles, abort long‑running requests, locate features by ID and inspect or update the running configuration. This guide describes the endpoints and their request and response formats. +Mapget exposes a small HTTP + WebSocket API that lets clients discover datasources, stream tiles, locate features by ID and inspect or update the running configuration. This guide describes the endpoints and their request and response formats. ## Base URL and formats The server started by `mapget serve` listens on the configured host and port (by default on all interfaces and an automatically chosen port). All endpoints are rooted at that host and port. -Requests that send JSON use `Content-Type: application/json`. Tile streaming supports two response encodings, selected via the `Accept` header: +Requests that send JSON use `Content-Type: application/json`. HTTP tile streaming supports two response encodings, selected via the `Accept` header: - `Accept: application/jsonl` returns a JSON‑Lines stream where each line is one JSON object. - `Accept: application/binary` returns a compact binary stream optimized for high-volume traffic. @@ -23,9 +23,9 @@ The binary format and the logical feature model are described in more detail in Each item contains map ID, available layers and basic metadata. This endpoint is typically used by frontends to discover which maps and layers can be requested via `/tiles`. -## `/tiles` – stream tiles +## `/tiles` – stream tiles (HTTP) -`POST /tiles` streams tiles for one or more map–layer combinations. It is the main data retrieval endpoint used by clients such as erdblick. +`POST /tiles` streams tiles for one or more map–layer combinations. - **Method:** `POST` - **Request body (JSON):** @@ -34,7 +34,6 @@ Each item contains map ID, available layers and basic metadata. This endpoint is - `layerId`: string, ID of the layer within that map. - `tileIds`: array of numeric tile IDs in mapget’s tiling scheme. - `stringPoolOffsets` (optional): dictionary from datasource node ID to last known string ID. Used by advanced clients to avoid receiving the same field names repeatedly in the binary stream. - - `clientId` (optional): arbitrary string identifying this client connection for abort handling. - **Response:** - `application/jsonl` if `Accept: application/jsonl` is sent. - `application/binary` if `Accept: application/binary` is sent, using the tile stream protocol. @@ -43,6 +42,21 @@ Tiles are streamed as they become available. In JSONL mode, each line is the JSO If `Accept-Encoding: gzip` is set, the server compresses responses where possible, which is especially useful for JSONL streams. +To cancel an in-flight HTTP stream, close the HTTP connection. + +## `/tiles` – stream tiles (WebSocket) + +`GET /tiles` supports WebSocket upgrades. This is the preferred tile streaming mode for interactive clients because it supports long-lived connections and request replacement without introducing an extra abort endpoint. + +- **Connect:** `ws://:/tiles` +- **Client → Server:** send one *text* message containing the same JSON body as for `POST /tiles` (`requests`, optional `stringPoolOffsets`). + - `stringPoolOffsets` is optional; the server remembers the latest offsets per WebSocket connection. Clients may re-send it to reset/resync offsets. +- **Server → Client:** sends only *binary* WebSocket messages. Each WebSocket message contains exactly one `TileLayerStream` VTLV frame. + - `StringPool`, `TileFeatureLayer`, `TileSourceDataLayer` are unchanged. + - `Status` frames contain UTF-8 JSON payload describing per-request `RequestStatus` transitions and a human-readable message. The final status frame has `"allDone": true`. + +To cancel, either send a new request message on the same connection (which replaces the current one) or close the WebSocket connection. + ### Why JSONL instead of JSON? JSON Lines is better suited to streaming large responses than a single JSON array. Clients can start processing the first tiles immediately, do not need to buffer the complete response in memory, and can naturally consume the stream with incremental parsers. @@ -87,16 +101,6 @@ Each line in the JSONL response is a GeoJSON-like FeatureCollection with additio The `error` object is only present if an error occurred while filling the tile. When present, the `features` array may be empty or contain partial data. -## `/abort` – cancel tile streaming - -`POST /abort` cancels a running `/tiles` request that was started with a matching `clientId`. It is useful when the viewport changes and the previous stream should be abandoned. - -- **Method:** `POST` -- **Request body (JSON):** `{ "clientId": "" }` -- **Response:** `text/plain` confirmation; a 400 status code if `clientId` is missing. - -Internally the service marks the matching tile requests as aborted and stops scheduling further work for them. - ### Curl Call Example For example, the following curl call could be used to stream GeoJSON feature objects diff --git a/docs/mapget-dev-guide.md b/docs/mapget-dev-guide.md index d7c9b905..3d95b117 100644 --- a/docs/mapget-dev-guide.md +++ b/docs/mapget-dev-guide.md @@ -150,7 +150,7 @@ sequenceDiagram participant Ds as DataSource participant Cache - Client->>Http: POST /tiles
requests, clientId + Client->>Http: POST /tiles
requests Http->>Service: request(requests, headers) Service->>Worker: enqueue jobs per datasource loop per tile @@ -169,30 +169,29 @@ sequenceDiagram Service-->>Http: request complete ``` -If a client supplies a `clientId` in the `/tiles` request, the HTTP layer uses it to track open requests and to implement `/abort`. +For interactive clients, tile streaming can also be done via WebSocket `GET /tiles`, where sending a new request message replaces the current in-flight request on that connection. ## HTTP service internals `mapget::HttpService` binds the core service to an HTTP server implementation. Its responsibilities are: -- map HTTP endpoints to service calls (`/sources`, `/tiles`, `/abort`, `/status`, `/locate`, `/config`), +- map HTTP/WebSocket endpoints to service calls (`/sources`, `/tiles`, `/status`, `/locate`, `/config`), - parse JSON requests and build `LayerTilesRequest` objects, - serialize tile responses as JSONL or binary streams, -- manage per‑client state such as `clientId` for abort handling, and - provide `/config` as a JSON view on the YAML config file. ### Tile streaming For `/tiles`, the HTTP layer: -- parses the JSON body to extract `requests`, `stringPoolOffsets` and an optional `clientId`, +- parses the JSON body to extract `requests` and optional `stringPoolOffsets`, - constructs one `LayerTilesRequest` per map–layer combination, - attaches callbacks that feed results into a shared `HttpTilesRequestState`, and - sends out each tile as soon as it is produced by the service. In JSONL mode the response is a sequence of newline‑separated JSON objects. In binary mode the HTTP layer uses `TileLayerStream::Writer` to serialize string pool updates and tile blobs. Binary responses can optionally be compressed using gzip if the client sends `Accept-Encoding: gzip`. -The `/abort` endpoint uses the `clientId` mechanism to cancel all open tile requests for a given client and to prevent further work from being scheduled for them. +WebSocket `/tiles` uses the same request JSON shape but responds with binary VTLV frames only, and includes `Status` frames (JSON payload) whenever a request’s `RequestStatus` changes. ### Configuration endpoints @@ -207,7 +206,7 @@ These endpoints are guarded by command‑line flags: `--no-get-config` disables The model library provides both the binary tile encoding and the simfil query integration: -- `TileLayerStream::Writer` and `TileLayerStream::Reader` handle versioned, type‑tagged messages for string pools and tile layers. Each message starts with a protocol version, a `MessageType` (string pool, feature tile, SourceData tile, end-of-stream), and a payload size. +- `TileLayerStream::Writer` and `TileLayerStream::Reader` handle versioned, type‑tagged messages for string pools and tile layers. Each message starts with a protocol version, a `MessageType` (string pool, feature tile, SourceData tile, status, end-of-stream), and a payload size. - `TileFeatureLayer` derives from `simfil::ModelPool` and exposes methods such as `evaluate(...)` and `complete(...)` to run simfil expressions and obtain completion candidates. String pools are streamed incrementally. The server keeps a `StringPoolOffsetMap` that tracks, for each ongoing tile request, the highest string ID known to a given client per datasource node id. When a tile is written, `TileLayerStream::Writer` compares that offset with the current `StringPool::highest()` value: diff --git a/docs/mapget-user-guide.md b/docs/mapget-user-guide.md index 978cf6c9..68964c29 100644 --- a/docs/mapget-user-guide.md +++ b/docs/mapget-user-guide.md @@ -10,7 +10,7 @@ The guide is split into several focused documents: - [**Setup Guide**](mapget-setup.md) explains how to install mapget via `pip`, how to build the native executable from source, and how to start a server or use the built‑in `fetch` client for quick experiments. - [**Configuration Guide**](mapget-config.md) documents the YAML configuration file used with `--config`, the supported datasource types (`DataSourceHost`, `DataSourceProcess`, `GridDataSource`, `GeoJsonFolder) and the optional `http-settings` section used by tools and UIs. -- [**REST API Guide**](mapget-api.md) describes the HTTP endpoints exposed by `mapget serve`, including `/sources`, `/tiles`, `/abort`, `/status`, `/locate` and `/config`, along with their request and response formats and example calls. +- [**HTTP / WebSocket API Guide**](mapget-api.md) describes the endpoints exposed by `mapget serve`, including `/sources`, `/tiles`, `/status`, `/locate` and `/config`, along with their request and response formats and example calls. - [**Caching Guide**](mapget-cache.md) covers the available cache modes (`memory`, `persistent`, `none`), explains how to configure cache size and location, and shows how to inspect cache statistics via the status endpoint. - [**Simfil Language Extensions**](mapget-simfil-extensions.md) introduces the feature model, tiling scheme, geometry and validity concepts, and the binary tile stream format. This chapter is especially relevant if you are writing datasources or low‑level clients. - [**Layered Data Model**](mapget-model.md) introduces the feature model, tiling scheme, geometry and validity concepts, and the binary tile stream format. This chapter is especially relevant if you are writing datasources or low‑level clients. diff --git a/libs/http-datasource/CMakeLists.txt b/libs/http-datasource/CMakeLists.txt index e46557b5..7b323329 100644 --- a/libs/http-datasource/CMakeLists.txt +++ b/libs/http-datasource/CMakeLists.txt @@ -17,7 +17,7 @@ target_include_directories(mapget-http-datasource target_link_libraries(mapget-http-datasource PUBLIC - httplib::httplib + drogon mapget-model mapget-service tiny-process-library) diff --git a/libs/http-datasource/include/mapget/detail/http-server.h b/libs/http-datasource/include/mapget/detail/http-server.h index c80b7b62..cc6db8cb 100644 --- a/libs/http-datasource/include/mapget/detail/http-server.h +++ b/libs/http-datasource/include/mapget/detail/http-server.h @@ -1,11 +1,12 @@ #pragma once +#include #include #include -// Pre-declare httplib::Server to avoid including httplib.h in header -namespace httplib { -class Server; +// Forward declare Drogon app type to avoid including drogon headers in public headers. +namespace drogon { +class HttpAppFramework; } namespace mapget { @@ -76,7 +77,7 @@ class HttpServer * This function is called upon the first call to go(), * and allows any derived server class to add endpoints. */ - virtual void setup(httplib::Server&) = 0; + virtual void setup(drogon::HttpAppFramework&) = 0; /** * Derived servers can use this to control whether diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h index 5bb03aee..5a00d000 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h @@ -3,15 +3,24 @@ #include "mapget/model/sourcedatalayer.h" #include "mapget/model/featurelayer.h" #include "mapget/service/datasource.h" -#include "httplib.h" #include +#include #include +#include namespace TinyProcessLib { class Process; } +namespace drogon { +class HttpClient; +} + +namespace trantor { +class EventLoopThread; +} + namespace mapget { @@ -32,12 +41,17 @@ class RemoteDataSource : public DataSource * fails for any reason. */ RemoteDataSource(std::string const& host, uint16_t port); + ~RemoteDataSource(); // DataSource method overrides DataSourceInfo info() override; void fill(TileFeatureLayer::Ptr const& featureTile) override; void fill(TileSourceDataLayer::Ptr const& blobTile) override; - TileLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) override; + TileLayer::Ptr get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback = {}) override; std::vector locate(const mapget::LocateRequest &req) override; private: @@ -48,7 +62,8 @@ class RemoteDataSource : public DataSource std::string error_; // Multiple http clients allow parallel GET requests - std::vector httpClients_; + std::unique_ptr httpClientLoop_; + std::vector> httpClients_; std::atomic_uint64_t nextClient_{0}; }; @@ -76,7 +91,11 @@ class RemoteDataSourceProcess : public DataSource DataSourceInfo info() override; void fill(TileFeatureLayer::Ptr const& featureTile) override; void fill(TileSourceDataLayer::Ptr const& sourceDataLayer) override; - TileLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) override; + TileLayer::Ptr get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback = {}) override; std::vector locate(const mapget::LocateRequest &req) override; private: diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h index ed11adde..a0deda49 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h @@ -50,7 +50,7 @@ class DataSourceServer : public HttpServer DataSourceInfo const& info(); private: - void setup(httplib::Server&) override; + void setup(drogon::HttpAppFramework&) override; struct Impl; std::unique_ptr impl_; diff --git a/libs/http-datasource/src/datasource-client.cpp b/libs/http-datasource/src/datasource-client.cpp index 1199c48c..0a17088b 100644 --- a/libs/http-datasource/src/datasource-client.cpp +++ b/libs/http-datasource/src/datasource-client.cpp @@ -3,6 +3,10 @@ #include "process.hpp" #include "mapget/log.h" +#include +#include +#include + #include #include @@ -11,25 +15,43 @@ namespace mapget RemoteDataSource::RemoteDataSource(const std::string& host, uint16_t port) { + httpClientLoop_ = std::make_unique("MapgetRemoteDataSource"); + httpClientLoop_->run(); + + const auto hostString = fmt::format("http://{}:{}/", host, port); + // Fetch data source info. - httplib::Client client(host, port); - auto fetchedInfoJson = client.Get("/info"); - if (!fetchedInfoJson || fetchedInfoJson->status >= 300) - raise("Failed to fetch datasource info."); - info_ = DataSourceInfo::fromJson(nlohmann::json::parse(fetchedInfoJson->body)); + auto infoClient = drogon::HttpClient::newHttpClient(hostString, httpClientLoop_->getLoop()); + auto infoReq = drogon::HttpRequest::newHttpRequest(); + infoReq->setMethod(drogon::Get); + infoReq->setPath("/info"); + + auto [result, fetchedInfoResp] = infoClient->sendRequest(infoReq); + if (result != drogon::ReqResult::Ok || !fetchedInfoResp) { + raise(fmt::format("Failed to fetch datasource info: [{}]", drogon::to_string_view(result))); + } + if ((int)fetchedInfoResp->statusCode() >= 300) { + raise(fmt::format("Failed to fetch datasource info: [{}]", (int)fetchedInfoResp->statusCode())); + } + info_ = DataSourceInfo::fromJson(nlohmann::json::parse(std::string(fetchedInfoResp->body()))); if (info_.nodeId_.empty()) { // Unique node IDs are required for the string pool offsets. raise( fmt::format("Remote data source is missing node ID! Source info: {}", - fetchedInfoJson->body)); + std::string(fetchedInfoResp->body()))); } // Create as many clients as parallel requests are allowed. - for (auto i = 0; i < std::max(info_.maxParallelJobs_, 1); ++i) - httpClients_.emplace_back(host, port); + const auto clientCount = (std::max)(info_.maxParallelJobs_, 1); + httpClients_.reserve(clientCount); + for (auto i = 0; i < clientCount; ++i) { + httpClients_.emplace_back(drogon::HttpClient::newHttpClient(hostString, httpClientLoop_->getLoop())); + } } +RemoteDataSource::~RemoteDataSource() = default; + DataSourceInfo RemoteDataSource::info() { return info_; @@ -48,42 +70,42 @@ void RemoteDataSource::fill(const TileSourceDataLayer::Ptr& blobTile) } TileLayer::Ptr -RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceInfo& info) +RemoteDataSource::get( + const MapTileKey& k, + Cache::Ptr& cache, + const DataSourceInfo& info, + TileLayer::LoadStateCallback loadStateCallback) { // Round-robin usage of http clients to facilitate parallel requests. auto& client = httpClients_[(nextClient_++) % httpClients_.size()]; // Send a GET tile request. - auto tileResponse = client.Get(fmt::format( + auto tileReq = drogon::HttpRequest::newHttpRequest(); + tileReq->setMethod(drogon::Get); + tileReq->setPath(fmt::format( "/tile?layer={}&tileId={}&stringPoolOffset={}", k.layerId_, k.tileId_.value_, cachedStringPoolOffset(info.nodeId_, cache))); + auto [resultCode, tileResponse] = client->sendRequest(tileReq); // Check that the response is OK. - if (!tileResponse || tileResponse->status >= 300) { + if (resultCode != drogon::ReqResult::Ok || !tileResponse || (int)tileResponse->statusCode() >= 300) { // Forward to base class get(). This will instantiate a // default TileLayer and call fill(). In our implementation // of fill, we set an error. - if (tileResponse) { - if (tileResponse->has_header("HTTPLIB_ERROR")) { - error_ = tileResponse->get_header_value("HTTPLIB_ERROR"); - } - else if (tileResponse->has_header("EXCEPTION_WHAT")) { - error_ = tileResponse->get_header_value("EXCEPTION_WHAT"); - } - else { - error_ = fmt::format("Code {}", tileResponse->status); - } - } - else { + if (resultCode != drogon::ReqResult::Ok) { + error_ = drogon::to_string(resultCode); + } else if (tileResponse) { + error_ = fmt::format("Code {}", (int)tileResponse->statusCode()); + } else { error_ = "No remote response."; } // Use tile instantiation logic of the base class, // the error is then set in fill(). - return DataSource::get(k, cache, info); + return DataSource::get(k, cache, info, std::move(loadStateCallback)); } // Check the response body for expected content. @@ -92,8 +114,11 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn [&](auto&& mapId, auto&& layerId) { return info.getLayer(std::string(layerId)); }, [&](auto&& tile) { result = tile; }, cache); - reader.read(tileResponse->body); + reader.read(std::string(tileResponse->body())); + if (result && loadStateCallback) { + result->setLoadStateCallback(std::move(loadStateCallback)); + } return result; } @@ -102,12 +127,15 @@ std::vector RemoteDataSource::locate(const LocateRequest& req) // Round-robin usage of http clients to facilitate parallel requests. auto& client = httpClients_[(nextClient_++) % httpClients_.size()]; - // Send a GET tile request. - auto locateResponse = client.Post( - fmt::format("/locate"), req.serialize().dump(), "application/json"); + auto locateReq = drogon::HttpRequest::newHttpRequest(); + locateReq->setMethod(drogon::Post); + locateReq->setPath("/locate"); + locateReq->setContentTypeCode(drogon::CT_APPLICATION_JSON); + locateReq->setBody(req.serialize().dump()); + auto [resultCode, locateResponse] = client->sendRequest(locateReq); // Check that the response is OK. - if (!locateResponse || locateResponse->status >= 300) { + if (resultCode != drogon::ReqResult::Ok || !locateResponse || (int)locateResponse->statusCode() >= 300) { // Forward to base class get(). This will instantiate a // default TileFeatureLayer and call fill(). In our implementation // of fill, we set an error. @@ -116,7 +144,7 @@ std::vector RemoteDataSource::locate(const LocateRequest& req) } // Check the response body for expected content. - auto responseJson = nlohmann::json::parse(locateResponse->body); + auto responseJson = nlohmann::json::parse(std::string(locateResponse->body())); if (responseJson.is_null()) { return {}; } @@ -222,11 +250,15 @@ void RemoteDataSourceProcess::fill(TileSourceDataLayer::Ptr const& sourceDataLay } TileLayer::Ptr -RemoteDataSourceProcess::get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) +RemoteDataSourceProcess::get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback) { if (!remoteSource_) raise("Remote data source is not initialized."); - return remoteSource_->get(k, cache, info); + return remoteSource_->get(k, cache, info, std::move(loadStateCallback)); } std::vector RemoteDataSourceProcess::locate(const LocateRequest& req) diff --git a/libs/http-datasource/src/datasource-server.cpp b/libs/http-datasource/src/datasource-server.cpp index f6ea10f0..98ea11b0 100644 --- a/libs/http-datasource/src/datasource-server.cpp +++ b/libs/http-datasource/src/datasource-server.cpp @@ -1,54 +1,52 @@ #include "datasource-server.h" -#include "mapget/detail/http-server.h" -#include "mapget/model/sourcedatalayer.h" -#include "mapget/model/featurelayer.h" + +#include "mapget/log.h" #include "mapget/model/info.h" -#include "mapget/model/layer.h" #include "mapget/model/stream.h" -#include "httplib.h" +#include +#include + #include #include +#include -namespace mapget { +#include "fmt/format.h" + +namespace mapget +{ struct DataSourceServer::Impl { DataSourceInfo info_; - std::function tileFeatureCallback_ = [](auto&&) - { + std::function tileFeatureCallback_ = [](auto&&) { throw std::runtime_error("TileFeatureLayer callback is unset!"); }; - std::function tileSourceDataCallback_ = [](auto&&) - { + std::function tileSourceDataCallback_ = [](auto&&) { throw std::runtime_error("TileSourceDataLayer callback is unset!"); }; std::function(const LocateRequest&)> locateCallback_; std::shared_ptr strings_; - explicit Impl(DataSourceInfo info) - : info_(std::move(info)), strings_(std::make_shared(info_.nodeId_)) + explicit Impl(DataSourceInfo info) : info_(std::move(info)), strings_(std::make_shared(info_.nodeId_)) { } }; -DataSourceServer::DataSourceServer(DataSourceInfo const& info) - : HttpServer(), impl_(new Impl(info)) +DataSourceServer::DataSourceServer(DataSourceInfo const& info) : HttpServer(), impl_(new Impl(info)) { printPortToStdOut(true); } DataSourceServer::~DataSourceServer() = default; -DataSourceServer& -DataSourceServer::onTileFeatureRequest(std::function const& callback) +DataSourceServer& DataSourceServer::onTileFeatureRequest(std::function const& callback) { impl_->tileFeatureCallback_ = callback; return *this; } -DataSourceServer& -DataSourceServer::onTileSourceDataRequest(std::function const& callback) +DataSourceServer& DataSourceServer::onTileSourceDataRequest(std::function const& callback) { impl_->tileSourceDataCallback_ = callback; return *this; @@ -61,97 +59,132 @@ DataSourceServer& DataSourceServer::onLocateRequest( return *this; } -DataSourceInfo const& DataSourceServer::info() { - return impl_->info_; -} +DataSourceInfo const& DataSourceServer::info() { return impl_->info_; } -void DataSourceServer::setup(httplib::Server& server) +void DataSourceServer::setup(drogon::HttpAppFramework& app) { - // Set up GET /tile endpoint - server.Get( + app.registerHandler( "/tile", - [this](const httplib::Request& req, httplib::Response& res) { - // Extract parameters from request. - auto layerIdParam = req.get_param_value("layer"); - auto layer = impl_->info_.getLayer(layerIdParam); - - auto tileIdParam = TileId{std::stoull(req.get_param_value("tileId"))}; - auto stringPoolOffsetParam = (simfil::StringId)0; - if (req.has_param("stringPoolOffset")) - stringPoolOffsetParam = (simfil::StringId) - std::stoul(req.get_param_value("stringPoolOffset")); - - std::string responseType = "binary"; - if (req.has_param("responseType")) - responseType = req.get_param_value("responseType"); - - // Create response TileFeatureLayer. - auto tileLayer = [&]() -> std::shared_ptr { - switch (layer->type_) { - case mapget::LayerType::Features: { - auto tileFeatureLayer = std::make_shared( - tileIdParam, - impl_->info_.nodeId_, - impl_->info_.mapId_, - layer, - impl_->strings_); - impl_->tileFeatureCallback_(tileFeatureLayer); - return tileFeatureLayer; + [this](const drogon::HttpRequestPtr& req, std::function&& callback) + { + try { + auto const& layerIdParam = req->getParameter("layer"); + auto const& tileIdParam = req->getParameter("tileId"); + + if (layerIdParam.empty() || tileIdParam.empty()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Missing query parameter: layer and/or tileId"); + callback(resp); + return; } - case mapget::LayerType::SourceData: { - auto tileSourceLayer = std::make_shared( - tileIdParam, - impl_->info_.nodeId_, - impl_->info_.mapId_, - layer, - impl_->strings_); - impl_->tileSourceDataCallback_(tileSourceLayer); - return tileSourceLayer; + + auto layer = impl_->info_.getLayer(layerIdParam); + TileId tileId{std::stoull(tileIdParam)}; + + auto stringPoolOffsetParam = (simfil::StringId)0; + auto const& stringPoolOffsetStr = req->getParameter("stringPoolOffset"); + if (!stringPoolOffsetStr.empty()) { + stringPoolOffsetParam = (simfil::StringId)std::stoul(stringPoolOffsetStr); } - default: - throw std::runtime_error(fmt::format("Unsupported layer type {}", (int)layer->type_)); + + std::string responseType = "binary"; + auto const& responseTypeStr = req->getParameter("responseType"); + if (!responseTypeStr.empty()) + responseType = responseTypeStr; + + auto tileLayer = [&]() -> std::shared_ptr + { + switch (layer->type_) { + case mapget::LayerType::Features: { + auto tileFeatureLayer = std::make_shared( + tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); + impl_->tileFeatureCallback_(tileFeatureLayer); + return tileFeatureLayer; + } + case mapget::LayerType::SourceData: { + auto tileSourceLayer = std::make_shared( + tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); + impl_->tileSourceDataCallback_(tileSourceLayer); + return tileSourceLayer; + } + default: + throw std::runtime_error(fmt::format("Unsupported layer type {}", (int)layer->type_)); + } + }(); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + + if (responseType == "binary") { + std::string content; + TileLayerStream::StringPoolOffsetMap stringPoolOffsets{{impl_->info_.nodeId_, stringPoolOffsetParam}}; + TileLayerStream::Writer layerWriter{ + [&](std::string const& bytes, TileLayerStream::MessageType) { content.append(bytes); }, + stringPoolOffsets}; + layerWriter.write(tileLayer); + + resp->setContentTypeString("application/binary"); + resp->setBody(std::move(content)); + } else { + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(tileLayer->toJson().dump()); } - }(); - - // Serialize TileLayer using TileLayerStream. - if (responseType == "binary") { - std::stringstream content; - TileLayerStream::StringPoolOffsetMap stringPoolOffsets{ - {impl_->info_.nodeId_, stringPoolOffsetParam}}; - TileLayerStream::Writer layerWriter{ - [&](auto&& msg, auto&& msgType) { content << msg; }, - stringPoolOffsets}; - layerWriter.write(tileLayer); - res.set_content(content.str(), "application/binary"); + + callback(resp); } - else { - res.set_content(nlohmann::to_string(tileLayer->toJson()), "application/json"); + catch (std::exception const& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error: ") + e.what()); + callback(resp); } - }); + }, + {drogon::Get}); - // Set up GET /info endpoint - server.Get( + app.registerHandler( "/info", - [this](const httplib::Request&, httplib::Response& res) { - nlohmann::json j = impl_->info_.toJson(); - res.set_content(j.dump(), "application/json"); - }); - - // Set up POST /locate endpoint - server.Post( + [this](const drogon::HttpRequestPtr&, std::function&& callback) + { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(impl_->info_.toJson().dump()); + callback(resp); + }, + {drogon::Get}); + + app.registerHandler( "/locate", - [this](const httplib::Request& req, httplib::Response& res) { - LocateRequest parsedReq(nlohmann::json::parse(req.body)); - auto responseJson = nlohmann::json::array(); - - if (impl_->locateCallback_) { - for (auto const& response : impl_->locateCallback_(parsedReq)) { - responseJson.emplace_back(response.serialize()); + [this](const drogon::HttpRequestPtr& req, std::function&& callback) + { + try { + LocateRequest parsedReq(nlohmann::json::parse(std::string(req->body()))); + auto responseJson = nlohmann::json::array(); + + if (impl_->locateCallback_) { + for (auto const& response : impl_->locateCallback_(parsedReq)) { + responseJson.emplace_back(response.serialize()); + } } - } - res.set_content(responseJson.dump(), "application/json"); - }); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(responseJson.dump()); + callback(resp); + } + catch (std::exception const& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid request: ") + e.what()); + callback(resp); + } + }, + {drogon::Post}); } } // namespace mapget diff --git a/libs/http-datasource/src/http-server.cpp b/libs/http-datasource/src/http-server.cpp index 6b49eb66..422ab1b6 100644 --- a/libs/http-datasource/src/http-server.cpp +++ b/libs/http-datasource/src/http-server.cpp @@ -1,130 +1,280 @@ #include "mapget/detail/http-server.h" #include "mapget/log.h" -#include "httplib.h" -#include +#include + +#include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "fmt/format.h" namespace mapget { -// initialize the atomic activeHttpServer with nullptr +// Used by waitForSignal() so the signal handler knows what to stop. static std::atomic activeHttpServer = nullptr; +// Drogon uses a singleton app instance; running multiple independent servers +// in-process is not supported. +static std::atomic activeDrogonServer = nullptr; + +namespace +{ + +struct MountPoint +{ + std::string urlPrefix; + std::filesystem::path fsRoot; +}; + +[[nodiscard]] bool looksLikeWindowsDrivePath(std::string_view s) +{ + if (s.size() < 3) + return false; + const unsigned char drive = static_cast(s[0]); + return std::isalpha(drive) && s[1] == ':' && (s[2] == '\\' || s[2] == '/'); +} + +[[nodiscard]] std::string normalizeUrlPrefix(std::string prefix) +{ + if (prefix.empty()) + prefix = "/"; + if (prefix.front() != '/') + prefix.insert(prefix.begin(), '/'); + if (prefix.size() > 1 && prefix.back() == '/') + prefix.pop_back(); + return prefix; +} + +} // namespace + struct HttpServer::Impl { - httplib::Server server_; std::thread serverThread_; + std::atomic_bool running_{false}; + + std::mutex startMutex_; + std::condition_variable startCv_; + bool startNotified_ = false; + std::string startError_; + uint16_t port_ = 0; - bool setupWasCalled_ = false; bool printPortToStdout_ = false; + bool startedOnce_ = false; + + std::mutex mountsMutex_; + std::vector mounts_; static void handleSignal(int) { - // Temporarily holds the current active HttpServer auto* expected = activeHttpServer.load(); - - // Stop the active instance when a signal is received. - // We use compare_exchange_strong to make the operation atomic. if (activeHttpServer.compare_exchange_strong(expected, nullptr)) { if (expected) { expected->stop(); } } } + + void notifyStart(std::string errorMessage = {}) + { + std::lock_guard lock(startMutex_); + startError_ = std::move(errorMessage); + startNotified_ = true; + startCv_.notify_one(); + } }; HttpServer::HttpServer() : impl_(new Impl()) {} -HttpServer::~HttpServer() { - if (isRunning()) - stop(); +HttpServer::~HttpServer() +{ + stop(); } -void HttpServer::go( - std::string const& interfaceAddr, - uint16_t port, - uint32_t waitMs) +void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t waitMs) { - if (!impl_->setupWasCalled_) { - // Allow derived class to set up the server - setup(impl_->server_); - impl_->setupWasCalled_ = true; - } - - if (impl_->server_.is_running() || impl_->serverThread_.joinable()) + if (impl_->running_ || impl_->serverThread_.joinable()) raise("HttpServer is already running"); + if (impl_->startedOnce_) + raise("HttpServer cannot be restarted in-process (Drogon singleton)"); - if (port == 0) { - impl_->port_ = impl_->server_.bind_to_any_port(interfaceAddr); - } - else { - impl_->port_ = port; - impl_->server_.bind_to_port(interfaceAddr, port); + HttpServer* expected = nullptr; + if (!activeDrogonServer.compare_exchange_strong(expected, this)) + raise("Only one HttpServer can run per process (Drogon singleton)"); + + impl_->startedOnce_ = true; + + // Reset start state. + { + std::lock_guard lock(impl_->startMutex_); + impl_->startNotified_ = false; + impl_->startError_.clear(); } impl_->serverThread_ = std::thread( - [this, interfaceAddr] + [this, interfaceAddr, port] { - if (impl_->printPortToStdout_) - std::cout << "====== Running on port " << impl_->port_ << " ======" << std::endl; - else - log().info("====== Running on port {} ======", impl_->port_); - impl_->server_.listen_after_bind(); + try { + auto& app = drogon::app(); + + // Copy mounts to avoid locking after the server thread starts. + std::vector mountsCopy; + { + std::lock_guard lock(impl_->mountsMutex_); + mountsCopy = impl_->mounts_; + } + + if (!mountsCopy.empty()) { + std::sort( + mountsCopy.begin(), + mountsCopy.end(), + [](MountPoint const& a, MountPoint const& b) { return a.urlPrefix.size() > b.urlPrefix.size(); }); + + // Using empty document root makes addALocation's "alias" parameter + // work with absolute Windows paths (e.g. "C:/path"). + app.setDocumentRoot(""); + + for (auto const& m : mountsCopy) { + app.addALocation(m.urlPrefix, "", m.fsRoot.generic_string()); + } + } + + // Allow derived class to set up the server. + setup(app); + + app.addListener(interfaceAddr, port); + + app.registerBeginningAdvice([this]() { + // Beginning advice runs before listeners start. Post the actual + // startup notification to run after startListening() completed. + drogon::app().getLoop()->queueInLoop([this]() { + auto listeners = drogon::app().getListeners(); + if (listeners.empty()) { + impl_->notifyStart("HttpServer started without listeners"); + return; + } + + impl_->port_ = listeners.front().toPort(); + impl_->running_ = true; + impl_->notifyStart(); + + if (impl_->printPortToStdout_) + std::cout << "====== Running on port " << impl_->port_ << " ======" << std::endl; + else + log().info("====== Running on port {} ======", impl_->port_); + }); + }); + + app.run(); + } + catch (std::exception const& e) { + impl_->notifyStart(e.what()); + } + + impl_->running_ = false; + + HttpServer* expected = this; + (void)activeDrogonServer.compare_exchange_strong(expected, nullptr); }); - std::this_thread::sleep_for(std::chrono::milliseconds(waitMs)); - if (!impl_->server_.is_running() || !impl_->server_.is_valid()) - raise(fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); + std::unique_lock lk(impl_->startMutex_); + if (!impl_->startCv_.wait_for( + lk, + std::chrono::milliseconds(waitMs), + [this] { return impl_->startNotified_; })) { + raise(fmt::format("Could not start HttpServer on {}:{} (timeout)", interfaceAddr, port)); + } + + if (!impl_->startError_.empty()) + raise(impl_->startError_); } -bool HttpServer::isRunning() { - return impl_->server_.is_running(); +bool HttpServer::isRunning() +{ + return impl_->running_; } -void HttpServer::stop() { - if (!impl_->server_.is_running()) +void HttpServer::stop() +{ + if (!impl_->serverThread_.joinable()) return; - impl_->server_.stop(); - impl_->serverThread_.join(); + if (drogon::app().isRunning()) { + drogon::app().quit(); + } + + if (impl_->serverThread_.get_id() != std::this_thread::get_id()) + impl_->serverThread_.join(); } -uint16_t HttpServer::port() const { +uint16_t HttpServer::port() const +{ return impl_->port_; } -void HttpServer::waitForSignal() { - // So the signal handler knows what to call +void HttpServer::waitForSignal() +{ activeHttpServer = this; - // Set the signal handler for SIGINT and SIGTERM. std::signal(SIGINT, Impl::handleSignal); std::signal(SIGTERM, Impl::handleSignal); - // Wait for the signal handler to stop us, or the server to shut down on its own. while (isRunning()) { - std::this_thread::sleep_for(std::chrono::milliseconds (200)); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); } activeHttpServer = nullptr; } -bool HttpServer::mountFileSystem(const std::string& pathFromTo) +bool HttpServer::mountFileSystem(std::string const& pathFromTo) { - using namespace std::ranges; - auto parts = pathFromTo | views::split(':') | views::transform([](auto&& s){return std::string(&*s.begin(), distance(s));}); - auto partsVec = std::vector(parts.begin(), parts.end()); + std::string urlPrefix; + std::string fsRootStr; + + const auto firstColon = pathFromTo.find(':'); + if (firstColon == std::string::npos || looksLikeWindowsDrivePath(pathFromTo)) { + urlPrefix = "/"; + fsRootStr = pathFromTo; + } else { + urlPrefix = pathFromTo.substr(0, firstColon); + fsRootStr = pathFromTo.substr(firstColon + 1); + if (fsRootStr.empty()) + return false; + } + + urlPrefix = normalizeUrlPrefix(std::move(urlPrefix)); - if (partsVec.size() == 1) - return impl_->server_.set_mount_point("/", partsVec[0]); - return impl_->server_.set_mount_point(partsVec[0], partsVec[1]); + std::filesystem::path fsRoot(fsRootStr); + std::error_code ec; + fsRoot = std::filesystem::absolute(fsRoot, ec); + if (ec) + return false; + + auto exists = std::filesystem::exists(fsRoot, ec); + if (!exists || ec) + return false; + auto isDirectory = std::filesystem::is_directory(fsRoot, ec); + if (!isDirectory || ec) + return false; + + std::scoped_lock lock(impl_->mountsMutex_); + impl_->mounts_.emplace_back(MountPoint{std::move(urlPrefix), std::move(fsRoot)}); + return true; } -void HttpServer::printPortToStdOut(bool enabled) { +void HttpServer::printPortToStdOut(bool enabled) +{ impl_->printPortToStdout_ = enabled; } diff --git a/libs/http-service/CMakeLists.txt b/libs/http-service/CMakeLists.txt index ce3d733e..99476b81 100644 --- a/libs/http-service/CMakeLists.txt +++ b/libs/http-service/CMakeLists.txt @@ -6,7 +6,15 @@ add_library(mapget-http-service STATIC include/mapget/http-service/cli.h src/http-service.cpp + src/http-service-impl.h + src/http-service-impl.cpp + src/config-handler.cpp src/http-client.cpp + src/locate-handler.cpp + src/sources-handler.cpp + src/status-handler.cpp + src/tiles-http-handler.cpp + src/tiles-ws-controller.cpp src/cli.cpp) target_include_directories(mapget-http-service @@ -17,7 +25,7 @@ target_include_directories(mapget-http-service target_link_libraries(mapget-http-service PUBLIC - httplib::httplib + drogon yaml-cpp CLI11::CLI11 nlohmann_json_schema_validator diff --git a/libs/http-service/include/mapget/http-service/http-client.h b/libs/http-service/include/mapget/http-service/http-client.h index 67c293d6..7fad612d 100644 --- a/libs/http-service/include/mapget/http-service/http-client.h +++ b/libs/http-service/include/mapget/http-service/http-client.h @@ -18,7 +18,7 @@ class HttpClient * endpoint, and caches the result for the lifetime of this object. * @param enableCompression Enable gzip compression for responses (default: true) */ - explicit HttpClient(std::string const& host, uint16_t port, httplib::Headers headers = {}, bool enableCompression = true); + explicit HttpClient(std::string const& host, uint16_t port, AuthHeaders headers = {}, bool enableCompression = true); ~HttpClient(); /** diff --git a/libs/http-service/include/mapget/http-service/http-service.h b/libs/http-service/include/mapget/http-service/http-service.h index 41f834c8..61ea4c8e 100644 --- a/libs/http-service/include/mapget/http-service/http-service.h +++ b/libs/http-service/include/mapget/http-service/http-service.h @@ -1,6 +1,5 @@ #pragma once -#include "httplib.h" #include "mapget/detail/http-server.h" #include "mapget/model/featurelayer.h" #include "mapget/model/stream.h" @@ -56,7 +55,7 @@ class HttpService : public HttpServer, public Service ~HttpService() override; protected: - void setup(httplib::Server& server) override; + void setup(drogon::HttpAppFramework& app) override; private: struct Impl; diff --git a/libs/http-service/src/cli.cpp b/libs/http-service/src/cli.cpp index 7e51c540..f8ea8c6a 100644 --- a/libs/http-service/src/cli.cpp +++ b/libs/http-service/src/cli.cpp @@ -446,7 +446,7 @@ struct ServeCommand log().info("Webapp: {}", webapp_); if (!srv.mountFileSystem(webapp_)) { log().error(" ...failed to mount!"); - exit(1); + raise("Failed to mount webapp filesystem path."); } } diff --git a/libs/http-service/src/config-handler.cpp b/libs/http-service/src/config-handler.cpp new file mode 100644 index 00000000..82ac86e6 --- /dev/null +++ b/libs/http-service/src/config-handler.cpp @@ -0,0 +1,229 @@ +#include "http-service-impl.h" + +#include "cli.h" +#include "mapget/log.h" +#include "mapget/service/config.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "nlohmann/json.hpp" +#include "yaml-cpp/yaml.h" + +namespace mapget +{ + +drogon::HttpResponsePtr HttpService::Impl::openConfigFile(std::ifstream& configFile) +{ + auto configFilePath = DataSourceConfigService::get().getConfigFilePath(); + if (!configFilePath.has_value()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k404NotFound); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The config file path is not set. Check the server configuration."); + return resp; + } + + std::filesystem::path path = *configFilePath; + if (!std::filesystem::exists(path)) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k404NotFound); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The server does not have a config file."); + return resp; + } + + configFile.open(*configFilePath); + if (!configFile) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Failed to open config file."); + return resp; + } + + return nullptr; +} + +void HttpService::Impl::handleGetConfigRequest( + const drogon::HttpRequestPtr& /*req*/, + std::function&& callback) +{ + if (!isGetConfigEndpointEnabled()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k403Forbidden); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The GET /config endpoint is disabled by the server administrator."); + callback(resp); + return; + } + + std::ifstream configFile; + if (auto errorResp = openConfigFile(configFile)) { + callback(errorResp); + return; + } + + nlohmann::json jsonSchema = DataSourceConfigService::get().getDataSourceConfigSchema(); + + try { + YAML::Node configYaml = YAML::Load(configFile); + nlohmann::json jsonConfig; + std::unordered_map maskedSecretMap; + for (const auto& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + if (auto configYamlEntry = configYaml[key]) + jsonConfig[key] = yamlToJson(configYaml[key], true, &maskedSecretMap); + } + + nlohmann::json combinedJson; + combinedJson["schema"] = jsonSchema; + combinedJson["model"] = jsonConfig; + combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(combinedJson.dump(2)); + callback(resp); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error processing config file: ") + e.what()); + callback(resp); + } +} + +void HttpService::Impl::handlePostConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + if (!isPostConfigEndpointEnabled()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k403Forbidden); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The POST /config endpoint is not enabled by the server administrator."); + callback(resp); + return; + } + + struct ConfigUpdateState : std::enable_shared_from_this + { + trantor::EventLoop* loop = nullptr; + std::atomic_bool done{false}; + std::atomic_bool wroteConfig{false}; + std::unique_ptr subscription; + std::function callback; + }; + + std::ifstream configFile; + if (auto errorResp = openConfigFile(configFile)) { + callback(errorResp); + return; + } + + nlohmann::json jsonConfig; + try { + jsonConfig = nlohmann::json::parse(std::string(req->body())); + } + catch (const nlohmann::json::parse_error& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON format: ") + e.what()); + callback(resp); + return; + } + + try { + DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Validation failed: ") + e.what()); + callback(resp); + return; + } + + auto yamlConfig = YAML::Load(configFile); + std::unordered_map maskedSecrets; + yamlToJson(yamlConfig, true, &maskedSecrets); + + for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + if (jsonConfig.contains(key)) + yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); + } + + auto state = std::make_shared(); + state->loop = drogon::app().getLoop(); + state->callback = std::move(callback); + + state->subscription = DataSourceConfigService::get().subscribe( + [state](std::vector const&) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Configuration updated and applied successfully."); + state->callback(resp); + state->subscription.reset(); + }); + }, + [state](std::string const& error) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state, error]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error applying the configuration: ") + error); + state->callback(resp); + state->subscription.reset(); + }); + }); + + configFile.close(); + log().trace("Writing new config."); + state->wroteConfig = true; + if (auto configFilePath = DataSourceConfigService::get().getConfigFilePath()) { + std::ofstream newConfigFile(*configFilePath); + newConfigFile << yamlConfig; + newConfigFile.close(); + } + + std::thread([weak = state->weak_from_this()]() { + std::this_thread::sleep_for(std::chrono::seconds(60)); + if (auto state = weak.lock()) { + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Timeout while waiting for config to update."); + state->callback(resp); + state->subscription.reset(); + }); + } + }).detach(); +} + +} // namespace mapget + diff --git a/libs/http-service/src/http-client.cpp b/libs/http-service/src/http-client.cpp index a002c34f..e618a649 100644 --- a/libs/http-service/src/http-client.cpp +++ b/libs/http-service/src/http-client.cpp @@ -1,48 +1,72 @@ #include "http-client.h" -#include "httplib.h" + #include "mapget/log.h" +#include +#include +#include + +#include + +#include "fmt/format.h" + namespace mapget { +namespace +{ + +void applyHeaders(drogon::HttpRequestPtr const& req, AuthHeaders const& headers) +{ + for (auto const& [k, v] : headers) { + req->addHeader(k, v); + } +} + +} // namespace + struct HttpClient::Impl { - httplib::Client client_; + std::unique_ptr loopThread_; + drogon::HttpClientPtr client_; std::unordered_map sources_; std::shared_ptr stringPoolProvider_; - httplib::Headers headers_; + AuthHeaders headers_; - Impl(std::string const& host, uint16_t port, httplib::Headers headers, bool enableCompression) : - client_(host, port), - headers_(std::move(headers)) + Impl(std::string const& host, uint16_t port, AuthHeaders headers, bool enableCompression) : headers_(std::move(headers)) { - // Add Accept-Encoding header if compression is enabled and not already present - if (enableCompression) { - bool hasAcceptEncoding = false; - for (const auto& [key, value] : headers_) { - if (key == "Accept-Encoding") { - hasAcceptEncoding = true; - break; - } - } - if (!hasAcceptEncoding) { - headers_.emplace("Accept-Encoding", "gzip"); - } + if (enableCompression && !(headers_.contains("Accept-Encoding") || headers_.contains("accept-encoding"))) { + headers_.emplace("Accept-Encoding", "gzip"); } - + + loopThread_ = std::make_unique("MapgetHttpClient"); + loopThread_->run(); + + const auto hostString = fmt::format("http://{}:{}/", host, port); + client_ = drogon::HttpClient::newHttpClient(hostString, loopThread_->getLoop()); + stringPoolProvider_ = std::make_shared(); - client_.set_keep_alive(false); - auto sourcesJson = client_.Get("/sources", headers_); - if (!sourcesJson || sourcesJson->status != 200) - raise( - fmt::format("Failed to fetch sources: [{}]", sourcesJson->status)); - for (auto const& info : nlohmann::json::parse(sourcesJson->body)) { + + // Fetch data sources (/sources). + auto req = drogon::HttpRequest::newHttpRequest(); + req->setMethod(drogon::Get); + req->setPath("/sources"); + applyHeaders(req, headers_); + + auto [result, resp] = client_->sendRequest(req); + if (result != drogon::ReqResult::Ok || !resp) { + raise(fmt::format("Failed to fetch sources: [{}]", drogon::to_string_view(result))); + } + if (resp->statusCode() != drogon::k200OK) { + raise(fmt::format("Failed to fetch sources: [{}]", (int)resp->statusCode())); + } + + for (auto const& info : nlohmann::json::parse(std::string(resp->body()))) { auto parsedInfo = DataSourceInfo::fromJson(info); sources_.emplace(parsedInfo.mapId_, parsedInfo); } } - [[nodiscard]] std::shared_ptr - resolve(std::string_view const& map, std::string_view const& layer) const + [[nodiscard]] std::shared_ptr resolve(std::string_view const& map, std::string_view const& layer) const { auto mapIt = sources_.find(std::string(map)); if (mapIt == sources_.end()) @@ -51,8 +75,10 @@ struct HttpClient::Impl { } }; -HttpClient::HttpClient(const std::string& host, uint16_t port, httplib::Headers headers, bool enableCompression) : impl_( - std::make_unique(host, port, std::move(headers), enableCompression)) {} +HttpClient::HttpClient(const std::string& host, uint16_t port, AuthHeaders headers, bool enableCompression) + : impl_(std::make_unique(host, port, std::move(headers), enableCompression)) +{ +} HttpClient::~HttpClient() = default; @@ -73,43 +99,44 @@ LayerTilesRequest::Ptr HttpClient::request(const LayerTilesRequest::Ptr& request } auto reader = std::make_unique( - [this](auto&& mapId, auto&& layerId){return impl_->resolve(mapId, layerId);}, + [this](auto&& mapId, auto&& layerId) { return impl_->resolve(mapId, layerId); }, [request](auto&& result) { request->notifyResult(result); }, impl_->stringPoolProvider_); using namespace nlohmann; - // TODO: Currently, cpp-httplib client-POST does not support async responses. - // Those are only supported by GET. So, currently, this HttpClient - // does not profit from the streaming response. However, erdblick is - // is fully able to process async responses as it uses the browser fetch()-API. - auto tileResponse = impl_->client_.Post( - "/tiles", - impl_->headers_, - json::object({ - {"requests", json::array({request->toJson()})}, - {"stringPoolOffsets", reader->stringPoolCache()->stringPoolOffsets()} - }).dump(), - "application/json"); - - if (tileResponse) { - if (tileResponse->status == 200) { - reader->read(tileResponse->body); - } - else if (tileResponse->status == 400) { + auto body = json::object({ + {"requests", json::array({request->toJson()})}, + {"stringPoolOffsets", reader->stringPoolCache()->stringPoolOffsets()}, + }).dump(); + + auto httpReq = drogon::HttpRequest::newHttpRequest(); + httpReq->setMethod(drogon::Post); + httpReq->setPath("/tiles"); + httpReq->setContentTypeCode(drogon::CT_APPLICATION_JSON); + httpReq->setBody(std::move(body)); + applyHeaders(httpReq, impl_->headers_); + + auto [result, resp] = impl_->client_->sendRequest(httpReq); + if (result == drogon::ReqResult::Ok && resp) { + if (resp->statusCode() == drogon::k200OK) { + // TODO: Support streamed/chunked tile responses. + // Drogon's `HttpClient` API only provides the full buffered body. + // True streaming would require a custom client built on + // `trantor::TcpClient` (still within the Drogon dependency). + reader->read(std::string(resp->body())); + } else if (resp->statusCode() == drogon::k400BadRequest) { request->setStatus(RequestStatus::NoDataSource); - } - else if (tileResponse->status == 403) { + } else if (resp->statusCode() == drogon::k403Forbidden) { request->setStatus(RequestStatus::Unauthorized); + } else { + request->setStatus(RequestStatus::Aborted); } - // TODO if multiple LayerTileRequests are ever sent by this client, - // additionally handle RequestStatus::Aborted. - } - else { + } else { request->setStatus(RequestStatus::Aborted); } return request; } -} +} // namespace mapget diff --git a/libs/http-service/src/http-service-impl.cpp b/libs/http-service/src/http-service-impl.cpp new file mode 100644 index 00000000..1f323b31 --- /dev/null +++ b/libs/http-service/src/http-service-impl.cpp @@ -0,0 +1,39 @@ +#include "http-service-impl.h" + +#include "mapget/log.h" + +#ifdef __linux__ +#include +#endif + +namespace mapget +{ + +HttpService::Impl::Impl(HttpService& self, const HttpServiceConfig& config) : self_(self), config_(config) {} + +void HttpService::Impl::tryMemoryTrim(ResponseType responseType) const +{ + uint64_t interval = + (responseType == ResponseType::Binary) ? config_.memoryTrimIntervalBinary : config_.memoryTrimIntervalJson; + + if (interval == 0) { + return; + } + + auto& counter = (responseType == ResponseType::Binary) ? binaryRequestCounter_ : jsonRequestCounter_; + auto count = counter.fetch_add(1, std::memory_order_relaxed); + if ((count % interval) != 0) { + return; + } + +#ifdef __linux__ +#ifndef NDEBUG + const char* typeStr = (responseType == ResponseType::Binary) ? "binary" : "JSON"; + log().debug("Trimming memory after {} {} requests (interval: {})", count, typeStr, interval); +#endif + malloc_trim(0); +#endif +} + +} // namespace mapget + diff --git a/libs/http-service/src/http-service-impl.h b/libs/http-service/src/http-service-impl.h new file mode 100644 index 00000000..2db77a6c --- /dev/null +++ b/libs/http-service/src/http-service-impl.h @@ -0,0 +1,72 @@ +#pragma once + +#include "http-service.h" + +#include +#include + +#include +#include +#include +#include + +namespace mapget +{ + +namespace detail +{ + +[[nodiscard]] inline AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) +{ + AuthHeaders headers; + for (auto const& [k, v] : req->headers()) { + headers.emplace(k, v); + } + return headers; +} + +} // namespace detail + +struct HttpService::Impl +{ + HttpService& self_; + HttpServiceConfig config_; + mutable std::atomic binaryRequestCounter_{0}; + mutable std::atomic jsonRequestCounter_{0}; + + explicit Impl(HttpService& self, const HttpServiceConfig& config); + + enum class ResponseType { Binary, Json }; + + void tryMemoryTrim(ResponseType responseType) const; + + struct TilesStreamState; + + void handleTilesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + void handleSourcesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + void handleStatusRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + void handleLocateRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + static drogon::HttpResponsePtr openConfigFile(std::ifstream& configFile); + + static void handleGetConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback); + + void handlePostConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; +}; + +} // namespace mapget diff --git a/libs/http-service/src/http-service.cpp b/libs/http-service/src/http-service.cpp index 0137a1b7..4e377448 100644 --- a/libs/http-service/src/http-service.cpp +++ b/libs/http-service/src/http-service.cpp @@ -1,668 +1,74 @@ -#include "http-service.h" -#include "mapget/log.h" -#include "mapget/service/config.h" +#include "http-service-impl.h" -#include -#include -#include -#include -#include -#include -#include -#include "cli.h" -#include "httplib.h" -#include "nlohmann/json-schema.hpp" -#include "nlohmann/json.hpp" -#include "yaml-cpp/yaml.h" -#include +#include "tiles-ws-controller.h" -#ifdef __linux__ -#include -#endif +#include +#include +#include -namespace mapget -{ - -namespace -{ - -/** - * Simple gzip compressor for streaming compression - */ -class GzipCompressor { -public: - GzipCompressor() { - strm_.zalloc = Z_NULL; - strm_.zfree = Z_NULL; - strm_.opaque = Z_NULL; - // 16+MAX_WBITS enables gzip format (not just deflate) - int ret = deflateInit2(&strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, - 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY); - if (ret != Z_OK) { - throw std::runtime_error("Failed to initialize gzip compressor"); - } - } - - ~GzipCompressor() { - deflateEnd(&strm_); - } - - std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) { - std::string result; - if (size == 0 && flush_mode == Z_NO_FLUSH) { - return result; - } - - strm_.avail_in = static_cast(size); - strm_.next_in = reinterpret_cast(const_cast(data)); - - char outbuf[8192]; - do { - strm_.avail_out = sizeof(outbuf); - strm_.next_out = reinterpret_cast(outbuf); - - int ret = deflate(&strm_, flush_mode); - if (ret == Z_STREAM_ERROR) { - throw std::runtime_error("Gzip compression failed"); - } - - size_t have = sizeof(outbuf) - strm_.avail_out; - result.append(outbuf, have); - } while (strm_.avail_out == 0); - - return result; - } +#include - std::string finish() { - return compress(nullptr, 0, Z_FINISH); - } - -private: - z_stream strm_{}; -}; - -/** - * Recursively convert a YAML node to a JSON object, - * with special handling for sensitive fields. - * The function returns a nlohmann::json object and updates maskedSecretMap. - */ -} // namespace - -struct HttpService::Impl +namespace mapget { - HttpService& self_; - HttpServiceConfig config_; - mutable std::atomic binaryRequestCounter_{0}; - mutable std::atomic jsonRequestCounter_{0}; - - explicit Impl(HttpService& self, const HttpServiceConfig& config) - : self_(self), config_(config) {} - - enum class ResponseType { - Binary, - Json - }; - - void tryMemoryTrim(ResponseType responseType) const { - uint64_t interval = (responseType == ResponseType::Binary) - ? config_.memoryTrimIntervalBinary - : config_.memoryTrimIntervalJson; - - if (interval > 0) { - auto& counter = (responseType == ResponseType::Binary) - ? binaryRequestCounter_ - : jsonRequestCounter_; - - auto count = counter.fetch_add(1, std::memory_order_relaxed); - if ((count % interval) == 0) { -#ifdef __linux__ - // Only log in debug builds to reduce overhead - #ifndef NDEBUG - const char* typeStr = (responseType == ResponseType::Binary) ? "binary" : "JSON"; - log().debug("Trimming memory after {} {} requests (interval: {})", count, typeStr, interval); - #endif - malloc_trim(0); -#endif - // On non-Linux platforms, this is a no-op but we still track the counter - } - } - } - - // Use a shared buffer for the responses and a mutex for thread safety. - struct HttpTilesRequestState - { - static constexpr auto binaryMimeType = "application/binary"; - static constexpr auto jsonlMimeType = "application/jsonl"; - static constexpr auto anyMimeType = "*/*"; - - std::mutex mutex_; - std::condition_variable resultEvent_; - - uint64_t requestId_; - std::stringstream buffer_; - std::string responseType_; - std::unique_ptr writer_; - std::vector requests_; - TileLayerStream::StringPoolOffsetMap stringOffsets_; - std::unique_ptr compressor_; // Store compressor per request - - HttpTilesRequestState() - { - static std::atomic_uint64_t nextRequestId; - writer_ = std::make_unique( - [&, this](auto&& msg, auto&& msgType) { buffer_ << msg; }, - stringOffsets_); - requestId_ = nextRequestId++; - } - - void parseRequestFromJson(nlohmann::json const& requestJson) - { - std::string mapId = requestJson["mapId"]; - std::string layerId = requestJson["layerId"]; - std::vector tileIds; - tileIds.reserve(requestJson["tileIds"].size()); - for (auto const& tid : requestJson["tileIds"].get>()) - tileIds.emplace_back(tid); - requests_ - .push_back(std::make_shared(mapId, layerId, std::move(tileIds))); - } - - void setResponseType(std::string const& s) - { - responseType_ = s; - if (responseType_ == HttpTilesRequestState::binaryMimeType) - return; - if (responseType_ == HttpTilesRequestState::jsonlMimeType) - return; - if (responseType_ == HttpTilesRequestState::anyMimeType) { - responseType_ = binaryMimeType; - return; - } - raise(fmt::format("Unknown Accept-Header value {}", responseType_)); - } - - void addResult(TileLayer::Ptr const& result) - { - std::unique_lock lock(mutex_); - log().debug("Response ready: {}", MapTileKey(*result).toString()); - if (responseType_ == binaryMimeType) { - // Binary response - writer_->write(result); - } - else { - // JSON response - optimize with compact dump settings - // TODO: Implement direct streaming with result->writeGeoJsonTo(buffer_) - // to avoid intermediate JSON object creation entirely - auto json = result->toJson(); - // Use compact dump: no indentation, no spaces, ignore errors - // This reduces string allocation overhead - buffer_ << json.dump(-1, ' ', false, nlohmann::json::error_handler_t::ignore) << "\n"; - } - resultEvent_.notify_one(); - } - }; - - mutable std::mutex clientRequestMapMutex_; - mutable std::unordered_map> requestStatePerClientId_; - - void abortRequestsForClientId(std::string clientId, std::shared_ptr newState = nullptr) const - { - std::unique_lock clientRequestMapAccess(clientRequestMapMutex_); - auto clientRequestIt = requestStatePerClientId_.find(clientId); - if (clientRequestIt != requestStatePerClientId_.end()) { - // Ensure that any previous requests from the same clientId - // are finished post-haste! - bool anySoftAbort = false; - for (auto const& req : clientRequestIt->second->requests_) { - if (!req->isDone()) { - self_.abort(req); - anySoftAbort = true; - } - } - if (anySoftAbort) - log().warn("Soft-aborting tiles request {}", clientRequestIt->second->requestId_); - requestStatePerClientId_.erase(clientRequestIt); - } - if (newState) { - requestStatePerClientId_.emplace(clientId, newState); - } - } - - /** - * Wraps around the generic mapget service's request() function - * to include httplib request decoding and response encoding. - */ - void handleTilesRequest(const httplib::Request& req, httplib::Response& res) const - { - // Parse the JSON request. - nlohmann::json j = nlohmann::json::parse(req.body); - auto requestsJson = j["requests"]; - - // TODO: Limit number of requests to avoid DoS to other users. - // Within one HTTP request, all requested tiles from the same map+layer - // combination should be in a single LayerTilesRequest. - auto state = std::make_shared(); - log().info("Processing tiles request {}", state->requestId_); - for (auto& requestJson : requestsJson) { - state->parseRequestFromJson(requestJson); - } - - // Parse stringPoolOffsets. - if (j.contains("stringPoolOffsets")) { - for (auto& item : j["stringPoolOffsets"].items()) { - state->stringOffsets_[item.key()] = item.value().get(); - } - } - - // Determine response type. - state->setResponseType(req.get_header_value("Accept")); - - // Process requests. - for (auto& request : state->requests_) { - request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); - request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); - request->onDone_ = [state](RequestStatus r) - { - state->resultEvent_.notify_one(); - }; - } - auto canProcess = self_.request( - state->requests_, - AuthHeaders{req.headers.begin(), req.headers.end()}); - - if (!canProcess) { - // Send a status report detailing for each request - // whether its data source is unavailable or it was aborted. - res.status = 400; - std::vector> requestStatuses{}; - for (const auto& r : state->requests_) { - requestStatuses.push_back(static_cast>(r->getStatus())); - if (r->getStatus() == RequestStatus::Unauthorized) { - res.status = 403; // Forbidden. - } - } - res.set_content( - nlohmann::json::object({{"requestStatuses", requestStatuses}}).dump(), - "application/json"); - return; - } - - // Parse/Process clientId. - if (j.contains("clientId")) { - auto clientId = j["clientId"].get(); - abortRequestsForClientId(clientId, state); - } - - // Check if client accepts gzip compression - bool enableGzip = false; - if (req.has_header("Accept-Encoding")) { - std::string acceptEncoding = req.get_header_value("Accept-Encoding"); - enableGzip = acceptEncoding.find("gzip") != std::string::npos; - log().debug("Accept-Encoding header: '{}', enableGzip: {}", acceptEncoding, enableGzip); - } else { - log().debug("No Accept-Encoding header present"); - } - - // Set Content-Encoding header if compression is enabled - if (enableGzip) { - res.set_header("Content-Encoding", "gzip"); - state->compressor_ = std::make_unique(); - log().debug("Set Content-Encoding: gzip header"); - } - - // For efficiency, set up httplib to stream tile layer responses to client: - // (1) Lambda continuously supplies response data to httplib's DataSink, - // picking up data from state->buffer_ until all tile requests are done. - // Then, signal sink->done() to close the stream with a 200 status. - // Using chunked transfer encoding with optional manual compression. - // (2) Lambda acts as a cleanup routine, triggered by httplib upon request wrap-up. - // The success flag indicates if wrap-up was due to sink->done() or external factors - // like network errors or request aborts in lengthy tile requests (e.g., map-viewer). - res.set_chunked_content_provider( - state->responseType_, - [state](size_t offset, httplib::DataSink& sink) - { - std::unique_lock lock(state->mutex_); - - // Wait until there is data to be read. - std::string strBuf; - bool allDone = false; - state->resultEvent_.wait( - lock, - [&] - { - allDone = std::all_of( - state->requests_.begin(), - state->requests_.end(), - [](const auto& r) { return r->isDone(); }); - if (allDone && state->responseType_ == HttpTilesRequestState::binaryMimeType) - state->writer_->sendEndOfStream(); - strBuf = state->buffer_.str(); - return !strBuf.empty() || allDone; - }); - - if (!strBuf.empty()) { - // Compress data if gzip is enabled - if (state->compressor_) { - std::string compressed = state->compressor_->compress(strBuf.data(), strBuf.size()); - if (!compressed.empty()) { - log().debug("Compressing: {} bytes -> {} bytes (request {})", - strBuf.size(), compressed.size(), state->requestId_); - sink.write(compressed.data(), compressed.size()); - } - } else { - log().debug("Streaming {} bytes (no compression)...", strBuf.size()); - sink.write(strBuf.data(), strBuf.size()); - } - sink.os.flush(); - state->buffer_.str(""); // Clear buffer content - state->buffer_.clear(); // Clear error flags - // Force release of internal buffer memory - std::stringstream().swap(state->buffer_); - } - - // Call sink.done() when all requests are done. - if (allDone) { - // Finish compression if enabled - if (state->compressor_) { - std::string finalChunk = state->compressor_->finish(); - log().debug( - "Final compression chunk is {} bytes.", - strBuf.size(), finalChunk.size(), state->requestId_); - if (!finalChunk.empty()) { - sink.write(finalChunk.data(), finalChunk.size()); - } - } - sink.done(); - } - - return true; - }, - // Network error/timeout of request to datasource: - // cleanup callback to abort the requests. - [state, this](bool success) - { - if (!success) { - log().warn("Aborting tiles request {}", state->requestId_); - for (auto& request : state->requests_) { - self_.abort(request); - } - } - else { - log().info("Tiles request {} was successful.", state->requestId_); - // Determine response type and trim accordingly - ResponseType respType = (state->responseType_ == HttpTilesRequestState::binaryMimeType) - ? ResponseType::Binary - : ResponseType::Json; - tryMemoryTrim(respType); - } - }); - } - - void handleAbortRequest(const httplib::Request& req, httplib::Response& res) const - { - // Parse the JSON request. - nlohmann::json j = nlohmann::json::parse(req.body); - if (j.contains("clientId")) { - auto const clientId = j["clientId"].get(); - abortRequestsForClientId(clientId); - } - else { - res.status = 400; - res.set_content("Missing clientId", "text/plain"); - } - } - - void handleSourcesRequest(const httplib::Request& req, httplib::Response& res) const - { - auto sourcesInfo = nlohmann::json::array(); - for (auto& source : self_.info(AuthHeaders{req.headers.begin(), req.headers.end()})) { - sourcesInfo.push_back(source.toJson()); - } - res.set_content(sourcesInfo.dump(), "application/json"); - } - - void handleStatusRequest(const httplib::Request&, httplib::Response& res) const - { - auto serviceStats = self_.getStatistics(); - auto cacheStats = self_.cache()->getStatistics(); - - std::ostringstream oss; - oss << ""; - oss << "

Status Information

"; - - // Output serviceStats - oss << "

Service Statistics

"; - oss << "
" << serviceStats.dump(4) << "
"; // Indentation of 4 for pretty printing - - // Output cacheStats - oss << "

Cache Statistics

"; - oss << "
" << cacheStats.dump(4) << "
"; // Indentation of 4 for pretty printing - - oss << ""; - res.set_content(oss.str(), "text/html"); - } - - void handleLocateRequest(const httplib::Request& req, httplib::Response& res) const - { - // Parse the JSON request. - nlohmann::json j = nlohmann::json::parse(req.body); - auto requestsJson = j["requests"]; - auto allResponsesJson = nlohmann::json::array(); - - for (auto const& locateReqJson : requestsJson) { - LocateRequest locateReq{locateReqJson}; - auto responsesJson = nlohmann::json::array(); - for (auto const& resp : self_.locate(locateReq)) - responsesJson.emplace_back(resp.serialize()); - allResponsesJson.emplace_back(responsesJson); - } - - res.set_content( - nlohmann::json::object({{"responses", allResponsesJson}}).dump(), - "application/json"); - } - - static bool openConfigFile(std::ifstream& configFile, httplib::Response& res) - { - auto configFilePath = DataSourceConfigService::get().getConfigFilePath(); - if (!configFilePath.has_value()) { - res.status = 404; // Not found. - res.set_content( - "The config file path is not set. Check the server configuration.", - "text/plain"); - return false; - } - - std::filesystem::path path = *configFilePath; - if (!configFilePath || !std::filesystem::exists(path)) { - res.status = 404; // Not found. - res.set_content("The server does not have a config file.", "text/plain"); - return false; - } - - configFile.open(*configFilePath); - if (!configFile) { - res.status = 500; // Internal Server Error. - res.set_content("Failed to open config file.", "text/plain"); - return false; - } - - return true; - } - - static void handleGetConfigRequest(const httplib::Request& req, httplib::Response& res) - { - if (!isGetConfigEndpointEnabled()) { - res.status = 403; // Forbidden. - res.set_content( - "The GET /config endpoint is disabled by the server administrator.", - "text/plain"); - return; - } - - std::ifstream configFile; - if (!openConfigFile(configFile, res)) { - return; - } - nlohmann::json jsonSchema = DataSourceConfigService::get().getDataSourceConfigSchema(); - - try { - // Load config YAML, expose the parts which clients may edit. - YAML::Node configYaml = YAML::Load(configFile); - nlohmann::json jsonConfig; - std::unordered_map maskedSecretMap; - for (const auto& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { - if (auto configYamlEntry = configYaml[key]) - jsonConfig[key] = yamlToJson(configYaml[key], true, &maskedSecretMap); - } - - nlohmann::json combinedJson; - combinedJson["schema"] = jsonSchema; - combinedJson["model"] = jsonConfig; - combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); - - // Set the response - res.status = 200; // OK - res.set_content(combinedJson.dump(2), "application/json"); - } - catch (const std::exception& e) { - res.status = 500; // Internal Server Error - res.set_content("Error processing config file: " + std::string(e.what()), "text/plain"); - } - } - - static void handlePostConfigRequest(const httplib::Request& req, httplib::Response& res) - { - if (!isPostConfigEndpointEnabled()) { - res.status = 403; // Forbidden. - res.set_content( - "The POST /config endpoint is not enabled by the server administrator.", - "text/plain"); - return; - } - - std::mutex mtx; - std::condition_variable cv; - bool update_done = false; - - std::ifstream configFile; - if (!openConfigFile(configFile, res)) { - return; - } - - // Subscribe to configuration changes. - auto subscription = DataSourceConfigService::get().subscribe( - [&](const std::vector& serviceConfigNodes) - { - std::lock_guard lock(mtx); - res.status = 200; - res.set_content("Configuration updated and applied successfully.", "text/plain"); - update_done = true; - cv.notify_one(); - }, - [&](const std::string& error) - { - std::lock_guard lock(mtx); - res.status = 500; - res.set_content("Error applying the configuration: " + error, "text/plain"); - update_done = true; - cv.notify_one(); - }); - - // Parse the JSON from the request body. - nlohmann::json jsonConfig; - try { - jsonConfig = nlohmann::json::parse(req.body); - } - catch (const nlohmann::json::parse_error& e) { - res.status = 400; // Bad Request - res.set_content("Invalid JSON format: " + std::string(e.what()), "text/plain"); - return; - } - - // Validate JSON against schema. - try { - DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); - } - catch (const std::exception& e) { - res.status = 500; // Internal Server Error. - res.set_content("Validation failed: " + std::string(e.what()), "text/plain"); - return; - } - - // Load the YAML, parse the secrets. - auto yamlConfig = YAML::Load(configFile); - std::unordered_map maskedSecrets; - yamlToJson(yamlConfig, true, &maskedSecrets); - - // Create YAML nodes from JSON nodes. - for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { - if (jsonConfig.contains(key)) - yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); - } - - // Write the YAML to configFilePath. - update_done = false; - configFile.close(); - log().trace("Writing new config."); - std::ofstream newConfigFile(*DataSourceConfigService::get().getConfigFilePath()); - newConfigFile << yamlConfig; - newConfigFile.close(); - - // Wait for the subscription callback. - std::unique_lock lk(mtx); - if (!cv.wait_for(lk, std::chrono::seconds(60), [&] { return update_done; })) { - res.status = 500; // Internal Server Error. - res.set_content("Timeout while waiting for config to update.", "text/plain"); - } - } -}; HttpService::HttpService(Cache::Ptr cache, const HttpServiceConfig& config) - : Service(std::move(cache), config.watchConfig, config.defaultTtl), - impl_(std::make_unique(*this, config)) + : Service(std::move(cache), config.watchConfig, config.defaultTtl), impl_(std::make_unique(*this, config)) { } HttpService::~HttpService() = default; -void HttpService::setup(httplib::Server& server) +void HttpService::setup(drogon::HttpAppFramework& app) { - server.Post( - "/tiles", - [&](const httplib::Request& req, httplib::Response& res) - { impl_->handleTilesRequest(req, res); }); + detail::registerTilesWebSocketController(app, *this); - server.Post( - "/abort", - [&](const httplib::Request& req, httplib::Response& res) - { impl_->handleAbortRequest(req, res); }); + app.registerHandler( + "/tiles", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleTilesRequest(req, std::move(callback)); + }, + {drogon::Post}); - server.Get( + app.registerHandler( "/sources", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleSourcesRequest(req, res); }); + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleSourcesRequest(req, std::move(callback)); + }, + {drogon::Get}); - server.Get( + app.registerHandler( "/status", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleStatusRequest(req, res); }); + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleStatusRequest(req, std::move(callback)); + }, + {drogon::Get}); - server.Post( + app.registerHandler( "/locate", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleLocateRequest(req, res); }); + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleLocateRequest(req, std::move(callback)); + }, + {drogon::Post}); - server.Get( + app.registerHandler( "/config", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleGetConfigRequest(req, res); }); + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + if (req->method() == drogon::Get) { + Impl::handleGetConfigRequest(req, std::move(callback)); + return; + } + if (req->method() == drogon::Post) { + impl_->handlePostConfigRequest(req, std::move(callback)); + return; + } - server.Post( - "/config", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handlePostConfigRequest(req, res); }); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k405MethodNotAllowed); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Method not allowed"); + callback(resp); + }, + {drogon::Get, drogon::Post}); } } // namespace mapget diff --git a/libs/http-service/src/locate-handler.cpp b/libs/http-service/src/locate-handler.cpp new file mode 100644 index 00000000..81ad9ea5 --- /dev/null +++ b/libs/http-service/src/locate-handler.cpp @@ -0,0 +1,43 @@ +#include "http-service-impl.h" + +#include + +#include "nlohmann/json.hpp" + +namespace mapget +{ + +void HttpService::Impl::handleLocateRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + try { + nlohmann::json j = nlohmann::json::parse(std::string(req->body())); + auto requestsJson = j["requests"]; + auto allResponsesJson = nlohmann::json::array(); + + for (auto const& locateReqJson : requestsJson) { + LocateRequest locateReq{locateReqJson}; + auto responsesJson = nlohmann::json::array(); + for (auto const& resp : self_.locate(locateReq)) + responsesJson.emplace_back(resp.serialize()); + allResponsesJson.emplace_back(responsesJson); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); + callback(resp); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + } +} + +} // namespace mapget + diff --git a/libs/http-service/src/sources-handler.cpp b/libs/http-service/src/sources-handler.cpp new file mode 100644 index 00000000..d085ac3f --- /dev/null +++ b/libs/http-service/src/sources-handler.cpp @@ -0,0 +1,27 @@ +#include "http-service-impl.h" + +#include + +#include "nlohmann/json.hpp" + +namespace mapget +{ + +void HttpService::Impl::handleSourcesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + auto sourcesInfo = nlohmann::json::array(); + for (auto& source : self_.info(detail::authHeadersFromRequest(req))) { + sourcesInfo.push_back(source.toJson()); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(sourcesInfo.dump()); + callback(resp); +} + +} // namespace mapget + diff --git a/libs/http-service/src/status-handler.cpp b/libs/http-service/src/status-handler.cpp new file mode 100644 index 00000000..b0a91216 --- /dev/null +++ b/libs/http-service/src/status-handler.cpp @@ -0,0 +1,34 @@ +#include "http-service-impl.h" + +#include + +#include + +namespace mapget +{ + +void HttpService::Impl::handleStatusRequest( + const drogon::HttpRequestPtr& /*req*/, + std::function&& callback) const +{ + auto serviceStats = self_.getStatistics(); + auto cacheStats = self_.cache()->getStatistics(); + + std::ostringstream oss; + oss << ""; + oss << "

Status Information

"; + oss << "

Service Statistics

"; + oss << "
" << serviceStats.dump(4) << "
"; + oss << "

Cache Statistics

"; + oss << "
" << cacheStats.dump(4) << "
"; + oss << ""; + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_HTML); + resp->setBody(oss.str()); + callback(resp); +} + +} // namespace mapget + diff --git a/libs/http-service/src/tiles-http-handler.cpp b/libs/http-service/src/tiles-http-handler.cpp new file mode 100644 index 00000000..3edaa1ad --- /dev/null +++ b/libs/http-service/src/tiles-http-handler.cpp @@ -0,0 +1,420 @@ +#include "http-service-impl.h" + +#include "mapget/log.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nlohmann/json.hpp" + +#include + +namespace mapget +{ +namespace +{ + +class GzipCompressor +{ +public: + GzipCompressor() + { + strm_.zalloc = Z_NULL; + strm_.zfree = Z_NULL; + strm_.opaque = Z_NULL; + // 16+MAX_WBITS enables gzip format (not just deflate) + int ret = deflateInit2( + &strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { + throw std::runtime_error("Failed to initialize gzip compressor"); + } + } + + ~GzipCompressor() { deflateEnd(&strm_); } + + GzipCompressor(GzipCompressor const&) = delete; + GzipCompressor(GzipCompressor&&) = delete; + + std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) + { + std::string result; + if (size == 0 && flush_mode == Z_NO_FLUSH) { + return result; + } + + strm_.avail_in = static_cast(size); + strm_.next_in = reinterpret_cast(const_cast(data)); + + char outbuf[8192]; + do { + strm_.avail_out = sizeof(outbuf); + strm_.next_out = reinterpret_cast(outbuf); + + int ret = deflate(&strm_, flush_mode); + if (ret == Z_STREAM_ERROR) { + throw std::runtime_error("Gzip compression failed"); + } + + size_t have = sizeof(outbuf) - strm_.avail_out; + result.append(outbuf, have); + } while (strm_.avail_out == 0); + + return result; + } + + std::string finish() { return compress(nullptr, 0, Z_FINISH); } + +private: + z_stream strm_{}; +}; + +[[nodiscard]] bool containsGzip(std::string_view acceptEncoding) +{ + return !acceptEncoding.empty() && acceptEncoding.find("gzip") != std::string_view::npos; +} + +} // namespace + +struct HttpService::Impl::TilesStreamState : std::enable_shared_from_this +{ + static constexpr auto binaryMimeType = "application/binary"; + static constexpr auto jsonlMimeType = "application/jsonl"; + static constexpr auto anyMimeType = "*/*"; + + explicit TilesStreamState(Impl const& impl, trantor::EventLoop* loop) : impl_(impl), loop_(loop) + { + static std::atomic_uint64_t nextRequestId; + requestId_ = nextRequestId++; + writer_ = std::make_unique( + [this](auto&& msg, auto&& /*msgType*/) { appendOutgoingUnlocked(msg); }, stringOffsets_); + } + + void attachStream(drogon::ResponseStreamPtr stream) + { + { + std::lock_guard lock(mutex_); + if (aborted_ || responseEnded_) { + if (stream) + stream->close(); + return; + } + stream_ = std::move(stream); + } + scheduleDrain(); + } + + void parseRequestFromJson(nlohmann::json const& requestJson) + { + std::string mapId = requestJson["mapId"]; + std::string layerId = requestJson["layerId"]; + std::vector tileIds; + tileIds.reserve(requestJson["tileIds"].size()); + for (auto const& tid : requestJson["tileIds"].get>()) { + tileIds.emplace_back(tid); + } + requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); + } + + [[nodiscard]] bool setResponseTypeFromAccept(std::string_view acceptHeader, std::string& error) + { + responseType_ = std::string(acceptHeader); + if (responseType_.empty()) + responseType_ = anyMimeType; + if (responseType_ == anyMimeType) + responseType_ = binaryMimeType; + + if (responseType_ == binaryMimeType) { + trimResponseType_ = HttpService::Impl::ResponseType::Binary; + return true; + } + if (responseType_ == jsonlMimeType) { + trimResponseType_ = HttpService::Impl::ResponseType::Json; + return true; + } + + error = "Unknown Accept header value: " + responseType_; + return false; + } + + void enableGzip() { compressor_ = std::make_unique(); } + + void onAborted() + { + if (aborted_.exchange(true)) + return; + for (auto const& req : requests_) { + if (!req->isDone()) { + impl_.self_.abort(req); + } + } + drogon::ResponseStreamPtr stream; + { + std::lock_guard lock(mutex_); + if (responseEnded_.exchange(true)) + return; + stream = std::move(stream_); + } + if (stream) + stream->close(); + } + + void addResult(TileLayer::Ptr const& result) + { + { + std::lock_guard lock(mutex_); + if (aborted_) + return; + + log().debug("Response ready: {}", MapTileKey(*result).toString()); + if (responseType_ == binaryMimeType) { + writer_->write(result); + } else { + auto dumped = result->toJson().dump(-1, ' ', false, nlohmann::json::error_handler_t::ignore); + appendOutgoingUnlocked(dumped); + appendOutgoingUnlocked("\n"); + } + } + scheduleDrain(); + } + + void onRequestDone() + { + { + std::lock_guard lock(mutex_); + if (aborted_) + return; + + bool allDoneNow = + std::all_of(requests_.begin(), requests_.end(), [](auto const& r) { return r->isDone(); }); + + if (allDoneNow && !allDone_) { + allDone_ = true; + if (responseType_ == binaryMimeType && !endOfStreamSent_) { + writer_->sendEndOfStream(); + endOfStreamSent_ = true; + } + } + } + scheduleDrain(); + } + + void scheduleDrain() + { + if (aborted_ || responseEnded_) + return; + if (drainScheduled_.exchange(true)) + return; + + auto weak = weak_from_this(); + loop_->queueInLoop([weak = std::move(weak)]() mutable { + if (auto self = weak.lock()) { + self->drainOnLoop(); + } + }); + } + + void drainOnLoop() + { + drainScheduled_ = false; + if (aborted_ || responseEnded_) + return; + + constexpr size_t maxChunk = 64 * 1024; + + for (;;) { + std::string chunk; + bool done = false; + bool needAbort = false; + bool scheduleAgain = false; + drogon::ResponseStreamPtr streamToClose; + { + std::lock_guard lock(mutex_); + if (!stream_) + return; + + if (!pending_.empty()) { + size_t n = std::min(pending_.size(), maxChunk); + chunk.assign(pending_.data(), n); + pending_.erase(0, n); + } else { + if (allDone_ && compressor_ && !compressionFinished_) { + pending_.append(compressor_->finish()); + compressionFinished_ = true; + continue; + } + done = allDone_; + } + + if (!chunk.empty()) { + if (!stream_->send(chunk)) { + needAbort = true; + } else if (!pending_.empty() || allDone_) { + scheduleAgain = true; + } + } else if (done) { + responseEnded_ = true; + streamToClose = std::move(stream_); + } + } + + if (needAbort) { + onAborted(); + return; + } + + if (done) { + if (streamToClose) + streamToClose->close(); + impl_.tryMemoryTrim(trimResponseType_); + return; + } + if (scheduleAgain) + scheduleDrain(); + return; + } + } + + void appendOutgoingUnlocked(std::string_view bytes) + { + if (bytes.empty()) + return; + + if (compressor_) { + pending_.append(compressor_->compress(bytes.data(), bytes.size())); + } else { + pending_.append(bytes); + } + } + + Impl const& impl_; + trantor::EventLoop* loop_; + + std::mutex mutex_; + uint64_t requestId_ = 0; + + std::string responseType_; + HttpService::Impl::ResponseType trimResponseType_ = HttpService::Impl::ResponseType::Binary; + + std::string pending_; + drogon::ResponseStreamPtr stream_; + std::unique_ptr writer_; + std::vector requests_; + TileLayerStream::StringPoolOffsetMap stringOffsets_; + + std::unique_ptr compressor_; + bool compressionFinished_ = false; + bool endOfStreamSent_ = false; + bool allDone_ = false; + + std::atomic_bool aborted_{false}; + std::atomic_bool drainScheduled_{false}; + std::atomic_bool responseEnded_{false}; +}; + +void HttpService::Impl::handleTilesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + auto state = std::make_shared(*this, drogon::app().getLoop()); + + const std::string accept = req->getHeader("accept"); + const std::string acceptEncoding = req->getHeader("accept-encoding"); + auto clientHeaders = detail::authHeadersFromRequest(req); + + nlohmann::json j; + try { + j = nlohmann::json::parse(std::string(req->body())); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + return; + } + + auto requestsIt = j.find("requests"); + if (requestsIt == j.end() || !requestsIt->is_array()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Missing or invalid 'requests' array"); + callback(resp); + return; + } + + log().info("Processing tiles request {}", state->requestId_); + for (auto& requestJson : *requestsIt) { + state->parseRequestFromJson(requestJson); + } + + if (j.contains("stringPoolOffsets")) { + for (auto& item : j["stringPoolOffsets"].items()) { + state->stringOffsets_[item.key()] = item.value().get(); + } + } + + std::string acceptError; + if (!state->setResponseTypeFromAccept(accept, acceptError)) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::move(acceptError)); + callback(resp); + return; + } + + const bool gzip = containsGzip(acceptEncoding); + if (gzip) { + state->enableGzip(); + } + + for (auto& request : state->requests_) { + request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); + request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); + request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; + } + + const auto canProcess = self_.request(state->requests_, clientHeaders); + if (!canProcess) { + std::vector> requestStatuses{}; + bool anyUnauthorized = false; + for (auto const& r : state->requests_) { + auto status = r->getStatus(); + requestStatuses.emplace_back(static_cast>(status)); + anyUnauthorized |= (status == RequestStatus::Unauthorized); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(anyUnauthorized ? drogon::k403Forbidden : drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(nlohmann::json::object({{"status", requestStatuses}}).dump()); + callback(resp); + return; + } + + auto resp = drogon::HttpResponse::newAsyncStreamResponse( + [state](drogon::ResponseStreamPtr stream) { state->attachStream(std::move(stream)); }, + true); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeString(state->responseType_); + if (gzip) { + resp->addHeader("Content-Encoding", "gzip"); + } + callback(resp); +} + +} // namespace mapget diff --git a/libs/http-service/src/tiles-ws-controller.cpp b/libs/http-service/src/tiles-ws-controller.cpp new file mode 100644 index 00000000..6d7f5d23 --- /dev/null +++ b/libs/http-service/src/tiles-ws-controller.cpp @@ -0,0 +1,604 @@ +#include "tiles-ws-controller.h" + +#include "mapget/http-service/http-service.h" + +#include "mapget/log.h" +#include "mapget/model/stream.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "nlohmann/json.hpp" + +namespace mapget::detail +{ +namespace +{ + +[[nodiscard]] AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) +{ + AuthHeaders headers; + for (auto const& [k, v] : req->headers()) { + headers.emplace(k, v); + } + return headers; +} + +[[nodiscard]] std::string_view requestStatusToString(RequestStatus s) +{ + switch (s) { + case RequestStatus::Open: + return "Open"; + case RequestStatus::Success: + return "Success"; + case RequestStatus::NoDataSource: + return "NoDataSource"; + case RequestStatus::Unauthorized: + return "Unauthorized"; + case RequestStatus::Aborted: + return "Aborted"; + } + return "Unknown"; +} + +[[nodiscard]] std::string_view loadStateToString(TileLayer::LoadState s) +{ + switch (s) { + case TileLayer::LoadState::LoadingQueued: + return "LoadingQueued"; + case TileLayer::LoadState::BackendFetching: + return "BackendFetching"; + case TileLayer::LoadState::BackendConverting: + return "BackendConverting"; + } + return "Unknown"; +} + +[[nodiscard]] std::string encodeStreamMessage(TileLayerStream::MessageType type, std::string_view payload) +{ + std::ostringstream headerStream; + bitsery::Serializer s(headerStream); + s.object(TileLayerStream::CurrentProtocolVersion); + s.value1b(type); + s.value4b(static_cast(payload.size())); + + auto message = headerStream.str(); + message.append(payload); + return message; +} + +struct WsConnectionState +{ + AuthHeaders authHeaders; + TileLayerStream::StringPoolOffsetMap stringPoolOffsets; + std::shared_ptr session; +}; + +class TilesWsSession : public std::enable_shared_from_this +{ +public: + TilesWsSession( + HttpService& service, + std::weak_ptr conn, + std::weak_ptr connState, + AuthHeaders authHeaders, + TileLayerStream::StringPoolOffsetMap initialOffsets) + : service_(service), + loop_(drogon::app().getLoop()), + conn_(std::move(conn)), + connState_(std::move(connState)), + authHeaders_(std::move(authHeaders)), + offsets_(std::move(initialOffsets)), + writer_( + std::make_unique( + [this](std::string msg, TileLayerStream::MessageType type) { onWriterMessage(std::move(msg), type); }, + offsets_)) + { + } + + ~TilesWsSession() + { + // Best-effort cleanup: abort any in-flight requests if the session is destroyed. + cancelNoStatus(); + } + + TilesWsSession(TilesWsSession const&) = delete; + TilesWsSession& operator=(TilesWsSession const&) = delete; + + void start(const nlohmann::json& j) + { + auto requestsIt = j.find("requests"); + if (requestsIt == j.end() || !requestsIt->is_array()) { + queueStatusMessage("Missing or invalid 'requests' array"); + scheduleDrain(); + return; + } + + try { + requests_.clear(); + requests_.reserve(requestsIt->size()); + requestStatuses_.clear(); + requestStatuses_.reserve(requestsIt->size()); + + for (auto const& requestJson : *requestsIt) { + const std::string mapId = requestJson.at("mapId").get(); + const std::string layerId = requestJson.at("layerId").get(); + const auto& tileIdsJson = requestJson.at("tileIds"); + if (!tileIdsJson.is_array()) { + throw std::runtime_error("tileIds must be an array"); + } + + std::vector tileIds; + tileIds.reserve(tileIdsJson.size()); + for (auto const& tid : tileIdsJson) { + tileIds.emplace_back(tid.get()); + } + + requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); + requestStatuses_.push_back(RequestStatus::Open); + } + } + catch (const std::exception& e) { + queueStatusMessage(fmt::format("Invalid request JSON: {}", e.what())); + scheduleDrain(); + return; + } + + // Hook request callbacks before calling service_.request so early + // failures (NoDataSource/Unauthorized) still produce status updates. + const auto weak = weak_from_this(); + for (size_t i = 0; i < requests_.size(); ++i) { + auto& req = requests_[i]; + req->onFeatureLayer([weak](auto&& layer) { + if (auto self = weak.lock()) { + self->onTileLayer(std::move(layer)); + } + }); + req->onSourceDataLayer([weak](auto&& layer) { + if (auto self = weak.lock()) { + self->onTileLayer(std::move(layer)); + } + }); + req->onLayerLoadStateChanged([weak](MapTileKey const& key, TileLayer::LoadState state) { + if (auto self = weak.lock()) { + self->onLoadStateChanged(key, state); + } + }); + req->onDone_ = [weak, i](RequestStatus status) { + if (auto self = weak.lock()) { + self->onRequestDone(i, status); + } + }; + } + + // Start processing (may synchronously set request statuses). + (void)service_.request(requests_, authHeaders_); + + { + std::lock_guard lock(mutex_); + statusEmissionEnabled_ = true; + } + queueStatusMessage({}); + scheduleDrain(); + } + + void cancel(std::string reason) + { + cancelled_ = true; + + // Stop sending any queued tile frames from this session. + { + std::lock_guard lock(mutex_); + outgoing_.clear(); + } + + // Abort in-flight requests (best-effort). + for (auto const& r : requests_) { + if (!r || r->isDone()) + continue; + service_.abort(r); + } + + // Refresh locally cached statuses after aborting. + { + std::lock_guard lock(mutex_); + for (size_t i = 0; i < requests_.size() && i < requestStatuses_.size(); ++i) { + if (requests_[i]) { + requestStatuses_[i] = requests_[i]->getStatus(); + } + } + } + + queueStatusMessage(std::move(reason)); + scheduleDrain(); + } + +private: + struct OutgoingFrame + { + std::string bytes; + std::optional> stringPoolCommit; + }; + + struct WriterMessage + { + std::string bytes; + TileLayerStream::MessageType type{TileLayerStream::MessageType::None}; + }; + + void cancelNoStatus() + { + if (cancelled_.exchange(true)) + return; + + // Ensure we stop emitting any further frames. + { + std::lock_guard lock(mutex_); + outgoing_.clear(); + } + + for (auto const& r : requests_) { + if (!r || r->isDone()) + continue; + service_.abort(r); + } + } + + void onWriterMessage(std::string msg, TileLayerStream::MessageType type) + { + // Writer messages are only generated from within onTileLayer under mutex_. + if (!currentWriteBatch_) { + raise("TilesWsSession writer callback used out-of-band"); + } + currentWriteBatch_->push_back(WriterMessage{std::move(msg), type}); + } + + void onTileLayer(TileLayer::Ptr layer) + { + if (cancelled_) + return; + if (!layer) + return; + + std::vector batch; + std::optional> stringPoolCommit; + + { + std::lock_guard lock(mutex_); + if (cancelled_) + return; + + currentWriteBatch_ = &batch; + writer_->write(layer); + currentWriteBatch_ = nullptr; + + // If a StringPool message was generated, the writer updates offsets_ + // to the new highest string ID for this node after emitting it. + const auto nodeId = layer->nodeId(); + const auto it = offsets_.find(nodeId); + if (it != offsets_.end()) { + const auto newOffset = it->second; + for (auto const& m : batch) { + if (m.type == TileLayerStream::MessageType::StringPool) { + stringPoolCommit = std::make_pair(nodeId, newOffset); + break; + } + } + } + + for (auto& m : batch) { + OutgoingFrame frame; + frame.bytes = std::move(m.bytes); + if (m.type == TileLayerStream::MessageType::StringPool) { + frame.stringPoolCommit = stringPoolCommit; + } + outgoing_.push_back(std::move(frame)); + } + } + + scheduleDrain(); + } + + void onRequestDone(size_t requestIndex, RequestStatus status) + { + if (cancelled_) + return; + + bool shouldEmit = false; + { + std::lock_guard lock(mutex_); + if (cancelled_) + return; + if (requestIndex >= requestStatuses_.size()) + return; + if (requestStatuses_[requestIndex] == status) + return; + requestStatuses_[requestIndex] = status; + shouldEmit = statusEmissionEnabled_; + } + + if (shouldEmit) { + queueStatusMessage({}); + scheduleDrain(); + } + } + + void queueStatusMessage(std::string message) + { + OutgoingFrame frame; + frame.bytes = encodeStreamMessage(TileLayerStream::MessageType::Status, buildStatusPayload(std::move(message))); + { + std::lock_guard lock(mutex_); + outgoing_.push_back(std::move(frame)); + } + } + + void onLoadStateChanged(MapTileKey const& key, TileLayer::LoadState state) + { + if (cancelled_) + return; + + OutgoingFrame frame; + frame.bytes = encodeStreamMessage( + TileLayerStream::MessageType::LoadStateChange, + buildLoadStatePayload(key, state)); + { + std::lock_guard lock(mutex_); + outgoing_.push_back(std::move(frame)); + } + scheduleDrain(); + } + + [[nodiscard]] std::string buildStatusPayload(std::string message) + { + nlohmann::json requestsJson = nlohmann::json::array(); + bool allDone = true; + + { + std::lock_guard lock(mutex_); + for (size_t i = 0; i < requests_.size(); ++i) { + const auto status = (i < requestStatuses_.size()) ? requestStatuses_[i] : RequestStatus::Open; + allDone &= (status != RequestStatus::Open); + + nlohmann::json reqJson = nlohmann::json::object(); + reqJson["index"] = i; + if (i < requests_.size() && requests_[i]) { + reqJson["mapId"] = requests_[i]->mapId_; + reqJson["layerId"] = requests_[i]->layerId_; + } else { + reqJson["mapId"] = ""; + reqJson["layerId"] = ""; + } + reqJson["status"] = static_cast>(status); + reqJson["statusText"] = std::string(requestStatusToString(status)); + requestsJson.push_back(std::move(reqJson)); + } + } + + return nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", allDone}, + {"requests", std::move(requestsJson)}, + {"message", std::move(message)}, + }).dump(); + } + + [[nodiscard]] std::string buildLoadStatePayload(MapTileKey const& key, TileLayer::LoadState state) + { + return nlohmann::json::object({ + {"type", "mapget.tiles.load-state"}, + {"mapId", key.mapId_}, + {"layerId", key.layerId_}, + {"tileId", key.tileId_.value_}, + {"state", static_cast(state)}, + {"stateText", std::string(loadStateToString(state))}, + }).dump(); + } + + void scheduleDrain() + { + if (drainScheduled_.exchange(true)) + return; + + auto weak = weak_from_this(); + loop_->queueInLoop([weak = std::move(weak)]() mutable { + if (auto self = weak.lock()) { + self->drainOnLoop(); + } + }); + } + + void drainOnLoop() + { + drainScheduled_ = false; + + auto conn = conn_.lock(); + if (!conn || conn->disconnected()) { + cancelNoStatus(); + return; + } + + constexpr size_t maxFramesPerDrain = 256; + for (size_t i = 0; i < maxFramesPerDrain; ++i) { + OutgoingFrame frame; + { + std::lock_guard lock(mutex_); + if (outgoing_.empty()) { + break; + } + frame = std::move(outgoing_.front()); + outgoing_.pop_front(); + } + + conn->send(frame.bytes, drogon::WebSocketMessageType::Binary); + if (frame.stringPoolCommit) { + if (auto state = connState_.lock()) { + state->stringPoolOffsets[frame.stringPoolCommit->first] = frame.stringPoolCommit->second; + } + } + } + + { + std::lock_guard lock(mutex_); + if (outgoing_.empty()) + return; + } + scheduleDrain(); + } + + HttpService& service_; + trantor::EventLoop* loop_; + std::weak_ptr conn_; + std::weak_ptr connState_; + + AuthHeaders authHeaders_; + + std::mutex mutex_; + std::deque outgoing_; + + std::vector requests_; + std::vector requestStatuses_; + bool statusEmissionEnabled_ = false; + + TileLayerStream::StringPoolOffsetMap offsets_; + std::unique_ptr writer_; + std::vector* currentWriteBatch_ = nullptr; + + std::atomic_bool drainScheduled_{false}; + std::atomic_bool cancelled_{false}; +}; + +class TilesWebSocketController final : public drogon::WebSocketController +{ +public: + explicit TilesWebSocketController(HttpService& service) : service_(service) {} + + void handleNewConnection(const drogon::HttpRequestPtr& req, const drogon::WebSocketConnectionPtr& conn) override + { + auto state = std::make_shared(); + state->authHeaders = authHeadersFromRequest(req); + conn->setContext(std::move(state)); + } + + void handleNewMessage( + const drogon::WebSocketConnectionPtr& conn, + std::string&& message, + const drogon::WebSocketMessageType& type) override + { + auto state = conn->getContext(); + if (!state) { + state = std::make_shared(); + conn->setContext(state); + } + + if (type != drogon::WebSocketMessageType::Text) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", "Expected a text message containing JSON."}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + + nlohmann::json j; + try { + j = nlohmann::json::parse(message); + } + catch (const std::exception& e) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", fmt::format("Invalid JSON: {}", e.what())}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + + // Patch per-connection string pool offsets if supplied. + if (j.contains("stringPoolOffsets")) { + if (!j["stringPoolOffsets"].is_object()) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", "stringPoolOffsets must be an object."}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + try { + for (auto const& item : j["stringPoolOffsets"].items()) { + state->stringPoolOffsets[item.key()] = item.value().get(); + } + } + catch (const std::exception& e) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", fmt::format("Invalid stringPoolOffsets: {}", e.what())}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + } + + if (state->session) { + state->session->cancel("Replaced by a new /tiles WebSocket request."); + state->session.reset(); + } + + state->session = std::make_shared( + service_, + conn, + state, + state->authHeaders, + state->stringPoolOffsets); + state->session->start(j); + } + + void handleConnectionClosed(const drogon::WebSocketConnectionPtr& conn) override + { + if (auto state = conn->getContext()) { + if (state->session) { + state->session->cancel("WebSocket connection closed."); + } + } + } + + WS_PATH_LIST_BEGIN + WS_PATH_ADD("/tiles", drogon::Get); + WS_PATH_LIST_END + +private: + HttpService& service_; +}; + +} // namespace + +void registerTilesWebSocketController(drogon::HttpAppFramework& app, HttpService& service) +{ + app.registerController(std::make_shared(service)); +} + +} // namespace mapget::detail diff --git a/libs/http-service/src/tiles-ws-controller.h b/libs/http-service/src/tiles-ws-controller.h new file mode 100644 index 00000000..4acaed45 --- /dev/null +++ b/libs/http-service/src/tiles-ws-controller.h @@ -0,0 +1,19 @@ +#pragma once + +namespace drogon +{ +class HttpAppFramework; +} + +namespace mapget +{ +class HttpService; +} + +namespace mapget::detail +{ + +void registerTilesWebSocketController(drogon::HttpAppFramework& app, HttpService& service); + +} // namespace mapget::detail + diff --git a/libs/model/include/mapget/model/featurelayer.h b/libs/model/include/mapget/model/featurelayer.h index 97c4f569..07888c60 100644 --- a/libs/model/include/mapget/model/featurelayer.h +++ b/libs/model/include/mapget/model/featurelayer.h @@ -20,6 +20,7 @@ #include "geometry.h" #include "sourcedatareference.h" #include "pointnode.h" +#include "nlohmann/json.hpp" namespace mapget { @@ -209,6 +210,9 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool /** Convert to (Geo-) JSON. */ nlohmann::json toJson() const override; + /** Report serialized size stats for feature-layer data and model-pool columns. */ + [[nodiscard]] nlohmann::json serializationSizeStats() const; + /** Access number of stored features */ size_t size() const; diff --git a/libs/model/include/mapget/model/layer.h b/libs/model/include/mapget/model/layer.h index 5a2acfa5..0f383fa4 100644 --- a/libs/model/include/mapget/model/layer.h +++ b/libs/model/include/mapget/model/layer.h @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace simfil { struct StringPool; } @@ -85,6 +86,12 @@ class TileLayer { public: using Ptr = std::shared_ptr; + enum class LoadState : uint8_t { + LoadingQueued = 0, + BackendFetching = 1, + BackendConverting = 2 + }; + using LoadStateCallback = std::function; /** * Constructor that takes tileId_, nodeId_, mapId_, layerInfo_, @@ -192,6 +199,15 @@ class TileLayer virtual tl::expected write(std::ostream& outputStream); virtual nlohmann::json toJson() const; + /** + * Set a load-state callback. Used by the service to forward state changes. + * Not serialized with the tile. + */ + void setLoadStateCallback(LoadStateCallback cb); + + /** Emit a load-state change (if a callback is registered). */ + void setLoadState(LoadState state); + protected: Version mapVersion_{0, 0, 0}; TileId tileId_; @@ -204,6 +220,7 @@ class TileLayer std::optional ttl_; nlohmann::json info_; std::optional legalInfo_; // Copyright-related information + LoadStateCallback onLoadStateChanged_; }; } diff --git a/libs/model/include/mapget/model/stream.h b/libs/model/include/mapget/model/stream.h index 4568e117..5eb0a5a5 100644 --- a/libs/model/include/mapget/model/stream.h +++ b/libs/model/include/mapget/model/stream.h @@ -29,13 +29,37 @@ class TileLayerStream StringPool = 1, TileFeatureLayer = 2, TileSourceDataLayer = 3, + /** + * JSON-encoded status updates, e.g. for WebSocket /tiles. + * + * Payload: UTF-8 JSON bytes (not null-terminated). + */ + Status = 4, + /** + * JSON-encoded load-state updates for individual tiles. + * + * Payload: UTF-8 JSON bytes (not null-terminated). + */ + LoadStateChange = 5, EndOfStream = 128 }; struct StringPoolCache; - /** Protocol Version which parsed blobs must be compatible with. */ - static constexpr Version CurrentProtocolVersion{0, 1, 1}; + /** + * Protocol Version which parsed blobs must be compatible with. + * Version History: + * - Version 1.0: + * + Added TileFeatureLayer Message + * + Added StringPool Message + * + Added TileSourceDataLayer Message + * + Added EndOfStream Message + * - Version 1.1: + * + Added errorCode field to TileLayer + * + Added Status Message + * + Added LoadStateChange Message + */ + static constexpr Version CurrentProtocolVersion{1, 1, 0}; /** Map to keep track of the highest sent string id per datasource node. */ using StringPoolOffsetMap = std::unordered_map; diff --git a/libs/model/src/featurelayer.cpp b/libs/model/src/featurelayer.cpp index 826883b4..e5cb6a56 100644 --- a/libs/model/src/featurelayer.cpp +++ b/libs/model/src/featurelayer.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -76,6 +77,39 @@ namespace throw std::out_of_range("Size out of range"); return (index << SourceAddressArenaSizeBits) | size; } + + class CountingStreambuf : public std::streambuf + { + public: + size_t size() const { return size_; } + + protected: + std::streamsize xsputn(const char* /*s*/, std::streamsize count) override + { + size_ += static_cast(count); + return count; + } + + int overflow(int ch) override + { + if (ch != EOF) + ++size_; + return ch; + } + + private: + size_t size_ = 0; + }; + + template + size_t measureBytes(Fn&& fn) + { + CountingStreambuf buf; + std::ostream os(&buf); + bitsery::Serializer s(os); + fn(s); + return buf.size(); + } } namespace mapget @@ -302,7 +336,7 @@ simfil::model_ptr TileFeatureLayer::newFeature( // contains only references to feature nodes, in the order // of the feature node column. addRoot(simfil::ModelNode::Ptr(result)); - setInfo("num-features", numRoots()); + setInfo("Size/Features", numRoots()); return result; } @@ -841,6 +875,64 @@ nlohmann::json TileFeatureLayer::toJson() const return result; } +nlohmann::json TileFeatureLayer::serializationSizeStats() const +{ + constexpr size_t maxColumnSize = std::numeric_limits::max(); + auto featureLayer = nlohmann::json::object(); + + featureLayer["features"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->features_, maxColumnSize); })); + featureLayer["attributes"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->attributes_, maxColumnSize); })); + featureLayer["validities"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->validities_, maxColumnSize); })); + featureLayer["feature-ids"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->featureIds_, maxColumnSize); })); + featureLayer["attribute-layers"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->attrLayers_, maxColumnSize); })); + featureLayer["attribute-layer-lists"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->attrLayerLists_, maxColumnSize); })); + featureLayer["feature-id-prefix"] = static_cast(measureBytes( + [&](auto& s) { s.object(impl_->featureIdPrefix_); })); + featureLayer["relations"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->relations_, maxColumnSize); })); + featureLayer["feature-hash-index"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->featureHashIndex_, maxColumnSize); })); + featureLayer["geometries"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->geom_, maxColumnSize); })); + featureLayer["point-buffers"] = static_cast(measureBytes( + [&](auto& s) { s.ext(impl_->pointBuffers_, bitsery::ext::ArrayArenaExt{}); })); + featureLayer["source-data-references"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->sourceDataReferences_, maxColumnSize); })); + + int64_t featureLayerTotal = 0; + for (const auto& [_, value] : featureLayer.items()) { + if (value.is_number_integer()) + featureLayerTotal += value.get(); + } + + auto modelStats = ModelPool::serializationSizeStats(); + auto modelPool = nlohmann::json::object({ + {"roots", static_cast(modelStats.rootsBytes)}, + {"int64", static_cast(modelStats.int64Bytes)}, + {"double", static_cast(modelStats.doubleBytes)}, + {"string-data", static_cast(modelStats.stringDataBytes)}, + {"string-ranges", static_cast(modelStats.stringRangeBytes)}, + {"object-members", static_cast(modelStats.objectMemberBytes)}, + {"array-members", static_cast(modelStats.arrayMemberBytes)}, + }); + + int64_t modelPoolTotal = static_cast(modelStats.totalBytes()); + + return { + {"feature-layer", featureLayer}, + {"model-pool", modelPool}, + {"feature-layer-total-bytes", featureLayerTotal}, + {"model-pool-total-bytes", modelPoolTotal}, + {"total-bytes", featureLayerTotal + modelPoolTotal} + }; +} + size_t TileFeatureLayer::size() const { return numRoots(); diff --git a/libs/model/src/layer.cpp b/libs/model/src/layer.cpp index be94dbac..199851c7 100644 --- a/libs/model/src/layer.cpp +++ b/libs/model/src/layer.cpp @@ -243,6 +243,18 @@ void TileLayer::setLegalInfo(const std::string& legalInfoString) legalInfo_ = legalInfoString; } +void TileLayer::setLoadStateCallback(LoadStateCallback cb) +{ + onLoadStateChanged_ = std::move(cb); +} + +void TileLayer::setLoadState(LoadState state) +{ + if (onLoadStateChanged_) { + onLoadStateChanged_(state); + } +} + tl::expected TileLayer::write(std::ostream& outputStream) { using namespace std::chrono; diff --git a/libs/model/src/stream.cpp b/libs/model/src/stream.cpp index f7d78c18..f59da85d 100644 --- a/libs/model/src/stream.cpp +++ b/libs/model/src/stream.cpp @@ -50,7 +50,6 @@ bool TileLayerStream::Reader::continueReading() } } - bitsery::Deserializer s(buffer_); auto numUnreadBytes = buffer_.tellp() - buffer_.tellg(); if (numUnreadBytes < nextValueSize_) return false; @@ -80,6 +79,12 @@ bool TileLayerStream::Reader::continueReading() std::string stringPoolNodeId = StringPool::readDataSourceNodeId(buffer_); stringPoolProvider_->getStringPool(stringPoolNodeId)->read(buffer_); } + else + { + // Skip unknown message types for forward compatibility (e.g. status + // messages on WebSocket streams). + buffer_.seekg(nextValueSize_, std::ios_base::cur); + } currentPhase_ = Phase::ReadHeader; return true; diff --git a/libs/pymapget/CMakeLists.txt b/libs/pymapget/CMakeLists.txt index 8d9a2016..fb7b8066 100644 --- a/libs/pymapget/CMakeLists.txt +++ b/libs/pymapget/CMakeLists.txt @@ -28,17 +28,7 @@ target_compile_features(pymapget INTERFACE cxx_std_17) -FetchContent_GetProperties(cpp-httplib) -if (MSVC AND CPP-HTTPLIB_POPULATED) - # Required because cpp-httplib speaks https via OpenSSL. - # Only needed if httplib came via FetchContent. - set(DEPLOY_FILES - "${OPENSSL_INCLUDE_DIR}/../libcrypto-1_1-x64.dll" - "${OPENSSL_INCLUDE_DIR}/../libssl-1_1-x64.dll" - "${CMAKE_CURRENT_LIST_DIR}/__main__.py") -else() - set(DEPLOY_FILES "${CMAKE_CURRENT_LIST_DIR}/__main__.py") -endif() +set(DEPLOY_FILES "${CMAKE_CURRENT_LIST_DIR}/__main__.py") add_wheel(pymapget NAME mapget diff --git a/libs/service/include/mapget/service/cache.h b/libs/service/include/mapget/service/cache.h index bee02b1d..50d92a28 100644 --- a/libs/service/include/mapget/service/cache.h +++ b/libs/service/include/mapget/service/cache.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "mapget/model/info.h" #include "mapget/model/featurelayer.h" @@ -29,6 +30,7 @@ class Cache : public TileLayerStream::StringPoolCache, public std::enable_shared }; using Ptr = std::shared_ptr; + using TileBlobVisitor = std::function; // The following methods are already implemented, // they forward to the virtual methods on-demand. @@ -52,6 +54,9 @@ class Cache : public TileLayerStream::StringPoolCache, public std::enable_shared /** Abstract: Upsert (update or insert) a TileLayer blob. */ virtual void putTileLayerBlob(MapTileKey const& k, std::string const& v) = 0; + /** Abstract: Iterate through all cached tile layer blobs. */ + virtual void forEachTileLayerBlob(const TileBlobVisitor& cb) const = 0; + /** Abstract: Retrieve a string-pool blob for a sourceNodeId. */ virtual std::optional getStringPoolBlob(std::string_view const& sourceNodeId) = 0; diff --git a/libs/service/include/mapget/service/datasource.h b/libs/service/include/mapget/service/datasource.h index a76c172e..dadf5ae7 100644 --- a/libs/service/include/mapget/service/datasource.h +++ b/libs/service/include/mapget/service/datasource.h @@ -58,7 +58,11 @@ class DataSource virtual std::vector locate(LocateRequest const& req); /** Called by mapget::Service worker. Dispatches to Cache or fill(...) on miss. */ - virtual TileLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info); + virtual TileLayer::Ptr get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback = {}); /** Add an authorization header-regex pair for this datasource. */ void requireAuthHeaderRegexMatchOption(std::string header, std::regex re); diff --git a/libs/service/include/mapget/service/memcache.h b/libs/service/include/mapget/service/memcache.h index 283c0e11..bdc2b452 100644 --- a/libs/service/include/mapget/service/memcache.h +++ b/libs/service/include/mapget/service/memcache.h @@ -29,6 +29,9 @@ class MemCache : public Cache /** Upsert a TileLayer blob. */ void putTileLayerBlob(MapTileKey const& k, std::string const& v) override; + /** Iterate over cached tile layer blobs. */ + void forEachTileLayerBlob(const TileBlobVisitor& cb) const override; + /** Retrieve a string-pool blob for a sourceNodeId -> No-Op */ std::optional getStringPoolBlob(std::string_view const& sourceNodeId) override {return {};} @@ -40,10 +43,10 @@ class MemCache : public Cache private: // Cached tile blobs. - std::shared_mutex cacheMutex_; + mutable std::shared_mutex cacheMutex_; std::unordered_map cachedTiles_; std::deque fifo_; uint32_t maxCachedTiles_ = 0; }; -} \ No newline at end of file +} diff --git a/libs/service/include/mapget/service/nullcache.h b/libs/service/include/mapget/service/nullcache.h index 00aa8e1a..66e29b71 100644 --- a/libs/service/include/mapget/service/nullcache.h +++ b/libs/service/include/mapget/service/nullcache.h @@ -20,6 +20,9 @@ class NullCache : public Cache /** Upsert a TileLayer blob - does nothing. */ void putTileLayerBlob(MapTileKey const& k, std::string const& v) override; + /** Iterate cached tile blobs - no-op. */ + void forEachTileLayerBlob(const TileBlobVisitor& cb) const override; + /** Retrieve a string-pool blob for a sourceNodeId - always returns empty. */ std::optional getStringPoolBlob(std::string_view const& sourceNodeId) override; @@ -27,4 +30,4 @@ class NullCache : public Cache void putStringPoolBlob(std::string_view const& sourceNodeId, std::string const& v) override; }; -} \ No newline at end of file +} diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index 79c9689e..cf1cb085 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace mapget { @@ -76,8 +77,15 @@ class LayerTilesRequest template LayerTilesRequest& onSourceDataLayer(Fun&& callback) { onSourceDataLayer_ = std::forward(callback); return *this; } + /** + * Callback for per-tile load-state changes. + */ + template + LayerTilesRequest& onLayerLoadStateChanged(Fun&& callback) { onLoadStateChanged_ = std::forward(callback); return *this; } + protected: virtual void notifyResult(TileLayer::Ptr); + void notifyLoadState(MapTileKey const& key, TileLayer::LoadState state) const; void setStatus(RequestStatus s); void notifyStatus(); nlohmann::json toJson(); @@ -88,18 +96,22 @@ class LayerTilesRequest */ std::function onFeatureLayer_; std::function onSourceDataLayer_; + std::function onLoadStateChanged_; // So the service can track which tileId index from tiles_ // is next in line to be processed. size_t nextTileIndex_ = 0; + // Track which tiles still need to be scheduled/served for this request. + std::set tileIdsNotStarted_; + // So the requester can track how many results have been received. size_t resultCount_ = 0; // Mutex/condition variable for reading/setting request status. std::mutex statusMutex_; std::condition_variable statusConditionVariable_; - RequestStatus status_ = RequestStatus::Open; + std::atomic status_ = RequestStatus::Open; }; /** diff --git a/libs/service/include/mapget/service/sqlitecache.h b/libs/service/include/mapget/service/sqlitecache.h index 55b42879..85387dcd 100644 --- a/libs/service/include/mapget/service/sqlitecache.h +++ b/libs/service/include/mapget/service/sqlitecache.h @@ -25,6 +25,7 @@ class SQLiteCache : public Cache std::optional getTileLayerBlob(MapTileKey const& k) override; void putTileLayerBlob(MapTileKey const& k, std::string const& v) override; + void forEachTileLayerBlob(const TileBlobVisitor& cb) const override; std::optional getStringPoolBlob(std::string_view const& sourceNodeId) override; void putStringPoolBlob(std::string_view const& sourceNodeId, std::string const& v) override; @@ -53,4 +54,4 @@ class SQLiteCache : public Cache } stmts_; }; -} // namespace mapget \ No newline at end of file +} // namespace mapget diff --git a/libs/service/src/datasource.cpp b/libs/service/src/datasource.cpp index 2e15d0a6..319c77c0 100644 --- a/libs/service/src/datasource.cpp +++ b/libs/service/src/datasource.cpp @@ -1,4 +1,6 @@ #include "datasource.h" +#include +#include #include #include #include @@ -8,7 +10,11 @@ namespace mapget { -TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourceInfo const& info) +TileLayer::Ptr DataSource::get( + const MapTileKey& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback) { auto layerInfo = info.getLayer(k.layerId_); if (!layerInfo) @@ -25,6 +31,9 @@ TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourc info.mapId_, info.getLayer(k.layerId_), cache->getStringPool(info.nodeId_)); + if (loadStateCallback) { + tileFeatureLayer->setLoadStateCallback(loadStateCallback); + } fill(tileFeatureLayer); result = tileFeatureLayer; break; @@ -36,6 +45,9 @@ TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourc info.mapId_, info.getLayer(k.layerId_), cache->getStringPool(info.nodeId_)); + if (loadStateCallback) { + tileSourceDataLayer->setLoadStateCallback(loadStateCallback); + } fill(tileSourceDataLayer); result = tileSourceDataLayer; break; @@ -47,13 +59,14 @@ TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourc // Notify the tile how long it took to fill. if (result) { auto duration = std::chrono::steady_clock::now() - start; - result->setInfo("fill-time-ms", std::chrono::duration_cast(duration).count()); + result->setInfo("Load+Convert/Total#ms", std::chrono::duration_cast(duration).count()); } return result; } void DataSource::requireAuthHeaderRegexMatchOption(std::string header, std::regex re) { + std::ranges::transform(header, header.begin(), [](unsigned char c) { return (char)std::tolower(c); }); authHeaderAlternatives_.insert({std::move(header), std::move(re)}); } @@ -64,7 +77,9 @@ bool DataSource::isDataSourceAuthorized( return true; for (auto const& [k, v] : clientHeaders) { - auto authHeaderPatternIt = authHeaderAlternatives_.find(k); + auto key = k; + std::ranges::transform(key, key.begin(), [](unsigned char c) { return (char)std::tolower(c); }); + auto authHeaderPatternIt = authHeaderAlternatives_.find(key); if (authHeaderPatternIt != authHeaderAlternatives_.end()) { if (std::regex_match(v, authHeaderPatternIt->second)) { return true; diff --git a/libs/service/src/memcache.cpp b/libs/service/src/memcache.cpp index d9a49dc8..f3d95a88 100644 --- a/libs/service/src/memcache.cpp +++ b/libs/service/src/memcache.cpp @@ -31,6 +31,14 @@ void MemCache::putTileLayerBlob(const MapTileKey& k, const std::string& v) } } +void MemCache::forEachTileLayerBlob(const TileBlobVisitor& cb) const +{ + std::shared_lock cacheLock(cacheMutex_); + for (const auto& [key, value] : cachedTiles_) { + cb(MapTileKey(key), value); + } +} + nlohmann::json MemCache::getStatistics() const { auto result = Cache::getStatistics(); result["memcache-map-size"] = (int64_t)cachedTiles_.size(); @@ -38,4 +46,4 @@ nlohmann::json MemCache::getStatistics() const { return result; } -} \ No newline at end of file +} diff --git a/libs/service/src/nullcache.cpp b/libs/service/src/nullcache.cpp index 0862a5ea..e1dc99e6 100644 --- a/libs/service/src/nullcache.cpp +++ b/libs/service/src/nullcache.cpp @@ -13,6 +13,11 @@ void NullCache::putTileLayerBlob(MapTileKey const& k, std::string const& v) // Do nothing - no caching } +void NullCache::forEachTileLayerBlob(const TileBlobVisitor& cb) const +{ + // No cached tiles. +} + std::optional NullCache::getStringPoolBlob(std::string_view const& sourceNodeId) { return std::nullopt; @@ -23,4 +28,4 @@ void NullCache::putStringPoolBlob(std::string_view const& sourceNodeId, std::str // Do nothing - no caching } -} \ No newline at end of file +} diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index c8449d8c..aa066c65 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -11,12 +11,12 @@ #include #include -#include #include #include #include #include #include +#include #include "simfil/types.h" @@ -31,6 +31,7 @@ LayerTilesRequest::LayerTilesRequest( layerId_(std::move(layerId)), tiles_(std::move(tiles)) { + tileIdsNotStarted_.insert(tiles_.begin(), tiles_.end()); if (tiles_.empty()) { // An empty request is always set to success, but the client/service // is responsible for triggering notifyStatus() in that case. @@ -39,18 +40,22 @@ LayerTilesRequest::LayerTilesRequest( } void LayerTilesRequest::notifyResult(TileLayer::Ptr r) { + if (isDone()) { + return; + } + const auto type = r->layerInfo()->type_; switch (type) { - case mapget::LayerType::Features: + case LayerType::Features: if (onFeatureLayer_) - onFeatureLayer_(std::move(std::static_pointer_cast(r))); + onFeatureLayer_(std::move(std::static_pointer_cast(r))); break; - case mapget::LayerType::SourceData: + case LayerType::SourceData: if (onSourceDataLayer_) - onSourceDataLayer_(std::move(std::static_pointer_cast(r))); + onSourceDataLayer_(std::move(std::static_pointer_cast(r))); break; default: - mapget::log().error(fmt::format("Unhandled layer type {}, no matching callback!", static_cast(type))); + log().error(fmt::format("Unhandled layer type {}, no matching callback!", static_cast(type))); break; } @@ -60,12 +65,15 @@ void LayerTilesRequest::notifyResult(TileLayer::Ptr r) { } } +void LayerTilesRequest::notifyLoadState(MapTileKey const& key, TileLayer::LoadState state) const { + if (onLoadStateChanged_) { + onLoadStateChanged_(key, state); + } +} + void LayerTilesRequest::setStatus(RequestStatus s) { - { - std::unique_lock statusLock(statusMutex_); - this->status_ = s; - } + this->status_ = s; notifyStatus(); } @@ -112,13 +120,16 @@ bool LayerTilesRequest::isDone() struct Service::Controller { + virtual ~Controller() = default; + struct Job { MapTileKey tileKey; - LayerTilesRequest::Ptr request; + std::vector waitingRequests; std::optional cacheExpiredAt; + TileLayer::LoadState loadStatus = TileLayer::LoadState::LoadingQueued; }; - std::set jobsInProgress_; // Set of jobs currently in progress + std::map> jobsInProgress_; // Jobs currently in progress + interested requests Cache::Ptr cache_; // The cache for the service std::optional defaultTtl_; // Default TTL applied when datasource does not override std::list requests_; // List of requests currently being processed @@ -127,78 +138,111 @@ struct Service::Controller explicit Controller(Cache::Ptr cache, std::optional defaultTtl) : cache_(std::move(cache)), - defaultTtl_(std::move(defaultTtl)) + defaultTtl_(defaultTtl) { if (!cache_) raise("Cache must not be null!"); } - std::optional nextJob(DataSourceInfo const& i) + std::shared_ptr nextJob(DataSourceInfo const& i, std::unique_lock& lock) { // Workers call the nextJob function when they are free. // Note: For thread safety, jobsMutex_ must be held - // when calling this function. + // when calling this function. The lock may be released/re-acquired + // between sweeps to allow external updates. - std::optional result; + std::shared_ptr result; // Return next job, if available. - bool cachedTilesServed = false; + bool cachedTilesServedOrInProgressSkipped = false; + bool anyTasksRemaining = false; do { - cachedTilesServed = false; + cachedTilesServedOrInProgressSkipped = false; + anyTasksRemaining = false; for (auto reqIt = requests_.begin(); reqIt != requests_.end(); ++reqIt) { auto const& request = *reqIt; auto layerIt = i.layers_.find(request->layerId_); - // Are there tiles left to be processed in the request? - if (request->mapId_ == i.mapId_ && layerIt != i.layers_.end()) { - if (request->nextTileIndex_ >= request->tiles_.size()) { - continue; + // Does the Datasource Info (i) of the worker fit the request? + // Or is it done (/aborted) but not yet removed from requests? + if (request->mapId_ != i.mapId_ || layerIt == i.layers_.end() || request->isDone()) + continue; + + // Find the next pending tile in the request's ordered list. + TileId tileId{}; + bool foundTile = false; + while (request->nextTileIndex_ < request->tiles_.size()) { + // Skip over tiles which were meanwhile done by other workers. + tileId = request->tiles_[request->nextTileIndex_++]; + if (request->tileIdsNotStarted_.find(tileId) != request->tileIdsNotStarted_.end()) { + foundTile = true; + break; } + } + if (!foundTile) + continue; + anyTasksRemaining = true; + auto resultTileKey = MapTileKey(layerIt->second->type_, request->mapId_, request->layerId_, tileId); + + // Cache lookup. + auto cachedResult = cache_->getTileLayer(resultTileKey, i); + if (cachedResult.tile) { + request->tileIdsNotStarted_.erase(tileId); + log().debug("Serving cached tile: {}", resultTileKey.toString()); + request->notifyResult(cachedResult.tile); + cachedTilesServedOrInProgressSkipped = true; + continue; + } - // Create result wrapper object. - auto tileId = request->tiles_[request->nextTileIndex_++]; - result = Job{MapTileKey(), request, std::nullopt}; - result->tileKey.layer_ = layerIt->second->type_; - result->tileKey.mapId_ = request->mapId_; - result->tileKey.layerId_ = request->layerId_; - result->tileKey.tileId_ = tileId; - - // Cache lookup. - auto cachedResult = cache_->getTileLayer(result->tileKey, i); - if (cachedResult.tile) { - log().debug("Serving cached tile: {}", result->tileKey.toString()); - request->notifyResult(cachedResult.tile); - result.reset(); - cachedTilesServed = true; + // If another worker is working on this tile, ensure that this request gets it as well. + if (auto inProgress = jobsInProgress_.find(resultTileKey); + inProgress != jobsInProgress_.end()) { + // This tile is already being processed. Register interest so the result + // can satisfy multiple requests, and allow this request to advance. + log().debug("Joining tile with job in progress: {}", + resultTileKey.toString()); + request->tileIdsNotStarted_.erase(tileId); + request->notifyLoadState(resultTileKey, inProgress->second->loadStatus); + inProgress->second->waitingRequests.push_back(request); + cachedTilesServedOrInProgressSkipped = true; + continue; + } + + // We found something to work on that is not cached and not in progress - + // enter it into the jobs-in-progress map with the requesting client. + request->tileIdsNotStarted_.erase(tileId); + result = std::make_shared(Job{resultTileKey, {request}, cachedResult.expiredAt}); + // Proactively attach other requests that need this tile. + for (auto const& otherRequest : requests_) { + if (!otherRequest || otherRequest == request) continue; - } - result->cacheExpiredAt = cachedResult.expiredAt; - - if (jobsInProgress_.find(result->tileKey) != jobsInProgress_.end()) { - // Don't work on something that is already being worked on. - // Wait for the work to finish, then send the (hopefully cached) result. - log().debug("Delaying tile with job in progress: {}", - result->tileKey.toString()); - --request->nextTileIndex_; - result.reset(); + if (otherRequest->mapId_ != request->mapId_ || otherRequest->layerId_ != request->layerId_) continue; - } + if (otherRequest->tileIdsNotStarted_.erase(tileId) == 0) + continue; + result->waitingRequests.push_back(otherRequest); + } + jobsInProgress_.emplace(result->tileKey, result); - // Enter into the jobs-in-progress set. - jobsInProgress_.insert(result->tileKey); + // Move this request to the end of the list, so others gain priority. + // It is ok to manipulate the list here, because we call `break` after the next line. + requests_.splice(requests_.end(), requests_, reqIt); - // Move this request to the end of the list, so others gain priority. - requests_.splice(requests_.end(), requests_, reqIt); + log().debug("Working on tile: {}", result->tileKey.toString()); + break; + } - log().debug("Working on tile: {}", result->tileKey.toString()); - break; - } + if (cachedTilesServedOrInProgressSkipped && !result && anyTasksRemaining) { + // Unlock and re-lock before we make another sweep over the request list, + // so that it can be updated externally; clients might want to add/remove requests. + lock.unlock(); + lock.lock(); } } - while (cachedTilesServed && !result); + while (cachedTilesServedOrInProgressSkipped && !result && anyTasksRemaining); // Clean up done requests. - requests_.remove_if([](auto&& r) {return r->nextTileIndex_ == r->tiles_.size(); }); + requests_.remove_if([](auto&& r) {return r->tileIdsNotStarted_.empty(); }); return result; } @@ -229,7 +273,7 @@ struct Service::Worker bool work() { - std::optional nextJob; + std::shared_ptr nextJob; { std::unique_lock lock(controller_.jobsMutex_); @@ -244,8 +288,8 @@ struct Service::Worker // is removed. All worker instances are expected to terminate. return true; } - nextJob = controller_.nextJob(info_); - return nextJob.has_value(); + nextJob = controller_.nextJob(info_, lock); + return !!nextJob; }); } @@ -260,7 +304,28 @@ struct Service::Worker dataSource_->onCacheExpired(job.tileKey, *job.cacheExpiredAt); } - auto layer = dataSource_->get(job.tileKey, controller_.cache_, info_); + auto notifyWaitingRequests = [&](TileLayer::LoadState state) { + std::vector waiting; + { + std::unique_lock lock(controller_.jobsMutex_); + job.loadStatus = state; + waiting = job.waitingRequests; + } + for (auto const& req : waiting) { + if (req) { + req->notifyLoadState(job.tileKey, state); + } + } + }; + + notifyWaitingRequests(TileLayer::LoadState::BackendFetching); + auto layer = dataSource_->get( + job.tileKey, + controller_.cache_, + info_, + [¬ifyWaitingRequests](TileLayer::LoadState state) { + notifyWaitingRequests(state); + }); if (!layer) raise("DataSource::get() returned null."); @@ -282,14 +347,20 @@ struct Service::Worker controller_.cache_->putTileLayer(layer); + std::vector notifyRequests; { std::unique_lock lock(controller_.jobsMutex_); controller_.jobsInProgress_.erase(job.tileKey); - job.request->notifyResult(layer); - // As we entered a tile into the cache, notify other workers - // that this tile can be served. - controller_.jobsAvailable_.notify_all(); + notifyRequests = job.waitingRequests; } + for (auto const& req : notifyRequests) { + if (req) { + req->notifyResult(layer); + } + } + // As we entered a tile into the cache, notify other workers + // that this tile can be served. + controller_.jobsAvailable_.notify_all(); } catch (std::exception& e) { log().error("Could not load tile {}: {}", @@ -314,7 +385,7 @@ struct Service::Impl : public Service::Controller Cache::Ptr cache, bool useDataSourceConfig, std::optional defaultTtl) - : Controller(std::move(cache), std::move(defaultTtl)) + : Controller(std::move(cache), defaultTtl) { if (!useDataSourceConfig) return; @@ -344,7 +415,7 @@ struct Service::Impl : public Service::Controller }); } - ~Impl() + ~Impl() override { // Ensure that no new datasources are added while we are cleaning up. configSubscription_.reset(); @@ -447,12 +518,13 @@ struct Service::Impl : public Service::Controller void abortRequest(LayerTilesRequest::Ptr const& r) { - std::unique_lock lock(jobsMutex_); - // Remove the request from the list of requests. - auto numRemoved = requests_.remove_if([r](auto&& request) { return r == request; }); - // Clear its jobs to mark it as done. - if (numRemoved) { - r->setStatus(RequestStatus::Aborted); + // Mark the request as aborted. + r->setStatus(RequestStatus::Aborted); + + // Remove the request from the list of requests (needs lock). + { + std::unique_lock lock(jobsMutex_); + requests_.remove_if([r](auto&& request) { return r == request; }); } } @@ -473,21 +545,21 @@ struct Service::Impl : public Service::Controller if (auxDataSource->info().mapId_ == baseTile->mapId()) { auto auxTile = [&]() -> TileFeatureLayer::Ptr { - auto auxTile = auxDataSource->get(baseTile->id(), cache_, auxDataSource->info()); - if (!auxTile) { + auto result = auxDataSource->get(baseTile->id(), cache_, auxDataSource->info()); + if (!result) { log().warn("auxDataSource returned null for {}", baseTile->id().toString()); return {}; } - if (auxTile->error()) { - log().warn("Error while fetching addon tile {}: {}", baseTile->id().toString(), *auxTile->error()); + if (result->error()) { + log().warn("Error while fetching addon tile {}: {}", baseTile->id().toString(), *result->error()); return {}; } - if (auxTile->layerInfo()->type_ != LayerType::Features) { + if (result->layerInfo()->type_ != LayerType::Features) { log().warn("Addon tile is not a feature layer"); return {}; } - return std::static_pointer_cast(auxTile); + return std::static_pointer_cast(result); }(); if (!auxTile) { @@ -561,7 +633,7 @@ struct Service::Impl : public Service::Controller }; Service::Service(Cache::Ptr cache, bool useDataSourceConfig, std::optional defaultTtl) - : impl_(std::make_unique(std::move(cache), useDataSourceConfig, std::move(defaultTtl))) + : impl_(std::make_unique(std::move(cache), useDataSourceConfig, defaultTtl)) { } @@ -673,10 +745,88 @@ nlohmann::json Service::getStatistics() const }); } - return { + auto result = nlohmann::json{ {"datasources", datasources}, {"active-requests", impl_->requests_.size()} }; + + auto layerInfoByMap = std::unordered_map>>{}; + for (auto const& [_, info] : impl_->dataSourceInfo_) { + auto& layers = layerInfoByMap[info.mapId_]; + for (auto const& [layerId, layerInfo] : info.layers_) { + layers[layerId] = layerInfo; + } + } + + auto resolveLayerInfo = [&](std::string_view mapId, std::string_view layerId) -> std::shared_ptr { + auto mapIt = layerInfoByMap.find(std::string(mapId)); + if (mapIt == layerInfoByMap.end()) + return std::make_shared(); + auto layerIt = mapIt->second.find(std::string(layerId)); + if (layerIt == mapIt->second.end()) { + auto fallback = std::make_shared(); + fallback->layerId_ = std::string(layerId); + return fallback; + } + return layerIt->second; + }; + + int64_t parsedTiles = 0; + int64_t totalTileBytes = 0; + int64_t parseErrors = 0; + auto featureLayerTotals = nlohmann::json::object(); + auto modelPoolTotals = nlohmann::json::object(); + + auto addTotals = [](nlohmann::json& totals, const nlohmann::json& stats) { + for (const auto& [key, value] : stats.items()) { + if (value.is_number_integer()) + { + totals[key] = totals.value(key, 0) + value.get(); + } + else if (value.is_number_float()) + { + totals[key] = totals.value(key, .0) + value.get(); + } + } + }; + + TileLayerStream::Reader tileReader( + resolveLayerInfo, + [&](auto&& parsedLayer) + { + auto tile = std::static_pointer_cast(parsedLayer); + auto sizeStats = tile->serializationSizeStats(); + addTotals(featureLayerTotals, sizeStats["feature-layer"]); + addTotals(modelPoolTotals, sizeStats["model-pool"]); + }, + impl_->cache_); + + impl_->cache_->forEachTileLayerBlob( + [&](const MapTileKey& key, const std::string& blob) + { + if (key.layer_ != LayerType::Features) + return; + ++parsedTiles; + totalTileBytes += static_cast(blob.size()); + try { + tileReader.read(blob); + } + catch (const std::exception&) { + ++parseErrors; + } + }); + + if (parsedTiles > 0) { + result["cached-feature-tree-bytes"] = nlohmann::json{ + {"tile-count", parsedTiles}, + {"total-tile-bytes", totalTileBytes}, + {"parse-errors", parseErrors}, + {"feature-layer", featureLayerTotals}, + {"model-pool", modelPoolTotals} + }; + } + + return result; } } // namespace mapget diff --git a/libs/service/src/sqlitecache.cpp b/libs/service/src/sqlitecache.cpp index 74b2b308..c062e1ec 100644 --- a/libs/service/src/sqlitecache.cpp +++ b/libs/service/src/sqlitecache.cpp @@ -259,6 +259,32 @@ void SQLiteCache::putTileLayerBlob(MapTileKey const& k, std::string const& v) } } +void SQLiteCache::forEachTileLayerBlob(const TileBlobVisitor& cb) const +{ + std::lock_guard lock(dbMutex_); + + sqlite3_stmt* stmt = nullptr; + int rc = sqlite3_prepare_v2(db_, "SELECT key, data FROM tiles", -1, &stmt, nullptr); + if (rc != SQLITE_OK) { + raise(fmt::format("Failed to prepare tile iteration statement: {}", sqlite3_errmsg(db_))); + } + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + const char* key = reinterpret_cast(sqlite3_column_text(stmt, 0)); + const void* data = sqlite3_column_blob(stmt, 1); + int size = sqlite3_column_bytes(stmt, 1); + if (key && data && size >= 0) { + cb(MapTileKey(key), std::string(static_cast(data), size)); + } + } + if (rc != SQLITE_DONE) { + sqlite3_finalize(stmt); + raise(fmt::format("Error iterating cached tiles: {}", sqlite3_errmsg(db_))); + } + + sqlite3_finalize(stmt); +} + void SQLiteCache::cleanupOldestTiles() { // Delete the oldest tile @@ -315,4 +341,4 @@ void SQLiteCache::putStringPoolBlob(std::string_view const& sourceNodeId, std::s } } -} // namespace mapget \ No newline at end of file +} // namespace mapget diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 6ce5ba32..1af0caf9 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -1,9 +1,7 @@ project(test.mapget.integration CXX) -# TODO: Figure out a way to do this without assuming free ports. -set (MAPGET_SERVER_PORT 61852) -set (DATASOURCE_CPP_PORT 61853) -set (DATASOURCE_PY_PORT 61854) +set(MAPGET_PICK_PORTS_PY "${CMAKE_CURRENT_LIST_DIR}/detect-ports-and-prepare-config-yaml.py") +set(MAPGET_RUN_WITH_PORTS "${CMAKE_CURRENT_LIST_DIR}/run-with-ports.bash") add_wheel_test(test-local-example WORKING_DIRECTORY @@ -20,20 +18,23 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports at test runtime (reduces CI flakiness vs. hard-coded ports) + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-cli-cpp" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "./mapget --log-level trace serve -p ${MAPGET_SERVER_PORT} -d 127.0.0.1:${DATASOURCE_CPP_PORT} -d 127.0.0.1:${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./mapget --log-level trace serve -p $MAPGET_SERVER_PORT -d 127.0.0.1:$DATASOURCE_CPP_PORT -d 127.0.0.1:$DATASOURCE_PY_PORT" # Request from cpp datasource - -f "./mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m Tropico -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m Tropico -l WayLayer -t 12345" # Request from py datasource - -f "./mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m TestMap -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m TestMap -l WayLayer -t 12345" ) endif () @@ -42,11 +43,14 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports at test runtime (reduces CI flakiness vs. hard-coded ports) + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-cli-datasource-exe" + # Run service with auto-launched python datasource - -b "${CMAKE_CURRENT_LIST_DIR}/mapget-exec-datasource.bash ${MAPGET_SERVER_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-datasource-exe/ports.env ${CMAKE_CURRENT_LIST_DIR}/mapget-exec-datasource.bash $MAPGET_SERVER_PORT" # Request from py datasource - -f "./mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m TestMap -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-datasource-exe/ports.env ./mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m TestMap -l WayLayer -t 12345" ) endif () @@ -56,17 +60,20 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports at test runtime (reduces CI flakiness vs. hard-coded ports) + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-cli-python" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "python -m mapget --log-level trace serve -p ${MAPGET_SERVER_PORT} -d 127.0.0.1:${DATASOURCE_CPP_PORT} -d 127.0.0.1:${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env python -m mapget --log-level trace serve -p $MAPGET_SERVER_PORT -d 127.0.0.1:$DATASOURCE_CPP_PORT -d 127.0.0.1:$DATASOURCE_PY_PORT" # Request from py datasource - -f "python -m mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m TestMap -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env python -m mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m TestMap -l WayLayer -t 12345" ) endif() @@ -76,20 +83,23 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports + write config YAMLs at test runtime. + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-config-cpp" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-cpp/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-cpp/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "./mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-service.yaml serve" + -b "./mapget --config .integration/test-config-cpp/sample-service.yaml serve" # Request from py datasource - -f "./mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-second-datasource.yaml fetch" + -f "./mapget --config .integration/test-config-cpp/sample-second-datasource.yaml fetch" # Request from cpp datasource - -f "./mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-first-datasource.yaml fetch" + -f "./mapget --config .integration/test-config-cpp/sample-first-datasource.yaml fetch" ) endif () @@ -99,19 +109,22 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports + write config YAMLs at test runtime. + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-config-py" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-py/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-py/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "python -m mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-service.yaml serve" + -b "python -m mapget --config .integration/test-config-py/sample-service.yaml serve" # Request from py datasource - -f "python -m mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-second-datasource.yaml fetch" + -f "python -m mapget --config .integration/test-config-py/sample-second-datasource.yaml fetch" # Request from cpp datasource - -f "python -m mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-first-datasource.yaml fetch" + -f "python -m mapget --config .integration/test-config-py/sample-first-datasource.yaml fetch" ) endif () diff --git a/test/integration/detect-ports-and-prepare-config-yaml.py b/test/integration/detect-ports-and-prepare-config-yaml.py new file mode 100644 index 00000000..0360ccdd --- /dev/null +++ b/test/integration/detect-ports-and-prepare-config-yaml.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import re +import socket +from pathlib import Path + + +def _pick_free_tcp_ports(count: int) -> list[int]: + if count <= 0: + raise ValueError("count must be > 0") + + sockets: list[socket.socket] = [] + ports: list[int] = [] + try: + for _ in range(count): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("127.0.0.1", 0)) + sockets.append(s) + ports.append(int(s.getsockname()[1])) + finally: + for s in sockets: + try: + s.close() + except Exception: + pass + + if len(set(ports)) != len(ports): + raise RuntimeError(f"Port picker returned duplicates: {ports}") + + return ports + + +def _patch_sample_service_yaml(text: str, mapget_port: int, datasource_cpp_port: int, datasource_py_port: int) -> str: + text = re.sub( + r"(?m)^(\s*port:\s*)\d+(\s*)$", + rf"\g<1>{mapget_port}\g<2>", + text, + count=1, + ) + text = text.replace("127.0.0.1:61853", f"127.0.0.1:{datasource_cpp_port}") + text = text.replace("127.0.0.1:61854", f"127.0.0.1:{datasource_py_port}") + return text + + +def _patch_sample_fetch_yaml(text: str, mapget_port: int) -> str: + return re.sub( + r"(?m)^(\s*server:\s*127\.0\.0\.1:)\d+(\s*)$", + rf"\g<1>{mapget_port}\g<2>", + text, + count=1, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--out-dir", required=True, help="Output directory for generated files (created if needed).") + args = parser.parse_args() + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + # Pick ports at test runtime (reduces collision risk vs. configure-time selection). + mapget_port, datasource_cpp_port, datasource_py_port = _pick_free_tcp_ports(3) + + ports_env = out_dir / "ports.env" + ports_env.write_text( + "\n".join( + [ + f"export MAPGET_SERVER_PORT={mapget_port}", + f"export DATASOURCE_CPP_PORT={datasource_cpp_port}", + f"export DATASOURCE_PY_PORT={datasource_py_port}", + "", + ] + ), + encoding="utf-8", + newline="\n", + ) + + repo_root = Path(__file__).resolve().parents[2] + examples_config = repo_root / "examples" / "config" + + sample_service = (examples_config / "sample-service.yaml").read_text(encoding="utf-8") + (out_dir / "sample-service.yaml").write_text( + _patch_sample_service_yaml(sample_service, mapget_port, datasource_cpp_port, datasource_py_port), + encoding="utf-8", + newline="\n", + ) + + sample_first = (examples_config / "sample-first-datasource.yaml").read_text(encoding="utf-8") + (out_dir / "sample-first-datasource.yaml").write_text( + _patch_sample_fetch_yaml(sample_first, mapget_port), + encoding="utf-8", + newline="\n", + ) + + sample_second = (examples_config / "sample-second-datasource.yaml").read_text(encoding="utf-8") + (out_dir / "sample-second-datasource.yaml").write_text( + _patch_sample_fetch_yaml(sample_second, mapget_port), + encoding="utf-8", + newline="\n", + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/test/integration/run-with-ports.bash b/test/integration/run-with-ports.bash new file mode 100644 index 00000000..b0f17c54 --- /dev/null +++ b/test/integration/run-with-ports.bash @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -lt 2 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +ports_env="$1" +shift + +# shellcheck disable=SC1090 +source "$ports_env" + +# Note: The wheel test harness passes commands as a single string, so `$MAPGET_SERVER_PORT` +# etc. are not expanded. We intentionally use `eval` here so the variables sourced above +# are expanded before executing the command. +eval "exec $*" + diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 8a2deff4..af35f5cb 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -1,6 +1,7 @@ project(test.mapget.unit CXX) add_executable(test.mapget + test-main.cpp test-model.cpp test-model-geometry.cpp test-simfil-geometry.cpp @@ -16,6 +17,9 @@ add_executable(test.mapget add_executable(test.mapget.filelog test-file-logging.cpp) +add_executable(test.mapget.datasource-server + test-datasource-server.cpp) + target_link_libraries(test.mapget PUBLIC mapget-log @@ -23,7 +27,16 @@ target_link_libraries(test.mapget mapget-http-datasource mapget-http-service geojsonsource - Catch2::Catch2WithMain) + Catch2::Catch2) + +target_link_libraries(test.mapget.datasource-server + PUBLIC + mapget-log + mapget-http-datasource) + +target_compile_definitions(test.mapget + PRIVATE + MAPGET_TEST_DATASOURCE_SERVER_EXE=\"$\") target_link_libraries(test.mapget.filelog PUBLIC diff --git a/test/unit/test-datasource-server.cpp b/test/unit/test-datasource-server.cpp new file mode 100644 index 00000000..e5ebfc4f --- /dev/null +++ b/test/unit/test-datasource-server.cpp @@ -0,0 +1,70 @@ +#include "mapget/http-datasource/datasource-server.h" +#include "mapget/log.h" + +#include "nlohmann/json.hpp" + +using namespace mapget; +using namespace nlohmann; + +int main() +{ + setLogLevel("trace", log()); + + auto info = DataSourceInfo::fromJson(R"( + { + "nodeId": "test-datasource", + "mapId": "Tropico", + "layers": { + "WayLayer": { + "featureTypes": + [ + { + "name": "Way", + "uniqueIdCompositions": + [ + [ + { + "partId": "areaId", + "description": "String which identifies the map area.", + "datatype": "STR" + }, + { + "partId": "wayId", + "description": "Globally Unique 32b integer.", + "datatype": "U32" + } + ] + ] + } + ] + }, + "SourceData-WayLayer": { + "type": "SourceData" + } + } + } + )"_json); + + DataSourceServer ds(info); + ds.onTileFeatureRequest( + [&](const auto& tile) + { + auto f = tile->newFeature("Way", {{"areaId", "Area42"}, {"wayId", 0}}); + auto g = f->geom()->newGeometry(GeomType::Line); + g->append({42., 11}); + g->append({42., 12}); + }); + ds.onTileSourceDataRequest([&](const auto&) {}); + ds.onLocateRequest( + [&](LocateRequest const& request) -> std::vector + { + LocateResponse response(request); + response.tileKey_.layerId_ = "WayLayer"; + response.tileKey_.tileId_.value_ = 1; + return {response}; + }); + + ds.go("127.0.0.1", 0, 5000); + ds.waitForSignal(); + return 0; +} diff --git a/test/unit/test-http-datasource.cpp b/test/unit/test-http-datasource.cpp index 8f58c08b..a2f12e2a 100644 --- a/test/unit/test-http-datasource.cpp +++ b/test/unit/test-http-datasource.cpp @@ -1,34 +1,170 @@ #include + +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include #include -#include +#include + #ifndef _WIN32 -#include #include +#include #endif -#include "httplib.h" -#include "mapget/log.h" -#include "nlohmann/json.hpp" -#include "utility.h" +#include +#include +#include +#include + +#include "process.hpp" + #include "mapget/http-datasource/datasource-client.h" -#include "mapget/http-datasource/datasource-server.h" +#include "mapget/http-service/cli.h" #include "mapget/http-service/http-client.h" #include "mapget/http-service/http-service.h" +#include "mapget/log.h" +#include "mapget/model/info.h" #include "mapget/model/stream.h" #include "mapget/service/config.h" -#include "mapget/http-service/cli.h" + +#include "nlohmann/json.hpp" + +#include "test-http-service-fixture.h" +#include "utility.h" using namespace mapget; namespace fs = std::filesystem; -TEST_CASE("HttpDataSource", "[HttpDataSource]") +namespace { - setLogLevel("trace", log()); - // Create DataSourceInfo. - auto info = DataSourceInfo::fromJson(R"( +class SyncHttpClient +{ +public: + SyncHttpClient(std::string host, uint16_t port) + { + loopThread_ = std::make_unique("MapgetTestHttpClient"); + loopThread_->run(); + + client_ = drogon::HttpClient::newHttpClient( + fmt::format("http://{}:{}/", host, port), + loopThread_->getLoop()); + } + + std::pair get(std::string path) + { + auto req = drogon::HttpRequest::newHttpRequest(); + req->setMethod(drogon::Get); + req->setPath(std::move(path)); + return client_->sendRequest(req); + } + + std::pair postJson(std::string path, std::string body) + { + auto req = drogon::HttpRequest::newHttpRequest(); + req->setMethod(drogon::Post); + req->setPath(std::move(path)); + req->setContentTypeCode(drogon::CT_APPLICATION_JSON); + req->setBody(std::move(body)); + return client_->sendRequest(req); + } + +private: + std::unique_ptr loopThread_; + drogon::HttpClientPtr client_; +}; + +class ChildProcessWithPort +{ +public: + explicit ChildProcessWithPort(std::string exePath) + { + auto stderrCallback = [](const char* bytes, size_t n) { + auto output = std::string(bytes, n); + output.erase(output.find_last_not_of(" \n\r\t") + 1); + if (!output.empty()) + std::cerr << output << std::endl; + }; + + auto stdoutCallback = [this](const char* bytes, size_t n) { + std::lock_guard lock(mutex_); + stdoutBuffer_.append(bytes, n); + + for (;;) { + auto nl = stdoutBuffer_.find_first_of("\r\n"); + if (nl == std::string::npos) + break; + + auto line = stdoutBuffer_.substr(0, nl); + stdoutBuffer_.erase(0, nl + 1); + line.erase(line.find_last_not_of(" \n\r\t") + 1); + + if (!portReady_) { + std::regex portRegex(R"(Running on port (\d+))"); + std::smatch matches; + if (std::regex_search(line, matches, portRegex) && matches.size() > 1) { + port_ = static_cast(std::stoi(matches.str(1))); + portReady_ = true; + cv_.notify_all(); + } + } + } + }; + + process_ = std::make_unique( + fmt::format("\"{}\"", exePath), + "", + stdoutCallback, + stderrCallback, + true); + + std::unique_lock lock(mutex_); +#if defined(NDEBUG) + if (!cv_.wait_for(lock, std::chrono::seconds(10), [this] { return portReady_; })) { + raise("Timeout waiting for the child process to start listening."); + } +#else + log().warn("Using Debug build: will wait forever!"); + cv_.wait(lock, [this] { return portReady_; }); +#endif + } + + ~ChildProcessWithPort() + { + if (process_) { + process_->kill(true); + process_->get_exit_status(); + } + } + + [[nodiscard]] uint16_t port() const + { + return port_; + } + +private: + std::unique_ptr process_; + mutable std::mutex mutex_; + std::condition_variable cv_; + std::string stdoutBuffer_; + uint16_t port_ = 0; + bool portReady_ = false; +}; + +nlohmann::json testDataSourceInfoJson() +{ + using nlohmann::json; + return json::parse(R"( { + "nodeId": "test-datasource", "mapId": "Tropico", "layers": { "WayLayer": { @@ -59,69 +195,41 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") } } } - )"_json); - - // Initialize a DataSource. - DataSourceServer ds(info); - std::atomic_uint32_t dataSourceFeatureRequestCount = 0; - std::atomic_uint32_t dataSourceSourceDataRequestCount = 0; - ds.onTileFeatureRequest( - [&](const auto& tile) - { - auto f = tile->newFeature("Way", {{"areaId", "Area42"}, {"wayId", 0}}); - auto g = f->geom()->newGeometry(GeomType::Line); - g->append({42., 11}); - g->append({42., 12}); - ++dataSourceFeatureRequestCount; - }); - ds.onTileSourceDataRequest( - [&](const auto& tile) { - ++dataSourceSourceDataRequestCount; - }); - ds.onLocateRequest( - [&](LocateRequest const& request) -> std::vector - { - REQUIRE(request.mapId_ == "Tropico"); - REQUIRE(request.typeId_ == "Way"); - REQUIRE(request.featureId_ == KeyValuePairs{{"wayId", 0}}); + )"); +} + +} // namespace + +TEST_CASE("HttpDataSource", "[HttpDataSource]") +{ + setLogLevel("trace", log()); - LocateResponse response(request); - response.tileKey_.layerId_ = "WayLayer"; - response.tileKey_.tileId_.value_ = 1; - return {response}; - }); + // Start datasource server in a separate process (Drogon is singleton). + ChildProcessWithPort dsProc(MAPGET_TEST_DATASOURCE_SERVER_EXE); - // Launch the DataSource on a separate thread. - ds.go(); + // Expected datasource info. + auto info = DataSourceInfo::fromJson(testDataSourceInfoJson()); - // Ensure the DataSource is running. - REQUIRE(ds.isRunning() == true); + SyncHttpClient dsClient("127.0.0.1", dsProc.port()); - SECTION("Fetch /info") + // Fetch /info { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); + auto [result, resp] = dsClient.get("/info"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); - // Send a GET info request. - auto fetchedInfoJson = cli.Get("/info"); - auto fetchedInfo = - DataSourceInfo::fromJson(nlohmann::json::parse(fetchedInfoJson->body)); + auto fetchedInfo = DataSourceInfo::fromJson(nlohmann::json::parse(std::string(resp->body()))); REQUIRE(fetchedInfo.toJson() == info.toJson()); } - SECTION("Fetch /tile") + // Fetch /tile { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); - - // Send a GET tile request. - auto tileResponse = cli.Get("/tile?layer=WayLayer&tileId=1"); - - // Check that the response is OK. - REQUIRE(tileResponse != nullptr); - REQUIRE(tileResponse->status == 200); + auto [result, resp] = dsClient.get("/tile?layer=WayLayer&tileId=1"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); - // Check the response body for expected content. auto receivedTileCount = 0; TileLayerStream::Reader reader( [&](auto&& mapId, auto&& layerId) @@ -133,24 +241,18 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(tile->id().layer_ == LayerType::Features); receivedTileCount++; }); - reader.read(tileResponse->body); + reader.read(std::string(resp->body())); REQUIRE(receivedTileCount == 1); } - SECTION("Fetch /tile SourceData") + // Fetch /tile SourceData { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); + auto [result, resp] = dsClient.get("/tile?layer=SourceData-WayLayer&tileId=1"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); - // Send a GET tile request - auto tileResponse = cli.Get("/tile?layer=SourceData-WayLayer&tileId=1"); - - // Check that the response is OK. - REQUIRE(tileResponse != nullptr); - REQUIRE(tileResponse->status == 200); - - // Check the response body for expected content. auto receivedTileCount = 0; TileLayerStream::Reader reader( [&](auto&& mapId, auto&& layerId) @@ -162,57 +264,49 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(tile->id().layer_ == LayerType::SourceData); receivedTileCount++; }); - reader.read(tileResponse->body); + reader.read(std::string(resp->body())); REQUIRE(receivedTileCount == 1); } - SECTION("Fetch /locate") + // Fetch /locate { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); - - // Send a POST locate request. - auto response = cli.Post("/locate", R"({ - "mapId": "Tropico", - "typeId": "Way", - "featureId": ["wayId", 0] - })", "application/json"); - - // Check that the response is OK. - REQUIRE(response != nullptr); - REQUIRE(response->status == 200); - - // Check the response body for expected content. - LocateResponse responseParsed(nlohmann::json::parse(response->body)[0]); + auto [result, resp] = dsClient.postJson( + "/locate", + R"({ + "mapId": "Tropico", + "typeId": "Way", + "featureId": ["wayId", 0] + })"); + + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); + + LocateResponse responseParsed(nlohmann::json::parse(std::string(resp->body()))[0]); REQUIRE(responseParsed.tileKey_.mapId_ == "Tropico"); REQUIRE(responseParsed.tileKey_.layer_ == LayerType::Features); REQUIRE(responseParsed.tileKey_.layerId_ == "WayLayer"); REQUIRE(responseParsed.tileKey_.tileId_.value_ == 1); } - SECTION("Query mapget HTTP service") + // Query mapget HTTP service (in-process, started once for entire test binary) { + auto& service = test::httpService(); + auto remoteDataSource = std::make_shared("127.0.0.1", dsProc.port()); + service.add(remoteDataSource); + auto countReceivedTiles = [](auto& client, auto mapId, auto layerId, auto tiles) { auto tileCount = 0; - auto request = std::make_shared(mapId, layerId, tiles); - request->onFeatureLayer([&](auto&& tile) { tileCount++; }); - //request->onSourceDataLayer([&](auto&& tile) { tileCount++; }); - + request->onFeatureLayer([&](auto&&) { tileCount++; }); client.request(request)->wait(); return std::make_tuple(request, tileCount); }; - HttpService service; - auto remoteDataSource = std::make_shared("localhost", ds.port()); - service.add(remoteDataSource); - - service.go(); - - SECTION("Query through mapget HTTP service") + // Query through mapget HTTP service { - HttpClient client("localhost", service.port()); + HttpClient client("127.0.0.1", service.port()); auto [request, receivedTileCount] = countReceivedTiles( client, @@ -221,54 +315,47 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") std::vector{{1234, 5678, 9112, 1234}}); REQUIRE(receivedTileCount == 4); - // One tile requested twice, so the cache was used. - REQUIRE(dataSourceFeatureRequestCount == 3); + REQUIRE(request->getStatus() == RequestStatus::Success); } - SECTION("Trigger 400 responses") + // Trigger 400 responses { - HttpClient client("localhost", service.port()); + HttpClient client("127.0.0.1", service.port()); { - auto [request, receivedTileCount] = countReceivedTiles( - client, - "UnknownMap", - "WayLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(client, "UnknownMap", "WayLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::NoDataSource); REQUIRE(receivedTileCount == 0); } { - auto [request, receivedTileCount] = countReceivedTiles( - client, - "Tropico", - "UnknownLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(client, "Tropico", "UnknownLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::NoDataSource); REQUIRE(receivedTileCount == 0); } } - SECTION("Run /locate through service") + // Run /locate through service { - httplib::Client client("localhost", service.port()); - - // Send a POST locate request. - auto response = client.Post("/locate", R"({ - "requests": [{ - "mapId": "Tropico", - "typeId": "Way", - "featureId": ["wayId", 0] - }] - })", "application/json"); - - // Check that the response is OK. - REQUIRE(response != nullptr); - REQUIRE(response->status == 200); - - // Check the response body for expected content. - auto responseJsonLists = nlohmann::json::parse(response->body)["responses"]; + SyncHttpClient serviceClient("127.0.0.1", service.port()); + + auto [result, resp] = serviceClient.postJson( + "/locate", + R"({ + "requests": [{ + "mapId": "Tropico", + "typeId": "Way", + "featureId": ["wayId", 0] + }] + })"); + + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); + + auto responseJsonLists = nlohmann::json::parse(std::string(resp->body()))["responses"]; REQUIRE(responseJsonLists.size() == 1); auto responseJsonList = responseJsonLists[0]; REQUIRE(responseJsonList.size() == 1); @@ -279,79 +366,342 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(responseParsed.tileKey_.tileId_.value_ == 1); } - SECTION("Test auth header requirement") + // Test auth header requirement { - remoteDataSource->requireAuthHeaderRegexMatchOption( - "X-USER-ROLE", - std::regex("\\bTropico-Viewer\\b")); + remoteDataSource->requireAuthHeaderRegexMatchOption("X-USER-ROLE", std::regex("\\bTropico-Viewer\\b")); - HttpClient badClient("localhost", service.port()); - HttpClient goodClient("localhost", service.port(), {{"X-USER-ROLE", "Tropico-Viewer"}}); + HttpClient badClient("127.0.0.1", service.port()); + HttpClient goodClient("127.0.0.1", service.port(), {{"X-USER-ROLE", "Tropico-Viewer"}}); - // Check sources REQUIRE(badClient.sources().empty()); REQUIRE(goodClient.sources().size() == 1); - // Try to load tiles with bad client { - auto [request, receivedTileCount] = countReceivedTiles( - badClient, - "Tropico", - "WayLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(badClient, "Tropico", "WayLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::Unauthorized); REQUIRE(receivedTileCount == 0); } - // Try to load tiles with good client { - auto [request, receivedTileCount] = countReceivedTiles( - goodClient, - "Tropico", - "WayLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(goodClient, "Tropico", "WayLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::Success); REQUIRE(receivedTileCount == 1); } - } - service.stop(); - REQUIRE(service.isRunning() == false); - } + auto runWsTilesRequest = [&](bool sendAuthHeader, std::string requestJson) { + auto wsLoopThread = std::make_unique("MapgetTestWsClient"); + wsLoopThread->run(); + + auto wsClient = drogon::WebSocketClient::newWebSocketClient( + fmt::format("ws://127.0.0.1:{}", service.port()), + wsLoopThread->getLoop()); + + std::mutex mutex; + std::condition_variable cv; + std::optional lastStatus; + std::atomic_int receivedTileCount{0}; + std::string error; + + const auto dsInfo = remoteDataSource->info(); + const auto layerInfo = dsInfo.getLayer("WayLayer"); + REQUIRE(layerInfo != nullptr); + + TileLayerStream::Reader reader( + [&](auto&&, auto&&) { return layerInfo; }, + [&](auto&& tile) { + if (tile->id().layer_ != LayerType::Features) { + std::lock_guard lock(mutex); + error = "Unexpected tile layer type"; + } + receivedTileCount.fetch_add(1, std::memory_order_relaxed); + }); + + wsClient->setMessageHandler( + [&](std::string&& msg, + const drogon::WebSocketClientPtr&, + const drogon::WebSocketMessageType& msgType) { + if (msgType != drogon::WebSocketMessageType::Binary) { + return; + } + + TileLayerStream::MessageType type = TileLayerStream::MessageType::None; + uint32_t payloadSize = 0; + std::stringstream ss; + ss.write(msg.data(), static_cast(msg.size())); + if (!TileLayerStream::Reader::readMessageHeader(ss, type, payloadSize)) { + std::lock_guard lock(mutex); + error = "Failed to read stream message header"; + cv.notify_all(); + return; + } + + if (type == TileLayerStream::MessageType::Status) { + std::string payload(payloadSize, '\0'); + ss.read(payload.data(), static_cast(payloadSize)); + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(payload); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse status JSON: ") + e.what(); + cv.notify_all(); + return; + } + { + std::lock_guard lock(mutex); + lastStatus = std::move(parsed); + } + cv.notify_all(); + return; + } + + try { + reader.read(msg); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse tile stream: ") + e.what(); + cv.notify_all(); + } + }); + + auto connectReq = drogon::HttpRequest::newHttpRequest(); + connectReq->setMethod(drogon::Get); + connectReq->setPath("/tiles"); + if (sendAuthHeader) { + connectReq->addHeader("X-USER-ROLE", "Tropico-Viewer"); + } + + std::promise connectPromise; + auto connectFuture = connectPromise.get_future(); + wsClient->connectToServer( + connectReq, + [&connectPromise]( + drogon::ReqResult result, + const drogon::HttpResponsePtr&, + const drogon::WebSocketClientPtr&) { connectPromise.set_value(result); }); + + REQUIRE(connectFuture.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + REQUIRE(connectFuture.get() == drogon::ReqResult::Ok); + + auto conn = wsClient->getConnection(); + if (!conn || !conn->connected()) { + wsClient->stop(); + FAIL("WebSocket connection not established"); + } + + conn->send(requestJson, drogon::WebSocketMessageType::Text); + + { + std::unique_lock lock(mutex); + REQUIRE(cv.wait_for(lock, std::chrono::seconds(10), [&] { + return !error.empty() || + (lastStatus.has_value() && lastStatus->value("allDone", false)); + })); + if (!error.empty()) { + wsClient->stop(); + FAIL(error); + } + } - SECTION("Wait for data source") - { - auto waitThread = std::thread([&] { ds.waitForSignal(); }); - ds.stop(); - waitThread.join(); - REQUIRE(ds.isRunning() == false); - } + wsClient->stop(); + + REQUIRE(lastStatus.has_value()); + return std::make_tuple(*lastStatus, receivedTileCount.load(std::memory_order_relaxed)); + }; + + // WebSocket tiles: unauthorized without auth header. + { + auto req = nlohmann::json::object({ + {"requests", nlohmann::json::array({nlohmann::json::object({ + {"mapId", "Tropico"}, + {"layerId", "WayLayer"}, + {"tileIds", nlohmann::json::array({1234})}, + })})}, + }).dump(); + + auto [status, wsTileCount] = runWsTilesRequest(false, req); + REQUIRE(wsTileCount == 0); + REQUIRE(status["requests"].size() == 1); + REQUIRE(status["requests"][0]["status"].get() == + static_cast(RequestStatus::Unauthorized)); + } + + // WebSocket tiles: invalid request stays on the same connection, then succeeds. + { + auto wsLoopThread = std::make_unique("MapgetTestWsClientReuse"); + wsLoopThread->run(); + + auto wsClient = drogon::WebSocketClient::newWebSocketClient( + fmt::format("ws://127.0.0.1:{}", service.port()), + wsLoopThread->getLoop()); + + std::mutex mutex; + std::condition_variable cv; + std::optional lastStatus; + std::atomic_int receivedTileCount{0}; + std::string error; + + const auto dsInfo = remoteDataSource->info(); + const auto layerInfo = dsInfo.getLayer("WayLayer"); + REQUIRE(layerInfo != nullptr); + + TileLayerStream::Reader reader( + [&](auto&&, auto&&) { return layerInfo; }, + [&](auto&&) { receivedTileCount.fetch_add(1, std::memory_order_relaxed); }); + + wsClient->setMessageHandler( + [&](std::string&& msg, + const drogon::WebSocketClientPtr&, + const drogon::WebSocketMessageType& msgType) { + if (msgType != drogon::WebSocketMessageType::Binary) { + return; + } + + TileLayerStream::MessageType type = TileLayerStream::MessageType::None; + uint32_t payloadSize = 0; + std::stringstream ss; + ss.write(msg.data(), static_cast(msg.size())); + if (!TileLayerStream::Reader::readMessageHeader(ss, type, payloadSize)) { + std::lock_guard lock(mutex); + error = "Failed to read stream message header"; + cv.notify_all(); + return; + } + + if (type == TileLayerStream::MessageType::Status) { + std::string payload(payloadSize, '\0'); + ss.read(payload.data(), static_cast(payloadSize)); + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(payload); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse status JSON: ") + e.what(); + cv.notify_all(); + return; + } + { + std::lock_guard lock(mutex); + lastStatus = std::move(parsed); + } + cv.notify_all(); + return; + } + + try { + reader.read(msg); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse tile stream: ") + e.what(); + cv.notify_all(); + } + }); + + auto connectReq = drogon::HttpRequest::newHttpRequest(); + connectReq->setMethod(drogon::Get); + connectReq->setPath("/tiles"); + connectReq->addHeader("X-USER-ROLE", "Tropico-Viewer"); + + std::promise connectPromise; + auto connectFuture = connectPromise.get_future(); + wsClient->connectToServer( + connectReq, + [&connectPromise]( + drogon::ReqResult result, + const drogon::HttpResponsePtr&, + const drogon::WebSocketClientPtr&) { connectPromise.set_value(result); }); + + REQUIRE(connectFuture.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + REQUIRE(connectFuture.get() == drogon::ReqResult::Ok); + + auto conn = wsClient->getConnection(); + if (!conn || !conn->connected()) { + wsClient->stop(); + FAIL("WebSocket connection not established"); + } + + // Invalid JSON: should yield a Status message but keep the socket open. + { + conn->send("{not json", drogon::WebSocketMessageType::Text); + std::unique_lock lock(mutex); + REQUIRE(cv.wait_for(lock, std::chrono::seconds(5), [&] { + return !error.empty() || + (lastStatus.has_value() && lastStatus->value("allDone", false)); + })); + if (!error.empty()) { + wsClient->stop(); + FAIL(error); + } + REQUIRE(lastStatus->value("message", "").find("Invalid JSON") != std::string::npos); + REQUIRE(conn->connected()); + } + + // Valid request should succeed afterwards. + { + { + std::lock_guard lock(mutex); + lastStatus.reset(); + } + receivedTileCount.store(0, std::memory_order_relaxed); + + auto req = nlohmann::json::object({ + {"requests", nlohmann::json::array({nlohmann::json::object({ + {"mapId", "Tropico"}, + {"layerId", "WayLayer"}, + {"tileIds", nlohmann::json::array({1234})}, + })})}, + }).dump(); + + conn->send(req, drogon::WebSocketMessageType::Text); + + std::unique_lock lock(mutex); + REQUIRE(cv.wait_for(lock, std::chrono::seconds(10), [&] { + return !error.empty() || + (lastStatus.has_value() && lastStatus->value("allDone", false)); + })); + if (!error.empty()) { + wsClient->stop(); + FAIL(error); + } + + REQUIRE(receivedTileCount.load(std::memory_order_relaxed) == 1); + REQUIRE(lastStatus->contains("requests")); + REQUIRE((*lastStatus)["requests"].size() == 1); + REQUIRE((*lastStatus)["requests"][0]["status"].get() == + static_cast(RequestStatus::Success)); + } - ds.stop(); - REQUIRE(ds.isRunning() == false); + wsClient->stop(); + } + } + + service.remove(remoteDataSource); + } } TEST_CASE("Configuration Endpoint Tests", "[Configuration]") { + auto& service = test::httpService(); + REQUIRE(service.isRunning() == true); + + SyncHttpClient cli("127.0.0.1", service.port()); + auto tempDir = fs::temp_directory_path() / test::generateTimestampedDirectoryName("mapget_test_http_config"); fs::create_directory(tempDir); auto tempConfigPath = tempDir / "temp_config.yaml"; - // Setting up the server and client. - HttpService service; - service.go(); - REQUIRE(service.isRunning() == true); - httplib::Client cli("localhost", service.port()); - // Set up the config file. DataSourceConfigService::get().reset(); struct SchemaPatchGuard { - ~SchemaPatchGuard() { - DataSourceConfigService::get().setDataSourceConfigSchemaPatch(nlohmann::json::object()); - } + ~SchemaPatchGuard() { DataSourceConfigService::get().setDataSourceConfigSchemaPatch(nlohmann::json::object()); } } schemaPatchGuard; - // Emulate the CLI-provided config-schema patch so http-settings participates in the auto schema. auto schemaPatch = nlohmann::json::parse(R"( { "properties": { @@ -364,110 +714,127 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") )"); DataSourceConfigService::get().setDataSourceConfigSchemaPatch(schemaPatch); - SECTION("Get Configuration - Config File Not Found") { + SECTION("Get Configuration - Config File Not Found") + { DataSourceConfigService::get().loadConfig(tempConfigPath.string()); - auto res = cli.Get("/config"); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 404); - REQUIRE(res->body == "The server does not have a config file."); + REQUIRE(res->statusCode() == drogon::k404NotFound); + REQUIRE(std::string(res->body()) == "The server does not have a config file."); } // Create config file for tests that need it { std::ofstream configFile(tempConfigPath); - configFile << "sources: []\nhttp-settings: [{'password': 'hunter2'}]"; // Update http-settings to an array. + configFile << "sources: []\nhttp-settings: [{'password': 'hunter2'}]"; configFile.flush(); configFile.close(); - - // Ensure file is synced to disk - #ifndef _WIN32 + +#ifndef _WIN32 int fd = open(tempConfigPath.c_str(), O_RDONLY); if (fd != -1) { fsync(fd); close(fd); } - #endif +#endif std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - - // Load the config after file is created + DataSourceConfigService::get().loadConfig(tempConfigPath.string()); - - // Give the config watcher time to detect the file std::this_thread::sleep_for(std::chrono::milliseconds(500)); - SECTION("Get Configuration - Not allowed") { + SECTION("Get Configuration - Not allowed") + { setGetConfigEndpointEnabled(false); - auto res = cli.Get("/config"); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 403); + REQUIRE(res->statusCode() == drogon::k403Forbidden); } - SECTION("Get Configuration - No Config File Path Set") { + SECTION("Get Configuration - No Config File Path Set") + { setGetConfigEndpointEnabled(true); - DataSourceConfigService::get().loadConfig(""); // Simulate no config path set. - auto res = cli.Get("/config"); + DataSourceConfigService::get().loadConfig(""); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 404); - REQUIRE(res->body == "The config file path is not set. Check the server configuration."); + REQUIRE(res->statusCode() == drogon::k404NotFound); + REQUIRE(std::string(res->body()) == + "The config file path is not set. Check the server configuration."); } - SECTION("Get Configuration - Success") { - auto res = cli.Get("/config"); + SECTION("Get Configuration - Success") + { + setGetConfigEndpointEnabled(true); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 200); - REQUIRE(res->body.find("sources") != std::string::npos); - REQUIRE(res->body.find("http-settings") != std::string::npos); - - // Ensure that the password is masked as SHA256. - REQUIRE(res->body.find("hunter2") == std::string::npos); - REQUIRE(res->body.find("MASKED:0:f52fbd32b2b3b86ff88ef6c490628285f482af15ddcb29541f94bcf526a3f6c7") != std::string::npos); + REQUIRE(res->statusCode() == drogon::k200OK); + + auto body = std::string(res->body()); + REQUIRE(body.find("sources") != std::string::npos); + REQUIRE(body.find("http-settings") != std::string::npos); + REQUIRE(body.find("hunter2") == std::string::npos); + REQUIRE( + body.find("MASKED:0:f52fbd32b2b3b86ff88ef6c490628285f482af15ddcb29541f94bcf526a3f6c7") != + std::string::npos); } - SECTION("Post Configuration - Not Enabled") { + SECTION("Post Configuration - Not Enabled") + { setPostConfigEndpointEnabled(false); - auto res = cli.Post("/config", "", "application/json"); + auto [result, res] = cli.postJson("/config", ""); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 403); + REQUIRE(res->statusCode() == drogon::k403Forbidden); } - SECTION("Post Configuration - Invalid JSON Format") { + SECTION("Post Configuration - Invalid JSON Format") + { setPostConfigEndpointEnabled(true); - std::string invalidJson = "this is not valid json"; - auto res = cli.Post("/config", invalidJson, "application/json"); + auto [result, res] = cli.postJson("/config", "this is not valid json"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 400); - REQUIRE(res->body.find("Invalid JSON format") != std::string::npos); + REQUIRE(res->statusCode() == drogon::k400BadRequest); + REQUIRE(std::string(res->body()).find("Invalid JSON format") != std::string::npos); } - SECTION("Post Configuration - Missing Sources") { - std::string newConfig = R"({"http-settings": []})"; - auto res = cli.Post("/config", newConfig, "application/json"); + SECTION("Post Configuration - Missing Sources") + { + setPostConfigEndpointEnabled(true); + auto [result, res] = cli.postJson("/config", R"({"http-settings": []})"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 500); - REQUIRE(res->body.starts_with("Validation failed")); + REQUIRE(res->statusCode() == drogon::k500InternalServerError); + REQUIRE(std::string(res->body()).starts_with("Validation failed")); } - SECTION("Post Configuration - Missing Http Settings") { - std::string newConfig = R"({"sources": []})"; - auto res = cli.Post("/config", newConfig, "application/json"); + SECTION("Post Configuration - Missing Http Settings") + { + setPostConfigEndpointEnabled(true); + auto [result, res] = cli.postJson("/config", R"({"sources": []})"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 500); - REQUIRE(res->body.starts_with("Validation failed")); + REQUIRE(res->statusCode() == drogon::k500InternalServerError); + REQUIRE(std::string(res->body()).starts_with("Validation failed")); } - SECTION("Post Configuration - Valid JSON Config") { + SECTION("Post Configuration - Valid JSON Config") + { + setPostConfigEndpointEnabled(true); std::string newConfig = R"({ "sources": [{"type": "TestDataSource"}], "http-settings": [{"scope": "https://example.com", "password": "MASKED:0:f52fbd32b2b3b86ff88ef6c490628285f482af15ddcb29541f94bcf526a3f6c7"}] })"; - log().set_level(spdlog::level::trace); - auto res = cli.Post("/config", newConfig, "application/json"); + + auto [result, res] = cli.postJson("/config", newConfig); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 200); - REQUIRE(res->body == "Configuration updated and applied successfully."); + REQUIRE(res->statusCode() == drogon::k200OK); + REQUIRE(std::string(res->body()) == "Configuration updated and applied successfully."); - // Check that the password SHA was re-substituted. std::ifstream config(*mapget::DataSourceConfigService::get().getConfigFilePath()); std::stringstream configContentStream; configContentStream << config.rdbuf(); @@ -475,9 +842,5 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") REQUIRE(configContent.find("hunter2") != std::string::npos); } - service.stop(); - REQUIRE(service.isRunning() == false); - - // Clean up the test configuration files. fs::remove(tempConfigPath); } diff --git a/test/unit/test-http-service-fixture.h b/test/unit/test-http-service-fixture.h new file mode 100644 index 00000000..bbb9b0c3 --- /dev/null +++ b/test/unit/test-http-service-fixture.h @@ -0,0 +1,17 @@ +#pragma once + +#include "mapget/http-service/http-service.h" + +namespace mapget::test +{ + +// Starts the HTTP service lazily (on first use) and keeps it alive for the +// lifetime of the test process. `shutdownHttpService()` stops the server and +// joins its server thread to avoid Drogon shutdown issues. +HttpService& httpService(); + +// Safe to call even if the service was never started. +void shutdownHttpService(); + +} // namespace mapget::test + diff --git a/test/unit/test-main.cpp b/test/unit/test-main.cpp new file mode 100644 index 00000000..eb67e32f --- /dev/null +++ b/test/unit/test-main.cpp @@ -0,0 +1,50 @@ +#define CATCH_CONFIG_RUNNER + +#include + +#include + +#include "mapget/log.h" +#include "test-http-service-fixture.h" + +namespace mapget::test +{ +namespace +{ + +std::mutex serviceMutex; +HttpService* servicePtr = nullptr; + +} // namespace + +HttpService& httpService() +{ + std::lock_guard lock(serviceMutex); + + if (!servicePtr) { + // Intentionally leaked to avoid destructor ordering issues at process shutdown. + servicePtr = new HttpService(); + servicePtr->go("127.0.0.1", 0, 5000); + } + + return *servicePtr; +} + +void shutdownHttpService() +{ + std::lock_guard lock(serviceMutex); + if (!servicePtr) + return; + + servicePtr->stop(); +} + +} // namespace mapget::test + +int main(int argc, char* argv[]) +{ + auto result = Catch::Session().run(argc, argv); + mapget::test::shutdownHttpService(); + return result; +} +