Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ endif()
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

if (WIN32)
add_compile_definitions(NOMINMAX)
endif()

include(FetchContent)
include(GNUInstallDirs)

Expand All @@ -22,12 +26,21 @@ if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
set(MAPGET_WITH_HTTPLIB ON CACHE BOOL "Enable mapget-http-datasource and mapget-http-service libraries.")
set(MAPGET_ENABLE_TESTING ON CACHE BOOL "Enable testing.")
set(MAPGET_BUILD_EXAMPLES ON CACHE BOOL "Build examples.")
# Prevent CPM dependencies (e.g. Drogon/Trantor) from registering their own
# tests into this project's ctest run.
set(BUILD_TESTING OFF CACHE BOOL "Disable dependency tests" FORCE)
endif()

option(MAPGET_WITH_WHEEL "Enable mapget Python wheel (output to WHEEL_DEPLOY_DIRECTORY).")
option(MAPGET_WITH_SERVICE "Enable mapget-service library. Requires threads.")
option(MAPGET_WITH_HTTPLIB "Enable mapget-http-datasource and mapget-http-service libraries.")

if (MAPGET_ENABLE_TESTING)
# Enable testing before adding CPM dependencies so stale/third-party CTest
# files don't linger in the build tree.
enable_testing()
endif()

set(Python3_FIND_STRATEGY LOCATION)

if (NOT MSVC)
Expand Down Expand Up @@ -112,7 +125,6 @@ endif()
# tests

if (MAPGET_ENABLE_TESTING)
enable_testing()
add_subdirectory(test/unit)

if (MAPGET_WITH_WHEEL)
Expand Down
92 changes: 78 additions & 14 deletions cmake/deps.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ CPMAddPackage(
"EXPECTED_BUILD_TESTS OFF"
"EXPECTED_BUILD_PACKAGE_DEB OFF")
CPMAddPackage(
URI "gh:Klebert-Engineering/simfil@0.6.2"
URI "gh:Klebert-Engineering/simfil@0.6.3#v0.6.3"
OPTIONS
"SIMFIL_WITH_MODEL_JSON ON"
"SIMFIL_SHARED OFF")
Expand All @@ -25,6 +25,21 @@ CPMAddPackage(
"BUILD_TESTING OFF")

if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
# OpenSSL's Configure script needs a "full" Perl distribution. Git for
# Windows ships a minimal perl that is missing required modules (e.g.
# Locale::Maketext::Simple), causing OpenSSL builds to fail.
if (WIN32)
if (NOT DEFINED PERL_EXECUTABLE OR PERL_EXECUTABLE MATCHES "[\\\\/]Git[\\\\/]usr[\\\\/]bin[\\\\/]perl\\.exe$")
find_program(_MAPGET_STRAWBERRY_PERL
NAMES perl.exe
PATHS "C:/Strawberry/perl/bin"
NO_DEFAULT_PATH)
if (_MAPGET_STRAWBERRY_PERL)
set(PERL_EXECUTABLE "${_MAPGET_STRAWBERRY_PERL}" CACHE FILEPATH "" FORCE)
endif()
endif()
endif()

set (OPENSSL_VERSION openssl-3.5.2)
CPMAddPackage("gh:klebert-engineering/openssl-cmake@1.0.0")
CPMAddPackage(
Expand All @@ -40,19 +55,67 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
endif()

CPMAddPackage(
URI "gh:yhirose/cpp-httplib@0.15.3"
NAME jsoncpp
GIT_REPOSITORY https://github.com/open-source-parsers/jsoncpp
GIT_TAG 1.9.5
GIT_SHALLOW ON
OPTIONS
"JSONCPP_WITH_TESTS OFF"
"JSONCPP_WITH_POST_BUILD_UNITTEST OFF"
"JSONCPP_WITH_PKGCONFIG_SUPPORT OFF"
"JSONCPP_WITH_CMAKE_PACKAGE OFF"
"BUILD_SHARED_LIBS OFF"
"BUILD_STATIC_LIBS ON"
"BUILD_OBJECT_LIBS OFF")
# Help Drogon's FindJsoncpp.cmake locate jsoncpp when built via CPM.
set(JSONCPP_INCLUDE_DIRS "${jsoncpp_SOURCE_DIR}/include" CACHE PATH "" FORCE)
set(JSONCPP_LIBRARIES jsoncpp_static CACHE STRING "" FORCE)
# CPM generates a dummy package redirect config at
# `${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/jsoncpp-config.cmake`. Drogon uses
# `find_package(Jsoncpp)` (config-first), so make that redirect actually
# define the expected `Jsoncpp_lib` target.
if (DEFINED CMAKE_FIND_PACKAGE_REDIRECTS_DIR)
file(MAKE_DIRECTORY "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}")
file(WRITE "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/jsoncpp-extra.cmake" [=[
if(NOT TARGET Jsoncpp_lib)
add_library(Jsoncpp_lib INTERFACE)
target_include_directories(Jsoncpp_lib INTERFACE "${JSONCPP_INCLUDE_DIRS}")
target_link_libraries(Jsoncpp_lib INTERFACE ${JSONCPP_LIBRARIES})
endif()
]=])
endif()

# Drogon defines install(EXPORT ...) rules unconditionally, which fail when
# used as a subproject with CPM-provided dependencies (zlib/jsoncpp/etc).
# Since mapget only needs Drogon for building, temporarily suppress install
# rule generation while configuring Drogon.
set(_MAPGET_PREV_SKIP_INSTALL_RULES "${CMAKE_SKIP_INSTALL_RULES}")
if (DEFINED BUILD_TESTING)
set(_MAPGET_PREV_BUILD_TESTING "${BUILD_TESTING}")
endif()
set(CMAKE_SKIP_INSTALL_RULES ON)
set(BUILD_TESTING OFF)

CPMAddPackage(
URI "gh:drogonframework/drogon@1.9.7"
OPTIONS
"CPPHTTPLIB_USE_POLL ON"
"HTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN OFF"
"HTTPLIB_INSTALL OFF"
"HTTPLIB_USE_OPENSSL_IF_AVAILABLE OFF"
"HTTPLIB_USE_ZLIB_IF_AVAILABLE OFF")
# Manually enable openssl/zlib in httplib to avoid FindPackage calls.
target_compile_definitions(httplib INTERFACE
CPPHTTPLIB_OPENSSL_SUPPORT
CPPHTTPLIB_ZLIB_SUPPORT)
target_link_libraries(httplib INTERFACE
OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB)
"BUILD_CTL OFF"
"BUILD_EXAMPLES OFF"
"BUILD_ORM OFF"
"BUILD_BROTLI OFF"
"BUILD_YAML_CONFIG OFF"
"BUILD_SHARED_LIBS OFF"
"USE_SUBMODULE ON"
"USE_STATIC_LIBS_ONLY OFF"
"USE_POSTGRESQL OFF"
"USE_MYSQL OFF"
"USE_SQLITE3 OFF"
GIT_SUBMODULES "trantor")

set(CMAKE_SKIP_INSTALL_RULES "${_MAPGET_PREV_SKIP_INSTALL_RULES}")
if (DEFINED _MAPGET_PREV_BUILD_TESTING)
set(BUILD_TESTING "${_MAPGET_PREV_BUILD_TESTING}")
endif()

CPMAddPackage(
URI "gh:jbeder/yaml-cpp#aa8d4e@0.8.0" # Use > 0.8.0 once available.
Expand All @@ -64,6 +127,7 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
CPMAddPackage("gh:CLIUtils/CLI11@2.5.0")
CPMAddPackage("gh:pboettch/json-schema-validator#2.3.0")
CPMAddPackage("gh:okdshin/PicoSHA2@1.0.1")

endif ()

if (MAPGET_WITH_WHEEL AND NOT TARGET pybind11)
Expand All @@ -76,7 +140,7 @@ if (MAPGET_WITH_SERVICE OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
endif()

if (MAPGET_WITH_WHEEL AND NOT TARGET python-cmake-wheel)
CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel@1.1.0")
CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel#80592e483cc2be044f64e35c4686a00a9126abd2@1.2.1")
endif()

if (MAPGET_ENABLE_TESTING)
Expand Down
36 changes: 20 additions & 16 deletions docs/mapget-api.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
# REST/GeoJSON API Guide
# HTTP / WebSocket API Guide

Mapget exposes a small HTTP API that lets clients discover datasources, stream tiles, abort long‑running requests, locate features by ID and inspect or update the running configuration. This guide describes the endpoints and their request and response formats.
Mapget exposes a small HTTP + WebSocket API that lets clients discover datasources, stream tiles, locate features by ID and inspect or update the running configuration. This guide describes the endpoints and their request and response formats.

## Base URL and formats

The server started by `mapget serve` listens on the configured host and port (by default on all interfaces and an automatically chosen port). All endpoints are rooted at that host and port.

Requests that send JSON use `Content-Type: application/json`. Tile streaming supports two response encodings, selected via the `Accept` header:
Requests that send JSON use `Content-Type: application/json`. HTTP tile streaming supports two response encodings, selected via the `Accept` header:

- `Accept: application/jsonl` returns a JSON‑Lines stream where each line is one JSON object.
- `Accept: application/binary` returns a compact binary stream optimized for high-volume traffic.
Expand All @@ -23,9 +23,9 @@ The binary format and the logical feature model are described in more detail in

Each item contains map ID, available layers and basic metadata. This endpoint is typically used by frontends to discover which maps and layers can be requested via `/tiles`.

## `/tiles` – stream tiles
## `/tiles` – stream tiles (HTTP)

`POST /tiles` streams tiles for one or more map–layer combinations. It is the main data retrieval endpoint used by clients such as erdblick.
`POST /tiles` streams tiles for one or more map–layer combinations.

- **Method:** `POST`
- **Request body (JSON):**
Expand All @@ -34,7 +34,6 @@ Each item contains map ID, available layers and basic metadata. This endpoint is
- `layerId`: string, ID of the layer within that map.
- `tileIds`: array of numeric tile IDs in mapget’s tiling scheme.
- `stringPoolOffsets` (optional): dictionary from datasource node ID to last known string ID. Used by advanced clients to avoid receiving the same field names repeatedly in the binary stream.
- `clientId` (optional): arbitrary string identifying this client connection for abort handling.
- **Response:**
- `application/jsonl` if `Accept: application/jsonl` is sent.
- `application/binary` if `Accept: application/binary` is sent, using the tile stream protocol.
Expand All @@ -43,6 +42,21 @@ Tiles are streamed as they become available. In JSONL mode, each line is the JSO

If `Accept-Encoding: gzip` is set, the server compresses responses where possible, which is especially useful for JSONL streams.

To cancel an in-flight HTTP stream, close the HTTP connection.

## `/tiles` – stream tiles (WebSocket)

`GET /tiles` supports WebSocket upgrades. This is the preferred tile streaming mode for interactive clients because it supports long-lived connections and request replacement without introducing an extra abort endpoint.

- **Connect:** `ws://<host>:<port>/tiles`
- **Client → Server:** send one *text* message containing the same JSON body as for `POST /tiles` (`requests`, optional `stringPoolOffsets`).
- `stringPoolOffsets` is optional; the server remembers the latest offsets per WebSocket connection. Clients may re-send it to reset/resync offsets.
- **Server → Client:** sends only *binary* WebSocket messages. Each WebSocket message contains exactly one `TileLayerStream` VTLV frame.
- `StringPool`, `TileFeatureLayer`, `TileSourceDataLayer` are unchanged.
- `Status` frames contain UTF-8 JSON payload describing per-request `RequestStatus` transitions and a human-readable message. The final status frame has `"allDone": true`.

To cancel, either send a new request message on the same connection (which replaces the current one) or close the WebSocket connection.

### Why JSONL instead of JSON?

JSON Lines is better suited to streaming large responses than a single JSON array. Clients can start processing the first tiles immediately, do not need to buffer the complete response in memory, and can naturally consume the stream with incremental parsers.
Expand Down Expand Up @@ -87,16 +101,6 @@ Each line in the JSONL response is a GeoJSON-like FeatureCollection with additio

The `error` object is only present if an error occurred while filling the tile. When present, the `features` array may be empty or contain partial data.

## `/abort` – cancel tile streaming

`POST /abort` cancels a running `/tiles` request that was started with a matching `clientId`. It is useful when the viewport changes and the previous stream should be abandoned.

- **Method:** `POST`
- **Request body (JSON):** `{ "clientId": "<same-id-used-for-tiles>" }`
- **Response:** `text/plain` confirmation; a 400 status code if `clientId` is missing.

Internally the service marks the matching tile requests as aborted and stops scheduling further work for them.

### Curl Call Example

For example, the following curl call could be used to stream GeoJSON feature objects
Expand Down
13 changes: 6 additions & 7 deletions docs/mapget-dev-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ sequenceDiagram
participant Ds as DataSource
participant Cache

Client->>Http: POST /tiles<br>requests, clientId
Client->>Http: POST /tiles<br>requests
Http->>Service: request(requests, headers)
Service->>Worker: enqueue jobs per datasource
loop per tile
Expand All @@ -169,30 +169,29 @@ sequenceDiagram
Service-->>Http: request complete
```

If a client supplies a `clientId` in the `/tiles` request, the HTTP layer uses it to track open requests and to implement `/abort`.
For interactive clients, tile streaming can also be done via WebSocket `GET /tiles`, where sending a new request message replaces the current in-flight request on that connection.

## HTTP service internals

`mapget::HttpService` binds the core service to an HTTP server implementation. Its responsibilities are:

- map HTTP endpoints to service calls (`/sources`, `/tiles`, `/abort`, `/status`, `/locate`, `/config`),
- map HTTP/WebSocket endpoints to service calls (`/sources`, `/tiles`, `/status`, `/locate`, `/config`),
- parse JSON requests and build `LayerTilesRequest` objects,
- serialize tile responses as JSONL or binary streams,
- manage per‑client state such as `clientId` for abort handling, and
- provide `/config` as a JSON view on the YAML config file.

### Tile streaming

For `/tiles`, the HTTP layer:

- parses the JSON body to extract `requests`, `stringPoolOffsets` and an optional `clientId`,
- parses the JSON body to extract `requests` and optional `stringPoolOffsets`,
- constructs one `LayerTilesRequest` per map–layer combination,
- attaches callbacks that feed results into a shared `HttpTilesRequestState`, and
- sends out each tile as soon as it is produced by the service.

In JSONL mode the response is a sequence of newline‑separated JSON objects. In binary mode the HTTP layer uses `TileLayerStream::Writer` to serialize string pool updates and tile blobs. Binary responses can optionally be compressed using gzip if the client sends `Accept-Encoding: gzip`.

The `/abort` endpoint uses the `clientId` mechanism to cancel all open tile requests for a given client and to prevent further work from being scheduled for them.
WebSocket `/tiles` uses the same request JSON shape but responds with binary VTLV frames only, and includes `Status` frames (JSON payload) whenever a request’s `RequestStatus` changes.

### Configuration endpoints

Expand All @@ -207,7 +206,7 @@ These endpoints are guarded by command‑line flags: `--no-get-config` disables

The model library provides both the binary tile encoding and the simfil query integration:

- `TileLayerStream::Writer` and `TileLayerStream::Reader` handle versioned, type‑tagged messages for string pools and tile layers. Each message starts with a protocol version, a `MessageType` (string pool, feature tile, SourceData tile, end-of-stream), and a payload size.
- `TileLayerStream::Writer` and `TileLayerStream::Reader` handle versioned, type‑tagged messages for string pools and tile layers. Each message starts with a protocol version, a `MessageType` (string pool, feature tile, SourceData tile, status, end-of-stream), and a payload size.
- `TileFeatureLayer` derives from `simfil::ModelPool` and exposes methods such as `evaluate(...)` and `complete(...)` to run simfil expressions and obtain completion candidates.

String pools are streamed incrementally. The server keeps a `StringPoolOffsetMap` that tracks, for each ongoing tile request, the highest string ID known to a given client per datasource node id. When a tile is written, `TileLayerStream::Writer` compares that offset with the current `StringPool::highest()` value:
Expand Down
2 changes: 1 addition & 1 deletion docs/mapget-user-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ The guide is split into several focused documents:

- [**Setup Guide**](mapget-setup.md) explains how to install mapget via `pip`, how to build the native executable from source, and how to start a server or use the built‑in `fetch` client for quick experiments.
- [**Configuration Guide**](mapget-config.md) documents the YAML configuration file used with `--config`, the supported datasource types (`DataSourceHost`, `DataSourceProcess`, `GridDataSource`, `GeoJsonFolder) and the optional `http-settings` section used by tools and UIs.
- [**REST API Guide**](mapget-api.md) describes the HTTP endpoints exposed by `mapget serve`, including `/sources`, `/tiles`, `/abort`, `/status`, `/locate` and `/config`, along with their request and response formats and example calls.
- [**HTTP / WebSocket API Guide**](mapget-api.md) describes the endpoints exposed by `mapget serve`, including `/sources`, `/tiles`, `/status`, `/locate` and `/config`, along with their request and response formats and example calls.
- [**Caching Guide**](mapget-cache.md) covers the available cache modes (`memory`, `persistent`, `none`), explains how to configure cache size and location, and shows how to inspect cache statistics via the status endpoint.
- [**Simfil Language Extensions**](mapget-simfil-extensions.md) introduces the feature model, tiling scheme, geometry and validity concepts, and the binary tile stream format. This chapter is especially relevant if you are writing datasources or low‑level clients.
- [**Layered Data Model**](mapget-model.md) introduces the feature model, tiling scheme, geometry and validity concepts, and the binary tile stream format. This chapter is especially relevant if you are writing datasources or low‑level clients.
Expand Down
2 changes: 1 addition & 1 deletion libs/http-datasource/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ target_include_directories(mapget-http-datasource

target_link_libraries(mapget-http-datasource
PUBLIC
httplib::httplib
drogon
mapget-model
mapget-service
tiny-process-library)
Expand Down
9 changes: 5 additions & 4 deletions libs/http-datasource/include/mapget/detail/http-server.h
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
#pragma once

#include <cstdint>
#include <memory>
#include <string>

// Pre-declare httplib::Server to avoid including httplib.h in header
namespace httplib {
class Server;
// Forward declare Drogon app type to avoid including drogon headers in public headers.
namespace drogon {
class HttpAppFramework;
}

namespace mapget {
Expand Down Expand Up @@ -76,7 +77,7 @@ class HttpServer
* This function is called upon the first call to go(),
* and allows any derived server class to add endpoints.
*/
virtual void setup(httplib::Server&) = 0;
virtual void setup(drogon::HttpAppFramework&) = 0;

/**
* Derived servers can use this to control whether
Expand Down
Loading
Loading