diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index a159ef7..a7b93a2 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -43,7 +43,8 @@ jobs: apt-get update apt-get install -y git \ cmake make ninja-build sudo \ - python3 python3-pip python3-venv + python3 python3-pip python3-venv \ + libcurl4-gnutls-dev - name: "Run CMake" shell: bash @@ -76,7 +77,7 @@ jobs: sudo apt-get install -y git \ cmake make ninja-build sudo \ python3 python3-pip python3-venv \ - gcc g++ + gcc g++ libcurl4-gnutls-dev - name: "Compile tests" run: | diff --git a/.github/workflows/python-bindings.yml b/.github/workflows/python-bindings.yml index 67c9a6c..4166c32 100644 --- a/.github/workflows/python-bindings.yml +++ b/.github/workflows/python-bindings.yml @@ -33,7 +33,7 @@ jobs: # Install platform build dependencies - name: "Install system packages (Ubuntu)" if: startsWith(matrix.on, 'ubuntu-') - run: sudo apt-get update && sudo apt-get install -y ninja-build g++ cmake + run: sudo apt-get update && sudo apt-get install -y ninja-build g++ cmake libcurl4-gnutls-dev - name: "Setup Homebrew (macOS)" if: startsWith(matrix.on, 'macos-') @@ -41,7 +41,7 @@ jobs: - name: "Install system packages (macOS)" if: startsWith(matrix.on, 'macos-') - run: brew install ninja gcc cmake + run: brew install ninja gcc cmake curl - name: "Create virtual environment" run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d11c15..4973a33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,11 +57,19 @@ FetchContent_Declare( GIT_TAG v1.4.3 ) +FetchContent_Declare( + httplib + GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git + GIT_TAG v0.29.0 +) + + set(JSONCONS_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(JSONCONS_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) set(JSONCONS_BUILD_FUZZERS OFF CACHE BOOL "" FORCE) +set(HTTPLIB_USE_ZSTD_IF_AVAILABLE OFF CACHE BOOL "" FORCE) -FetchContent_MakeAvailable(jsoncons tomlplusplus) +FetchContent_MakeAvailable(jsoncons tomlplusplus httplib) if (BUILD_PYTHON_BINDINGS) FetchContent_Declare( @@ -135,10 +143,19 @@ target_include_directories(libcapio_cl PUBLIC ${jsoncons_SOURCE_DIR}/include ${CAPIOCL_JSON_SCHEMAS_DIRECTORY} ${TOMLPLUSPLUS_SOURCE_DIR}/include + ${httplib_SOURCE_DIR} ) target_link_libraries(libcapio_cl PUBLIC) -target_link_libraries(libcapio_cl PRIVATE tomlplusplus::tomlplusplus) +target_link_libraries(libcapio_cl PRIVATE + tomlplusplus::tomlplusplus + httplib::httplib +) + +find_library(LIBANL anl) +if(LIBANL) + target_link_libraries(libcapio_cl PRIVATE ${LIBANL}) +endif () ##################################### # Install rules @@ -173,6 +190,9 @@ endif () # Tests (only when built standalone) ##################################### if (CAPIO_CL_BUILD_TESTS) + + find_package(CURL REQUIRED) + message(STATUS "Building CAPIO-CL tests") FetchContent_Declare( @@ -189,8 +209,13 @@ if (CAPIO_CL_BUILD_TESTS) target_link_libraries(CAPIO_CL_tests PRIVATE libcapio_cl GTest::gtest_main + CURL::libcurl ) + if(LIBANL) + target_link_libraries(CAPIO_CL_tests PRIVATE ${LIBANL}) + endif () + target_include_directories(CAPIO_CL_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/src diff --git a/README.md b/README.md index 9cf0c71..1bfabc2 100644 --- a/README.md +++ b/README.md @@ -10,28 +10,25 @@ ![C++](https://img.shields.io/badge/C%2B%2B-%E2%89%A517-blueviolet?logo=c%2B%2B&logoColor=white) ![Python Bindings](https://img.shields.io/badge/Python_Bindings-3.10–3.14-darkgreen?style=flat&logo=python&logoColor=white&labelColor=gray) - #### Platform support -| OS / Arch | ![x86_64](https://img.shields.io/badge/x86__64-121212?logo=intel&logoColor=blue) | ![ARM](https://img.shields.io/badge/ARM-121212?logo=arm&logoColor=0091BD) | ![RISC-V](https://img.shields.io/badge/RISC--V-121212?logo=riscv&logoColor=F9A825) | -|-----------|----------------------------------------------------------------------------------|---------------------------------------------------------------------------|------------------------------------------------------------------------------------| +| OS / Arch | ![x86_64](https://img.shields.io/badge/x86__64-121212?logo=intel&logoColor=blue) | ![ARM](https://img.shields.io/badge/ARM-121212?logo=arm&logoColor=0091BD) | ![RISC-V](https://img.shields.io/badge/RISC--V-121212?logo=riscv&logoColor=F9A825) | +|------------------------------------------------------------------------------------|----------------------------------------------------------------------------------|---------------------------------------------------------------------------|------------------------------------------------------------------------------------| | ![Ubuntu](https://img.shields.io/badge/Ubuntu-121212?logo=ubuntu&logoColor=E95420) | YES | YES | YES | | ![macOS](https://img.shields.io/badge/macOS-121212?logo=apple&logoColor=white) | YES | YES | N.A. | - #### Documentation - [![Core Language](https://img.shields.io/badge/Core%20Language-10.1007%2Fs10766--025--00789--0-%23cc5500?logo=doi&logoColor=white&labelColor=2b2b2b)](https://doi.org/10.1007/s10766-025-00789-0) - [![Metadata Streaming](https://img.shields.io/badge/Metadata%20Streaming-10.1145%2F3731599.3767577-%23cc5500?logo=doi&logoColor=white&labelColor=2b2b2b)](https://doi.org/10.1145/3731599.3767577) - [![Doxygen documentation](https://img.shields.io/github/v/release/High-Performance-IO/CAPIO-CL?label=Doxygen%20documentation&labelColor=2b2b2b&color=brown&logo=readthedocs&logoColor=white)](https://github.com/High-Performance-IO/CAPIO-CL/releases/latest/download/documentation.pdf) - **CAPIO-CL** is a novel I/O coordination language that enables users to annotate file-based workflow data dependencies with **synchronization semantics** for files and directories. Designed to facilitate **transparent overlap between computation and I/O operations**, CAPIO-CL allows multiple producer–consumer application modules to coordinate efficiently using a **JSON-based syntax**. -For detailed documentation and examples, please visit: +For detailed documentation and examples, please visit: [![CAPIO Website](https://img.shields.io/badge/CAPIO%20Website-Documentation-brightgreen?logo=readthedocs&logoColor=white)](https://capio.hpc4ai.it/docs/coord-language/) @@ -184,7 +181,49 @@ engine.print() Serializer.dump(engine, "my_workflow", "my_workflow.json") ``` -# Team +## CapioCL Web API Documentation + +This section describes the REST-style Web API exposed by the CapioCL Web Server. +The server provides HTTP endpoints for configuring and querying the CapioCL engine at runtime. +Within the `bruno_webapi_tests` you can find several tests and examples on how to perform +requests to the API webserver using [bruno](https://www.usebruno.com). + +All endpoints communicate using JSON over HTTP. To enable the webserver, users needs to explicitly start it with: + +```cpp +capiocl::engine::Engine engine(); + +// start engine with default parameters +engine.startApiServer(); + +// or by specifying the address and port: +engine.startApiServer("127.0.0.1", 5520); +``` + + +or equivalently in python with: + +```python +engine = py_capio_cl.Engine() + +#start engine with default parameters +engine.startApiServer() + +# or by specifying the address and port: +engine.startApiServer("127.0.0.1", 5520) +``` + +By default, the webserver listens only on local connection at the following address: ```127.0.0.1:5520```. No +authentication +services are currently available, and as such, users should put particular care when allowing connections from external +endpoints. + +## Notes + +- All GET endpoints expect a JSON body containing the targeted file path. +- The API is intended for local control and orchestration, not public exposure. + +--- ## Developing team diff --git a/bindings/python_bindings.cpp b/bindings/python_bindings.cpp index 91fe5f2..864199d 100644 --- a/bindings/python_bindings.cpp +++ b/bindings/python_bindings.cpp @@ -98,6 +98,8 @@ PYBIND11_MODULE(_py_capio_cl, m) { .def("isCommitted", &capiocl::engine::Engine::isCommitted, py::arg("path")) .def("setHomeNode", &capiocl::engine::Engine::setHomeNode, py::arg("path")) .def("getPaths", &capiocl::engine::Engine::getPaths) + .def("startApiServer", &capiocl::engine::Engine::startApiServer, + py::arg("address") = "127.0.0.1", py::arg("port") = 5520) .def("__str__", &capiocl::engine::Engine::print) .def("__repr__", [](const capiocl::engine::Engine &e) { diff --git a/bruno_webapi_tests/add_consumer.bru b/bruno_webapi_tests/add_consumer.bru new file mode 100644 index 0000000..8a1db14 --- /dev/null +++ b/bruno_webapi_tests/add_consumer.bru @@ -0,0 +1,23 @@ +meta { + name: add_consumer + type: http + seq: 2 +} + +post { + url: http://localhost:5520/consumer + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt", + "consumer" : "sample2" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/add_file_dependency.bru b/bruno_webapi_tests/add_file_dependency.bru new file mode 100644 index 0000000..20bccbd --- /dev/null +++ b/bruno_webapi_tests/add_file_dependency.bru @@ -0,0 +1,23 @@ +meta { + name: add_file_dependency + type: http + seq: 4 +} + +post { + url: http://localhost:5520/dependency + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt", + "dependency" : "myFile.dat" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/add_producer.bru b/bruno_webapi_tests/add_producer.bru new file mode 100644 index 0000000..8165600 --- /dev/null +++ b/bruno_webapi_tests/add_producer.bru @@ -0,0 +1,23 @@ +meta { + name: add_producer + type: http + seq: 3 +} + +post { + url: http://localhost:5520/producer + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt", + "producer" : "sample1" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/add_workflow_name.bru b/bruno_webapi_tests/add_workflow_name.bru new file mode 100644 index 0000000..caad638 --- /dev/null +++ b/bruno_webapi_tests/add_workflow_name.bru @@ -0,0 +1,22 @@ +meta { + name: add_workflow_name + type: http + seq: 1 +} + +post { + url: http://localhost:5520/workflow + body: json + auth: inherit +} + +body:json { + { + "name": "workflow_demo" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/bruno.json b/bruno_webapi_tests/bruno.json new file mode 100644 index 0000000..04d4d50 --- /dev/null +++ b/bruno_webapi_tests/bruno.json @@ -0,0 +1,9 @@ +{ + "version": "1", + "name": "bruno_webapi_tests", + "type": "collection", + "ignore": [ + "node_modules", + ".git" + ] +} \ No newline at end of file diff --git a/bruno_webapi_tests/collection.bru b/bruno_webapi_tests/collection.bru new file mode 100644 index 0000000..e69de29 diff --git a/bruno_webapi_tests/get_commit_close_count.bru b/bruno_webapi_tests/get_commit_close_count.bru new file mode 100644 index 0000000..49a1ed3 --- /dev/null +++ b/bruno_webapi_tests/get_commit_close_count.bru @@ -0,0 +1,22 @@ +meta { + name: get_commit_close_count + type: http + seq: 18 +} + +get { + url: http://localhost:5520/commit/close-count + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/get_commit_on_n_files_count.bru b/bruno_webapi_tests/get_commit_on_n_files_count.bru new file mode 100644 index 0000000..dbc6e2b --- /dev/null +++ b/bruno_webapi_tests/get_commit_on_n_files_count.bru @@ -0,0 +1,22 @@ +meta { + name: get_commit_on_n_files_count + type: http + seq: 17 +} + +get { + url: http://localhost:5520/commit/file-count + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/get_commit_rule.bru b/bruno_webapi_tests/get_commit_rule.bru new file mode 100644 index 0000000..3950caf --- /dev/null +++ b/bruno_webapi_tests/get_commit_rule.bru @@ -0,0 +1,22 @@ +meta { + name: get_commit_rule + type: http + seq: 16 +} + +get { + url: http://localhost:5520/commit + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/get_consumer.bru b/bruno_webapi_tests/get_consumer.bru new file mode 100644 index 0000000..8b13fba --- /dev/null +++ b/bruno_webapi_tests/get_consumer.bru @@ -0,0 +1,22 @@ +meta { + name: get_consumer + type: http + seq: 14 +} + +get { + url: http://localhost:5520/consumer + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/get_directory.bru b/bruno_webapi_tests/get_directory.bru new file mode 100644 index 0000000..50c9b5c --- /dev/null +++ b/bruno_webapi_tests/get_directory.bru @@ -0,0 +1,22 @@ +meta { + name: get_directory + type: http + seq: 22 +} + +get { + url: http://localhost:5520/directory + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/get_excluded.bru b/bruno_webapi_tests/get_excluded.bru new file mode 100644 index 0000000..56ad262 --- /dev/null +++ b/bruno_webapi_tests/get_excluded.bru @@ -0,0 +1,22 @@ +meta { + name: get_excluded + type: http + seq: 21 +} + +get { + url: http://localhost:5520/exclude + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/get_file_dependencies.bru b/bruno_webapi_tests/get_file_dependencies.bru new file mode 100644 index 0000000..cff2964 --- /dev/null +++ b/bruno_webapi_tests/get_file_dependencies.bru @@ -0,0 +1,22 @@ +meta { + name: get_file_dependencies + type: http + seq: 15 +} + +get { + url: http://localhost:5520/dependency + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/get_fire_rule.bru b/bruno_webapi_tests/get_fire_rule.bru new file mode 100644 index 0000000..fe4845f --- /dev/null +++ b/bruno_webapi_tests/get_fire_rule.bru @@ -0,0 +1,22 @@ +meta { + name: get_fire_rule + type: http + seq: 19 +} + +get { + url: http://localhost:5520/fire + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/get_permanent.bru b/bruno_webapi_tests/get_permanent.bru new file mode 100644 index 0000000..87f3d53 --- /dev/null +++ b/bruno_webapi_tests/get_permanent.bru @@ -0,0 +1,22 @@ +meta { + name: get_permanent + type: http + seq: 20 +} + +get { + url: http://localhost:5520/permanent + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/get_producer.bru b/bruno_webapi_tests/get_producer.bru new file mode 100644 index 0000000..a612e47 --- /dev/null +++ b/bruno_webapi_tests/get_producer.bru @@ -0,0 +1,22 @@ +meta { + name: get_producer + type: http + seq: 13 +} + +get { + url: http://localhost:5520/producer + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/get_workflow_name.bru b/bruno_webapi_tests/get_workflow_name.bru new file mode 100644 index 0000000..6270aa1 --- /dev/null +++ b/bruno_webapi_tests/get_workflow_name.bru @@ -0,0 +1,16 @@ +meta { + name: get_workflow_name + type: http + seq: 12 +} + +get { + url: http://localhost:5520/workflow + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/set_commit_close_count.bru b/bruno_webapi_tests/set_commit_close_count.bru new file mode 100644 index 0000000..bf4e62d --- /dev/null +++ b/bruno_webapi_tests/set_commit_close_count.bru @@ -0,0 +1,23 @@ +meta { + name: set_commit_close_count + type: http + seq: 7 +} + +post { + url: http://localhost:5520/commit/close-count + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt", + "count" : 123 + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/set_commit_on_n_files_count.bru b/bruno_webapi_tests/set_commit_on_n_files_count.bru new file mode 100644 index 0000000..b0a50f2 --- /dev/null +++ b/bruno_webapi_tests/set_commit_on_n_files_count.bru @@ -0,0 +1,23 @@ +meta { + name: set_commit_on_n_files_count + type: http + seq: 6 +} + +post { + url: http://localhost:5520/commit/file-count + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt", + "count" : 10 + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/set_commit_rule.bru b/bruno_webapi_tests/set_commit_rule.bru new file mode 100644 index 0000000..c12c188 --- /dev/null +++ b/bruno_webapi_tests/set_commit_rule.bru @@ -0,0 +1,23 @@ +meta { + name: set_commit_rule + type: http + seq: 5 +} + +post { + url: http://localhost:5520/commit + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt", + "commit" : "on_close" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/set_directory.bru b/bruno_webapi_tests/set_directory.bru new file mode 100644 index 0000000..6dd873c --- /dev/null +++ b/bruno_webapi_tests/set_directory.bru @@ -0,0 +1,23 @@ +meta { + name: set_directory + type: http + seq: 11 +} + +post { + url: http://localhost:5520/directory + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt", + "directory" : true + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/set_excluded.bru b/bruno_webapi_tests/set_excluded.bru new file mode 100644 index 0000000..72584db --- /dev/null +++ b/bruno_webapi_tests/set_excluded.bru @@ -0,0 +1,23 @@ +meta { + name: set_excluded + type: http + seq: 10 +} + +post { + url: http://localhost:5520/exclude + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt", + "exclude" : true + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/set_fire_rule.bru b/bruno_webapi_tests/set_fire_rule.bru new file mode 100644 index 0000000..6edad97 --- /dev/null +++ b/bruno_webapi_tests/set_fire_rule.bru @@ -0,0 +1,23 @@ +meta { + name: set_fire_rule + type: http + seq: 8 +} + +post { + url: http://localhost:5520/fire + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt", + "fire" : "no_update" + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/bruno_webapi_tests/set_permanent.bru b/bruno_webapi_tests/set_permanent.bru new file mode 100644 index 0000000..1505180 --- /dev/null +++ b/bruno_webapi_tests/set_permanent.bru @@ -0,0 +1,23 @@ +meta { + name: set_permanent + type: http + seq: 9 +} + +post { + url: http://localhost:5520/permanent + body: json + auth: inherit +} + +body:json { + { + "path" : "/tmp/test.txt", + "permanent" : true + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/capiocl.hpp b/capiocl.hpp index aeac085..078366b 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -28,7 +28,7 @@ inline std::string sanitize(const std::string &input) { } else if (input == UPDATE) { return UPDATE; } else { - throw std::invalid_argument("Input commit rule is not a vlid CAPIO-CL rule"); + throw std::invalid_argument("Input fire rule: " + input + " is not a valid CAPIO-CL rule"); } } } // namespace fireRules @@ -59,7 +59,8 @@ inline std::string sanitize(const std::string &input) { } else if (input == ON_TERMINATION) { return ON_TERMINATION; } else { - throw std::invalid_argument("Input commit rule is not a vlid CAPIO-CL rule"); + throw std::invalid_argument("Input commit rule: " + input + + " is not a valid CAPIO-CL rule"); } } } // namespace commitRules @@ -91,6 +92,11 @@ class CapioClConfiguration; class CapioClConfigurationException; struct defaults; } // namespace configuration + +namespace webapi { +class CapioClWebApiServer; +} + } // namespace capiocl #endif // CAPIO_CL_CAPIOCL_HPP \ No newline at end of file diff --git a/capiocl/engine.h b/capiocl/engine.h index b336969..d3b1ae0 100644 --- a/capiocl/engine.h +++ b/capiocl/engine.h @@ -5,6 +5,7 @@ #include "capiocl.hpp" #include "capiocl/monitor.h" #include "capiocl/serializer.h" +#include "capiocl/webapi.h" /// @brief Namespace containing the CAPIO-CL Engine namespace capiocl::engine { @@ -39,6 +40,9 @@ class Engine final { /// @brief Name of the current workflow name std::string workflow_name; + /// @brief CAPIO-CL APIs Web Server + std::unique_ptr webapi_server; + // LCOV_EXCL_START /// @brief Internal CAPIO-CL Engine storage entity. Each CapioCLEntry is an entry for a given /// file handled by CAPIO-CL @@ -404,6 +408,13 @@ class Engine final { * Use default CAPIO-CL TOML configuration. */ void useDefaultConfiguration(); + + /** + * Start the thread involved in the handling of dynamic changes to CapioCl configuration + * @param address address to listen to. defaulto to 127.0.0.1 + * @param port Port to listen to. defaults to 5520 + */ + void startApiServer(const std::string &address = "127.0.0.1", int port = 5520); }; } // namespace capiocl::engine diff --git a/capiocl/webapi.h b/capiocl/webapi.h new file mode 100644 index 0000000..b322c2c --- /dev/null +++ b/capiocl/webapi.h @@ -0,0 +1,25 @@ +#ifndef CAPIO_CL_WEBAPI_H +#define CAPIO_CL_WEBAPI_H +#include + +#include "capiocl.hpp" + +/// @brief Class that exposes a REST Web Server to interact with the current configuration +class capiocl::webapi::CapioClWebApiServer { + + /// @brief asynchronous running webserver thread + std::thread _webApiThread; + + /// @brief port on which the current server runs + int _port; + + public: + /// @brief default constructor. + CapioClWebApiServer(engine::Engine *engine, const std::string &web_server_address, + int web_server_port); + + /// @brief Default Destructor + ~CapioClWebApiServer(); +}; + +#endif // CAPIO_CL_WEBAPI_H diff --git a/py_capio_cl/decorators.py b/py_capio_cl/decorators.py new file mode 100644 index 0000000..04af942 --- /dev/null +++ b/py_capio_cl/decorators.py @@ -0,0 +1,121 @@ +from functools import wraps + +import requests + + +def CapioCLRule(path: str, + committed: str | None = None, + fire: str | None = None, + close_count: int | None = None, + directory_n_file_expected: int | None = None, + is_directory: bool | None = None, + is_permanent: bool | None = None, + is_excluded: bool | None = None, + producers: list[str] | None = None, + consumers: list[str] | None = None, + file_dependencies: list[str] | None = None + ): + if not path: + raise RuntimeError("ERROR: cannot specify a CAPIO-CL rule without setting a path!") + + def _perform_request(endpoint, payload=None): + response = requests.post(endpoint, json=payload, headers={"content-type": "application/json"}) + json = response.json() + if "OK" not in json["status"]: + print(f"ERR: {json["what"]}") + + if committed: + _perform_request( + endpoint="http://localhost:5520/commit", + payload={ + "path": path, + "commit": committed + }) + + if fire: + _perform_request( + endpoint="http://localhost:5520/fire", + payload={ + "path": path, + "fire": fire + }) + + if close_count: + _perform_request( + endpoint="http://localhost:5520/commit/close-count", + payload={ + "path": path, + "count": close_count + }) + + if directory_n_file_expected: + _perform_request( + endpoint="http://localhost:5520/commit/file-count", + payload={ + "path": path, + "count": directory_n_file_expected + }) + + if is_directory is not None: + _perform_request( + endpoint="http://localhost:5520/directory", + payload={ + "path": path, + "directory": is_directory + } + ) + + if is_permanent is not None: + _perform_request( + endpoint="http://localhost:5520/permanent", + payload={ + "path": path, + "permanent": is_permanent + } + ) + if is_excluded is not None: + _perform_request( + endpoint="http://localhost:5520/exclude", + payload={ + "path": path, + "excluded": is_excluded + } + ) + + if producers: + for producer in producers: + _perform_request( + endpoint="http://localhost:5520/producer", + payload={ + "path": path, + "producer": producer + } + ) + + if consumers: + for consumer in consumers: + _perform_request( + endpoint="http://localhost:5520/consumer", + payload={ + "path": path, + "consumer": consumer + } + ) + + if file_dependencies: + for dependency in file_dependencies: + _perform_request( + endpoint="http://localhost:5520/dependency", + payload={ + "path": path, + "dependency": dependency + } + ) + + def _capiocl_rule(func): + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + + return _capiocl_rule diff --git a/pyproject.toml b/pyproject.toml index f7e3aee..e884c7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,9 @@ requires-python = ">=3.10" authors = [ { name = "Marco Edoardo Santimaria", email = "marcoedoardo.santimaria@unito.it" } ] +dependencies = [ + "requests==2.32.5" +] [project.urls] Homepage = "https://capio.hpc4ai.it" diff --git a/src/Engine.cpp b/src/Engine.cpp index 213ed1d..d6a37c2 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -5,6 +5,7 @@ #include "capiocl.hpp" #include "capiocl/configuration.h" #include "capiocl/engine.h" +#include "capiocl/monitor.h" #include "capiocl/printer.h" void capiocl::engine::Engine::print() const { @@ -770,4 +771,9 @@ void capiocl::engine::Engine::useDefaultConfiguration() { monitor.registerMonitorBackend(new monitor::MulticastMonitor(def_config)); monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); +} + +void capiocl::engine::Engine::startApiServer(const std::string &address, const int port) { + webapi_server = std::unique_ptr( + new webapi::CapioClWebApiServer(this, address, port)); } \ No newline at end of file diff --git a/src/webapi.cpp b/src/webapi.cpp new file mode 100644 index 0000000..5542911 --- /dev/null +++ b/src/webapi.cpp @@ -0,0 +1,259 @@ +#include "httplib.h" +#include "jsoncons/json.hpp" + +#include "capiocl/engine.h" +#include "capiocl/printer.h" +#include "capiocl/webapi.h" + +template void ok_response(Res &res) { + res.status = 200; + res.set_content(R"({"status" : "OK"})", "application/json"); +} + +template void error_response(Res &res, const std::exception &e) { + res.status = 400; + res.set_content(std::string(R"({"status" : "error", "what" : ")") + + "Invalid request BODY data: " + e.what() + "\"}", + "application/json"); +} + +template void json_response(Res &res, const jsoncons::json &body) { + res.status = 200; + res.set_content(body.as_string(), "application/json"); +} + +template +void process_post_request(const Req &req, Res &res, Fn &&handler) { + try { + jsoncons::json request_body = jsoncons::json::parse(req.body.empty() ? "{}" : req.body); + + handler(request_body); + ok_response(res); + } catch (const std::exception &e) { + error_response(res, e); + } +} + +template +void process_get_request(const Req &req, Res &res, Fn &&handler) { + try { + jsoncons::json request_body = jsoncons::json::parse(req.body.empty() ? "{}" : req.body); + + jsoncons::json reply; + handler(request_body, reply); + json_response(res, reply); + } catch (const std::exception &e) { + error_response(res, e); + } +} + +/// @brief Main WebServer thread function +void server(const std::string &address, const int port, capiocl::engine::Engine *engine) { + + capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, + "Starting API server @ " + address + ":" + std::to_string(port)); + + httplib::Server _server; + + _server.Post("/producer", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto path = request_body["path"].as(); + auto producer = request_body["producer"].as(); + engine->addProducer(path, producer); + }); + }); + + _server.Get("/producer", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request(req, res, [&](jsoncons::json &request_body, jsoncons::json &reply) { + const auto path = request_body["path"].as(); + reply["producers"] = engine->getProducers(path); + }); + }); + + _server.Post("/consumer", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto path = request_body["path"].as(); + auto consumer = request_body["consumer"].as(); + engine->addConsumer(path, consumer); + }); + }); + + _server.Get("/consumer", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request(req, res, [&](jsoncons::json &request_body, jsoncons::json &reply) { + const auto path = request_body["path"].as(); + reply["consumers"] = engine->getConsumers(path); + }); + }); + + _server.Post("/dependency", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto path = request_body["path"].as(); + auto dependency = std::filesystem::path(request_body["dependency"].as()); + engine->addFileDependency(path, dependency); + }); + }); + + _server.Get("/dependency", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request(req, res, [&](jsoncons::json &request_body, jsoncons::json &reply) { + const auto path = request_body["path"].as(); + std::vector deps; + for (const auto &file : engine->getCommitOnFileDependencies(path)) { + deps.emplace_back(file); + } + reply["dependencies"] = deps; + }); + }); + + _server.Post("/commit", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto path = request_body["path"].as(); + auto commit_rule = request_body["commit"].as(); + engine->setCommitRule(path, commit_rule); + }); + }); + + _server.Get("/commit", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request(req, res, [&](jsoncons::json &request_body, jsoncons::json &reply) { + const auto path = request_body["path"].as(); + reply["commit"] = engine->getCommitRule(path); + }); + }); + + _server.Post("/commit/file-count", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto path = request_body["path"].as(); + auto count = request_body["count"].as(); + engine->setDirectoryFileCount(path, count); + }); + }); + + _server.Get("/commit/file-count", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request(req, res, [&](jsoncons::json &request_body, jsoncons::json &reply) { + const auto path = request_body["path"].as(); + reply["count"] = engine->getDirectoryFileCount(path); + }); + }); + + _server.Post("/commit/close-count", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto path = request_body["path"].as(); + auto count = request_body["count"].as(); + engine->setCommitedCloseNumber(path, count); + }); + }); + + _server.Get("/commit/close-count", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request(req, res, [&](jsoncons::json &request_body, jsoncons::json &reply) { + const auto path = request_body["path"].as(); + reply["count"] = engine->getCommitCloseCount(path); + }); + }); + + _server.Post("/fire", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto path = request_body["path"].as(); + auto fire_rule = request_body["fire"].as(); + engine->setFireRule(path, fire_rule); + }); + }); + + _server.Get("/fire", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request(req, res, [&](jsoncons::json &request_body, jsoncons::json &reply) { + const auto path = request_body["path"].as(); + reply["fire"] = engine->getFireRule(path); + }); + }); + + _server.Post("/permanent", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto path = request_body["path"].as(); + const auto permanent = request_body["permanent"].as(); + engine->setPermanent(path, permanent); + }); + }); + + _server.Get("/permanent", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request(req, res, [&](jsoncons::json &request_body, jsoncons::json &reply) { + const auto path = request_body["path"].as(); + reply["permanent"] = engine->isPermanent(path); + }); + }); + + _server.Post("/exclude", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto path = request_body["path"].as(); + const auto excluded = request_body["exclude"].as(); + engine->setExclude(path, excluded); + }); + }); + + _server.Get("/exclude", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request(req, res, [&](jsoncons::json &request_body, jsoncons::json &reply) { + const auto path = request_body["path"].as(); + reply["exclude"] = engine->isExcluded(path); + }); + }); + + _server.Post("/directory", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto path = request_body["path"].as(); + if (request_body["directory"].as()) { + engine->setDirectory(path); + } else { + engine->setFile(path); + } + }); + }); + + _server.Get("/directory", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request(req, res, [&](jsoncons::json &request_body, jsoncons::json &reply) { + const auto path = request_body["path"].as(); + reply["directory"] = engine->isDirectory(path); + }); + }); + + _server.Post("/workflow", [&](const httplib::Request &req, httplib::Response &res) { + process_post_request(req, res, [&](jsoncons::json &request_body) { + const auto workflow_name = request_body["name"].as(); + engine->setWorkflowName(workflow_name); + }); + }); + + _server.Get("/workflow", [&](const httplib::Request &req, httplib::Response &res) { + process_get_request( + req, res, [&]([[maybe_unused]] jsoncons::json &request_body, jsoncons::json &reply) { + reply["name"] = engine->getWorkflowName(); + }); + }); + + _server.Get("/terminate", [&]([[maybe_unused]] const httplib::Request &req, + [[maybe_unused]] httplib::Response &res) { + process_get_request(req, res, + [&]([[maybe_unused]] jsoncons::json &request_body, + [[maybe_unused]] jsoncons::json &reply) { + capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, + "API server stopped"); + _server.stop(); + }); + }); + + _server.listen(address, port); +} + +capiocl::webapi::CapioClWebApiServer::CapioClWebApiServer(engine::Engine *engine, + const std::string &web_server_address, + const int web_server_port) + : _port(web_server_port) { + _webApiThread = std::thread(server, web_server_address, web_server_port, engine); +} + +capiocl::webapi::CapioClWebApiServer::~CapioClWebApiServer() { + + httplib::Client client("http://127.0.0.1:" + std::to_string(_port)); + client.Get("/terminate"); + if (_webApiThread.joinable()) { + _webApiThread.join(); + } else { + return; + } +} diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 2839dfb..29f742f 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -21,6 +21,7 @@ template std::string demangled_name(const T &obj) { #include "capiocl/printer.h" #include "capiocl/serializer.h" +#include "test_apis.hpp" #include "test_configuration.hpp" #include "test_engine.hpp" #include "test_exceptions.hpp" diff --git a/tests/cpp/test_apis.hpp b/tests/cpp/test_apis.hpp new file mode 100644 index 0000000..c3312b4 --- /dev/null +++ b/tests/cpp/test_apis.hpp @@ -0,0 +1,238 @@ +#ifndef CAPIO_CL_TEST_APIS_HPP +#define CAPIO_CL_TEST_APIS_HPP + +#define WEBSERVER_SUITE_NAME TestWebServerAPIS + +#include "jsoncons/json.hpp" +#include +#include +#include +#include +#include + +enum class HttpMethod { GET, POST, DELETE }; + +static size_t curl_write_response_handler(const char *ptr, const size_t size, size_t nmemb, + void *userdata) { + auto *response = static_cast(userdata); + response->append(ptr, size * nmemb); + return size * nmemb; +} + +inline jsoncons::json perform_request(const std::string &url, + const std::string &request_params_json_encode, + HttpMethod method = HttpMethod::GET) { + CURL *curl = curl_easy_init(); + if (!curl) { + throw std::runtime_error("curl_easy_init failed"); + } + + std::string response; + + curl_slist *headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + headers = curl_slist_append(headers, "Accept: application/json"); + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request_params_json_encode.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, request_params_json_encode.size()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_response_handler); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response); + + switch (method) { + case HttpMethod::GET: + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET"); + break; + + case HttpMethod::POST: + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST"); + break; + + case HttpMethod::DELETE: + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + break; + } + + const CURLcode res = curl_easy_perform(curl); + + long http_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + if (res != CURLE_OK) { + throw std::runtime_error(curl_easy_strerror(res)); + } + + std::cout << "DBG RES: " << response << std::endl; + return jsoncons::json::parse(std::string(response)); +} + +TEST(WEBSERVER_SUITE_NAME, testGetAndSetWorkflowName) { + + // clean environment for wf name + unsetenv("WORKFLOW_NAME"); + + auto engine = capiocl::engine::Engine(); + engine.startApiServer(); + + sleep(1); + + auto response = perform_request("http://localhost:5520/workflow", "{}", HttpMethod::GET); + + EXPECT_FALSE(response.empty()); + EXPECT_TRUE(response["name"] == capiocl::CAPIO_CL_DEFAULT_WF_NAME); + + perform_request("http://localhost:5520/workflow", R"({"name": "test_workflow_0"})", + HttpMethod::POST); + response = perform_request("http://localhost:5520/workflow", "{}", HttpMethod::GET); + EXPECT_FALSE(response.empty()); + EXPECT_TRUE(response["name"] == "test_workflow_0"); +} + +TEST(WEBSERVER_SUITE_NAME, consumer) { + capiocl::engine::Engine engine; + engine.startApiServer(); + sleep(1); + + auto result = + perform_request("http://localhost:5520/consumer", + R"({"path" : "/tmp/test.txt", "consumer" : "sample2"})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/consumer", R"({"path" : "/tmp/test.txt"})", + HttpMethod::GET); + EXPECT_TRUE(result["consumers"][0] == "sample2"); +} + +TEST(WEBSERVER_SUITE_NAME, producer) { + capiocl::engine::Engine engine; + engine.startApiServer(); + sleep(1); + + auto result = + perform_request("http://localhost:5520/producer", + R"({"path" : "/tmp/test.txt", "producer" : "sample1"})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/producer", R"({"path" : "/tmp/test.txt"})", + HttpMethod::GET); + EXPECT_TRUE(result["producers"][0] == "sample1"); +} + +TEST(WEBSERVER_SUITE_NAME, commit) { + capiocl::engine::Engine engine; + engine.startApiServer(); + sleep(1); + + auto result = + perform_request("http://localhost:5520/commit", + R"({"path" : "/tmp/test.txt","commit" : "on_file"})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/commit", R"({"path" : "/tmp/test.txt"})", + HttpMethod::GET); + EXPECT_TRUE(result["commit"] == "on_file"); +} + +TEST(WEBSERVER_SUITE_NAME, fire) { + capiocl::engine::Engine engine; + engine.startApiServer(); + sleep(1); + + auto result = + perform_request("http://localhost:5520/fire", + R"({"path" : "/tmp/test.txt","fire" : "no_update"})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/fire", R"({"path" : "/tmp/test.txt"})", + HttpMethod::GET); + EXPECT_TRUE(result["fire"] == "no_update"); +} + +TEST(WEBSERVER_SUITE_NAME, fileDependency) { + capiocl::engine::Engine engine; + engine.startApiServer(); + sleep(1); + + auto result = perform_request("http://localhost:5520/dependency", + R"({"path" : "/tmp/test.txt", "dependency" : "myFile.dat"})", + HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/dependency", R"({"path" : "/tmp/test.txt"})", + HttpMethod::GET); + EXPECT_TRUE(result["dependencies"][0] == "myFile.dat"); +} + +TEST(WEBSERVER_SUITE_NAME, on_n_files) { + capiocl::engine::Engine engine; + engine.startApiServer(); + sleep(1); + + auto result = perform_request("http://localhost:5520/commit/file-count", + R"({"path" : "/tmp/test.txt","count" : 7892})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/commit/file-count", + R"({"path" : "/tmp/test.txt"})", HttpMethod::GET); + EXPECT_EQ(result["count"], 7892); +} + +TEST(WEBSERVER_SUITE_NAME, close_count) { + capiocl::engine::Engine engine; + engine.startApiServer(); + sleep(1); + + auto result = + perform_request("http://localhost:5520/commit/close-count", + R"({"path" : "/tmp/test.txt","count" : 12345})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/commit/close-count", + R"({"path" : "/tmp/test.txt"})", HttpMethod::GET); + EXPECT_EQ(result["count"], 12345); +} + +TEST(WEBSERVER_SUITE_NAME, test_error) { + capiocl::engine::Engine engine; + engine.startApiServer(); + sleep(1); + + auto result = perform_request("http://localhost:5520/commit", R"({})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "error"); + EXPECT_GT(result["what"].as_string().size(), 0); +} + +TEST(WEBSERVER_SUITE_NAME, boolean_flag) { + + capiocl::engine::Engine engine; + engine.startApiServer(); + sleep(1); + + auto result = + perform_request("http://localhost:5520/permanent", + R"({"path" : "/tmp/test.txt","permanent" : true})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/permanent", R"({"path" : "/tmp/test.txt"})", + HttpMethod::GET); + EXPECT_TRUE(result["permanent"].as_bool()); + + result = perform_request("http://localhost:5520/exclude", + R"({"path" : "/tmp/test.txt","exclude" : true})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/exclude", R"({"path" : "/tmp/test.txt"})", + HttpMethod::GET); + EXPECT_TRUE(result["exclude"].as_bool()); + + result = perform_request("http://localhost:5520/directory", + R"({"path" : "/tmp/test.txt","directory" : true})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/directory", R"({"path" : "/tmp/test.txt"})", + HttpMethod::GET); + EXPECT_TRUE(result["directory"].as_bool()); + + result = perform_request("http://localhost:5520/directory", + R"({"path" : "/tmp/test.txt","directory" : false})", HttpMethod::POST); + EXPECT_TRUE(result["status"] == "OK"); + result = perform_request("http://localhost:5520/directory", R"({"path" : "/tmp/test.txt"})", + HttpMethod::GET); + EXPECT_FALSE(result["directory"].as_bool()); +} + +#endif // CAPIO_CL_TEST_APIS_HPP