From 670d2923b79c8191aa267a9e9656cc5333c5e8ba Mon Sep 17 00:00:00 2001 From: David Alonso de la Torre Date: Wed, 5 Mar 2025 12:19:05 +0100 Subject: [PATCH 1/9] Add zstd support --- CMakeLists.txt | 17 ++++ httplib.h | 131 ++++++++++++++++++++++++- test/test.cc | 253 ++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 397 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 61419c63a8..a64a2b28b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,7 @@ * HTTPLIB_USE_OPENSSL_IF_AVAILABLE (default on) * HTTPLIB_USE_ZLIB_IF_AVAILABLE (default on) * HTTPLIB_USE_BROTLI_IF_AVAILABLE (default on) + * HTTPLIB_USE_ZSTD_IF_AVAILABLE (default on) * HTTPLIB_REQUIRE_OPENSSL (default off) * HTTPLIB_REQUIRE_ZLIB (default off) * HTTPLIB_REQUIRE_BROTLI (default off) @@ -45,6 +46,7 @@ * HTTPLIB_IS_USING_OPENSSL - a bool for if OpenSSL support is enabled. * HTTPLIB_IS_USING_ZLIB - a bool for if ZLIB support is enabled. * HTTPLIB_IS_USING_BROTLI - a bool for if Brotli support is enabled. + * HTTPLIB_IS_USING_ZSTD - a bool for if ZSTD support is enabled. * HTTPLIB_IS_USING_CERTS_FROM_MACOSX_KEYCHAIN - a bool for if support of loading system certs from the Apple Keychain is enabled. * HTTPLIB_IS_COMPILED - a bool for if the library is compiled, or otherwise header-only. * HTTPLIB_INCLUDE_DIR - the root path to httplib's header (e.g. /usr/include). @@ -101,6 +103,8 @@ option(HTTPLIB_TEST "Enables testing and builds tests" OFF) option(HTTPLIB_REQUIRE_BROTLI "Requires Brotli to be found & linked, or fails build." OFF) option(HTTPLIB_USE_BROTLI_IF_AVAILABLE "Uses Brotli (if available) to enable Brotli decompression support." ON) option(HTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN "Enable feature to load system certs from the Apple Keychain." ON) +option(HTTPLIB_REQUIRE_ZSTD "Requires ZSTD to be found & linked, or fails build." OFF) +option(HTTPLIB_USE_ZSTD_IF_AVAILABLE "Uses ZSTD (if available) to enable zstd support." ON) # Defaults to static library option(BUILD_SHARED_LIBS "Build the library as a shared library instead of static. Has no effect if using header-only." OFF) if (BUILD_SHARED_LIBS AND WIN32 AND HTTPLIB_COMPILE) @@ -153,6 +157,17 @@ elseif(HTTPLIB_USE_BROTLI_IF_AVAILABLE) set(HTTPLIB_IS_USING_BROTLI ${Brotli_FOUND}) endif() +if(HTTPLIB_REQUIRE_ZSTD) + find_package(ZSTD REQUIRED) + set(HTTPLIB_IS_USING_ZSTD TRUE) +elseif(HTTPLIB_USE_ZSTD_IF_AVAILABLE) + find_package(ZSTD QUIET) + # FindZLIB doesn't have a ZLIB_FOUND variable, so check the target. + if(TARGET ZSTD::ZSTD) + set(HTTPLIB_IS_USING_ZSTD TRUE) + endif() +endif() + # Used for default, common dirs that the end-user can change (if needed) # like CMAKE_INSTALL_INCLUDEDIR or CMAKE_INSTALL_DATADIR include(GNUInstallDirs) @@ -227,6 +242,7 @@ target_link_libraries(${PROJECT_NAME} ${_INTERFACE_OR_PUBLIC} $<$:Brotli::encoder> $<$:Brotli::decoder> $<$:ZLIB::ZLIB> + $<$:ZSTD::ZSTD> $<$:OpenSSL::SSL> $<$:OpenSSL::Crypto> ) @@ -236,6 +252,7 @@ target_compile_definitions(${PROJECT_NAME} ${_INTERFACE_OR_PUBLIC} $<$:CPPHTTPLIB_NO_EXCEPTIONS> $<$:CPPHTTPLIB_BROTLI_SUPPORT> $<$:CPPHTTPLIB_ZLIB_SUPPORT> + $<$:CPPHTTPLIB_ZSTD_SUPPORT> $<$:CPPHTTPLIB_OPENSSL_SUPPORT> $<$,$,$>:CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN> ) diff --git a/httplib.h b/httplib.h index 3f2bdaccfd..90478b9d44 100644 --- a/httplib.h +++ b/httplib.h @@ -312,6 +312,10 @@ using socket_t = int; #include #endif +#ifdef CPPHTTPLIB_ZSTD_SUPPORT +#include +#endif + /* * Declaration */ @@ -2445,7 +2449,7 @@ ssize_t send_socket(socket_t sock, const void *ptr, size_t size, int flags); ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags); -enum class EncodingType { None = 0, Gzip, Brotli }; +enum class EncodingType { None = 0, Gzip, Brotli, Zstd}; EncodingType encoding_type(const Request &req, const Response &res); @@ -2558,6 +2562,34 @@ class brotli_decompressor final : public decompressor { }; #endif +#ifdef CPPHTTPLIB_ZSTD_SUPPORT +class zstd_compressor : public compressor { +public: + zstd_compressor(); + ~zstd_compressor(); + + bool compress(const char *data, size_t data_length, bool last, + Callback callback) override; + +private: + ZSTD_CCtx *ctx_ = nullptr; +}; + +class zstd_decompressor : public decompressor { +public: + zstd_decompressor(); + ~zstd_decompressor(); + + bool is_valid() const override; + + bool decompress(const char *data, size_t data_length, + Callback callback) override; + +private: + ZSTD_DCtx *ctx_ = nullptr; +}; +#endif + // NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer` // to store data. The call can set memory on stack for performance. class stream_line_reader { @@ -3949,6 +3981,12 @@ inline EncodingType encoding_type(const Request &req, const Response &res) { if (ret) { return EncodingType::Gzip; } #endif +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + // TODO: 'Accept-Encoding' has zstd, not zstd;q=0 + ret = s.find("zstd") != std::string::npos; + if (ret) { return EncodingType::Zstd; } +#endif + return EncodingType::None; } @@ -4157,6 +4195,75 @@ inline bool brotli_decompressor::decompress(const char *data, } #endif +#ifdef CPPHTTPLIB_ZSTD_SUPPORT +inline zstd_compressor::zstd_compressor() { + ctx_ = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(ctx_, ZSTD_c_compressionLevel, ZSTD_fast); +} + +inline zstd_compressor::~zstd_compressor() { + ZSTD_freeCCtx(ctx_); +} + +inline bool zstd_compressor::compress(const char *data, size_t data_length, + bool last, Callback callback) { + std::array buff{}; + + ZSTD_EndDirective mode = last ? ZSTD_e_end : ZSTD_e_continue; + ZSTD_inBuffer input = { data, data_length, 0 }; + + bool finished; + do { + ZSTD_outBuffer output = { buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0 }; + size_t const remaining = ZSTD_compressStream2(ctx_, &output, &input, mode); + + if (ZSTD_isError(remaining)) { + return false; + } + + if (!callback(buff.data(), output.pos)) { + return false; + } + + finished = last ? (remaining == 0) : (input.pos == input.size); + + } while(!finished); + + return true; +} + +inline zstd_decompressor::zstd_decompressor() { + ctx_ = ZSTD_createDCtx(); +} + +inline zstd_decompressor::~zstd_decompressor() { + ZSTD_freeDCtx(ctx_); +} + +inline bool zstd_decompressor::is_valid() const { return ctx_ != nullptr; } + +inline bool zstd_decompressor::decompress(const char *data, size_t data_length, + Callback callback) { + std::array buff{}; + ZSTD_inBuffer input = { data, data_length, 0 }; + + while (input.pos < input.size) { + ZSTD_outBuffer output = { buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0 }; + size_t const remaining = ZSTD_decompressStream(ctx_, &output , &input); + + if (ZSTD_isError(remaining)) { + return false; + } + + if (!callback(buff.data(), output.pos)) { + return false; + } + } + + return true; +} +#endif + inline bool has_header(const Headers &headers, const std::string &key) { return headers.find(key) != headers.end(); } @@ -4397,6 +4504,13 @@ bool prepare_content_receiver(T &x, int &status, #else status = StatusCode::UnsupportedMediaType_415; return false; +#endif + } else if (encoding == "zstd") { +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + decompressor = detail::make_unique(); +#else + status = StatusCode::UnsupportedMediaType_415; + return false; #endif } @@ -6634,6 +6748,10 @@ Server::write_content_with_provider(Stream &strm, const Request &req, } else if (type == detail::EncodingType::Brotli) { #ifdef CPPHTTPLIB_BROTLI_SUPPORT compressor = detail::make_unique(); +#endif + } else if (type == detail::EncodingType::Zstd) { +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + compressor = detail::make_unique(); #endif } else { compressor = detail::make_unique(); @@ -7049,6 +7167,8 @@ inline void Server::apply_ranges(const Request &req, Response &res, res.set_header("Content-Encoding", "gzip"); } else if (type == detail::EncodingType::Brotli) { res.set_header("Content-Encoding", "br"); + } else if (type == detail::EncodingType::Zstd) { + res.set_header("Content-Encoding", "zstd"); } } } @@ -7088,6 +7208,11 @@ inline void Server::apply_ranges(const Request &req, Response &res, #ifdef CPPHTTPLIB_BROTLI_SUPPORT compressor = detail::make_unique(); content_encoding = "br"; +#endif + } else if (type == detail::EncodingType::Zstd) { +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + compressor = detail::make_unique(); + content_encoding = "zstd"; #endif } @@ -7812,6 +7937,10 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req, #ifdef CPPHTTPLIB_ZLIB_SUPPORT if (!accept_encoding.empty()) { accept_encoding += ", "; } accept_encoding += "gzip, deflate"; +#endif +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + if (!accept_encoding.empty()) { accept_encoding += ", "; } + accept_encoding += "zstd"; #endif req.set_header("Accept-Encoding", accept_encoding); } diff --git a/test/test.cc b/test/test.cc index 81a5e33a6c..f0455d1992 100644 --- a/test/test.cc +++ b/test/test.cc @@ -668,7 +668,7 @@ TEST(ParseAcceptEncoding1, AcceptEncoding) { TEST(ParseAcceptEncoding2, AcceptEncoding) { Request req; - req.set_header("Accept-Encoding", "gzip, deflate, br"); + req.set_header("Accept-Encoding", "gzip, deflate, br, zstd"); Response res; res.set_header("Content-Type", "text/plain"); @@ -679,6 +679,8 @@ TEST(ParseAcceptEncoding2, AcceptEncoding) { EXPECT_TRUE(ret == detail::EncodingType::Brotli); #elif CPPHTTPLIB_ZLIB_SUPPORT EXPECT_TRUE(ret == detail::EncodingType::Gzip); +#elif CPPHTTPLIB_ZSTD_SUPPORT + EXPECT_TRUE(ret == detail::EncodingType::Zstd); #else EXPECT_TRUE(ret == detail::EncodingType::None); #endif @@ -686,7 +688,7 @@ TEST(ParseAcceptEncoding2, AcceptEncoding) { TEST(ParseAcceptEncoding3, AcceptEncoding) { Request req; - req.set_header("Accept-Encoding", "br;q=1.0, gzip;q=0.8, *;q=0.1"); + req.set_header("Accept-Encoding", "br;q=1.0, gzip;q=0.8, zstd;q=0.8, *;q=0.1"); Response res; res.set_header("Content-Type", "text/plain"); @@ -697,6 +699,8 @@ TEST(ParseAcceptEncoding3, AcceptEncoding) { EXPECT_TRUE(ret == detail::EncodingType::Brotli); #elif CPPHTTPLIB_ZLIB_SUPPORT EXPECT_TRUE(ret == detail::EncodingType::Gzip); +#elif CPPHTTPLIB_ZSTD_SUPPORT + EXPECT_TRUE(ret == detail::EncodingType::Zstd); #else EXPECT_TRUE(ret == detail::EncodingType::None); #endif @@ -3007,7 +3011,7 @@ class ServerTest : public ::testing::Test { const httplib::ContentReader &) { res.set_content("ok", "text/plain"); }) -#if defined(CPPHTTPLIB_ZLIB_SUPPORT) || defined(CPPHTTPLIB_BROTLI_SUPPORT) +#if defined(CPPHTTPLIB_ZLIB_SUPPORT) || defined(CPPHTTPLIB_BROTLI_SUPPORT) || defined(CPPHTTPLIB_ZSTD_SUPPORT) .Get("/compress", [&](const Request & /*req*/, Response &res) { res.set_content( @@ -4928,6 +4932,249 @@ TEST_F(ServerTest, Brotli) { } #endif +#ifdef CPPHTTPLIB_ZSTD_SUPPORT +TEST_F(ServerTest, Zstd) { + Headers headers; + headers.emplace("Accept-Encoding", "zstd"); + auto res = cli_.Get("/compress", headers); + + ASSERT_TRUE(res); + EXPECT_EQ("zstd", res->get_header_value("Content-Encoding")); + EXPECT_EQ("text/plain", res->get_header_value("Content-Type")); + EXPECT_EQ("26", res->get_header_value("Content-Length")); + EXPECT_EQ("123456789012345678901234567890123456789012345678901234567890123456" + "7890123456789012345678901234567890", + res->body); + EXPECT_EQ(StatusCode::OK_200, res->status); +} + +TEST_F(ServerTest, ZstdWithoutAcceptEncoding) { + Headers headers; + headers.emplace("Accept-Encoding", ""); + auto res = cli_.Get("/compress", headers); + + ASSERT_TRUE(res); + EXPECT_TRUE(res->get_header_value("Content-Encoding").empty()); + EXPECT_EQ("text/plain", res->get_header_value("Content-Type")); + EXPECT_EQ("100", res->get_header_value("Content-Length")); + EXPECT_EQ("123456789012345678901234567890123456789012345678901234567890123456" + "7890123456789012345678901234567890", + res->body); + EXPECT_EQ(StatusCode::OK_200, res->status); +} + +TEST_F(ServerTest, ZstdWithContentReceiver) { + Headers headers; + headers.emplace("Accept-Encoding", "zstd"); + std::string body; + auto res = cli_.Get("/compress", headers, + [&](const char *data, uint64_t data_length) { + EXPECT_EQ(100U, data_length); + body.append(data, data_length); + return true; + }); + + ASSERT_TRUE(res); + EXPECT_EQ("zstd", res->get_header_value("Content-Encoding")); + EXPECT_EQ("text/plain", res->get_header_value("Content-Type")); + EXPECT_EQ("26", res->get_header_value("Content-Length")); + EXPECT_EQ("123456789012345678901234567890123456789012345678901234567890123456" + "7890123456789012345678901234567890", + body); + EXPECT_EQ(StatusCode::OK_200, res->status); +} + +TEST_F(ServerTest, ZstdWithoutDecompressing) { + Headers headers; + headers.emplace("Accept-Encoding", "zstd"); + + cli_.set_decompress(false); + auto res = cli_.Get("/compress", headers); + + unsigned char compressed[26] = { + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x64, 0x8d, 0x00, + 0x00, 0x50, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x30, 0x01, 0x00, 0xd7, 0xa9, + 0x20, 0x01 + }; + + ASSERT_TRUE(res); + EXPECT_EQ("zstd", res->get_header_value("Content-Encoding")); + EXPECT_EQ("text/plain", res->get_header_value("Content-Type")); + EXPECT_EQ("26", res->get_header_value("Content-Length")); + EXPECT_EQ(StatusCode::OK_200, res->status); + ASSERT_EQ(26U, res->body.size()); + EXPECT_TRUE(std::memcmp(compressed, res->body.data(), sizeof(compressed)) == 0); +} + +TEST_F(ServerTest, ZstdWithContentReceiverWithoutAcceptEncoding) { + Headers headers; + headers.emplace("Accept-Encoding", ""); + + std::string body; + auto res = cli_.Get("/compress", headers, + [&](const char *data, uint64_t data_length) { + EXPECT_EQ(100U, data_length); + body.append(data, data_length); + return true; + }); + + ASSERT_TRUE(res); + EXPECT_TRUE(res->get_header_value("Content-Encoding").empty()); + EXPECT_EQ("text/plain", res->get_header_value("Content-Type")); + EXPECT_EQ("100", res->get_header_value("Content-Length")); + EXPECT_EQ("123456789012345678901234567890123456789012345678901234567890123456" + "7890123456789012345678901234567890", + body); + EXPECT_EQ(StatusCode::OK_200, res->status); +} + +TEST_F(ServerTest, NoZstd) { + Headers headers; + headers.emplace("Accept-Encoding", "zstd"); + auto res = cli_.Get("/nocompress", headers); + + ASSERT_TRUE(res); + EXPECT_EQ(false, res->has_header("Content-Encoding")); + EXPECT_EQ("application/octet-stream", res->get_header_value("Content-Type")); + EXPECT_EQ("100", res->get_header_value("Content-Length")); + EXPECT_EQ("123456789012345678901234567890123456789012345678901234567890123456" + "7890123456789012345678901234567890", + res->body); + EXPECT_EQ(StatusCode::OK_200, res->status); +} + +TEST_F(ServerTest, NoZstdWithContentReceiver) { + Headers headers; + headers.emplace("Accept-Encoding", "zstd"); + std::string body; + auto res = cli_.Get("/nocompress", headers, + [&](const char *data, uint64_t data_length) { + EXPECT_EQ(100U, data_length); + body.append(data, data_length); + return true; + }); + + ASSERT_TRUE(res); + EXPECT_EQ(false, res->has_header("Content-Encoding")); + EXPECT_EQ("application/octet-stream", res->get_header_value("Content-Type")); + EXPECT_EQ("100", res->get_header_value("Content-Length")); + EXPECT_EQ("123456789012345678901234567890123456789012345678901234567890123456" + "7890123456789012345678901234567890", + body); + EXPECT_EQ(StatusCode::OK_200, res->status); +} + +// TODO: How to enable zstd ?? +TEST_F(ServerTest, MultipartFormDataZstd) { + MultipartFormDataItems items = { + {"key1", "test", "", ""}, + {"key2", "--abcdefg123", "", ""}, + }; + Headers headers; + headers.emplace("Accept-Encoding", "zstd"); + + + cli_.set_compress(true); + auto res = cli_.Post("/compress-multipart", headers, items); + + ASSERT_TRUE(res); + EXPECT_EQ(StatusCode::OK_200, res->status); +} + +TEST_F(ServerTest, PutWithContentProviderWithZstd) { + Headers headers; + headers.emplace("Accept-Encoding", "zstd"); + + cli_.set_compress(true); + auto res = cli_.Put( + "/put", headers, 3, + [](size_t /*offset*/, size_t /*length*/, DataSink &sink) { + sink.os << "PUT"; + return true; + }, + "text/plain"); + + ASSERT_TRUE(res); + EXPECT_EQ(StatusCode::OK_200, res->status); + EXPECT_EQ("PUT", res->body); +} + +TEST(ZstdDecompressor, ChunkedDecompression) { + std::string data; + for (size_t i = 0; i < 32 * 1024; ++i) { + data.push_back(static_cast('a' + i % 26)); + } + + std::string compressed_data; + { + httplib::detail::zstd_compressor compressor; + bool result = compressor.compress( + data.data(), data.size(), + /*last=*/true, + [&](const char *compressed_data_chunk, size_t compressed_data_size) { + compressed_data.insert(compressed_data.size(), compressed_data_chunk, + compressed_data_size); + return true; + }); + ASSERT_TRUE(result); + } + + std::string decompressed_data; + { + httplib::detail::zstd_decompressor decompressor; + + // Chunk size is chosen specifically to have a decompressed chunk size equal + // to 16384 bytes 16384 bytes is the size of decompressor output buffer + size_t chunk_size = 130; + for (size_t chunk_begin = 0; chunk_begin < compressed_data.size(); + chunk_begin += chunk_size) { + size_t current_chunk_size = + std::min(compressed_data.size() - chunk_begin, chunk_size); + bool result = decompressor.decompress( + compressed_data.data() + chunk_begin, current_chunk_size, + [&](const char *decompressed_data_chunk, + size_t decompressed_data_chunk_size) { + decompressed_data.insert(decompressed_data.size(), + decompressed_data_chunk, + decompressed_data_chunk_size); + return true; + }); + ASSERT_TRUE(result); + } + } + ASSERT_EQ(data, decompressed_data); +} + +TEST(ZstdDecompressor, Decompress) { + std::string original_text = "Compressed with ZSTD"; + unsigned char data[29] = { + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x14, 0xa1, 0x00, + 0x00, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, + 0x73, 0x65, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, + 0x20, 0x5a, 0x53, 0x54, 0x44 + }; + std::string compressed_data(data, data + sizeof(data) / sizeof(data[0])); + + std::string decompressed_data; + { + httplib::detail::zstd_decompressor decompressor; + + bool result = decompressor.decompress( + compressed_data.data(), compressed_data.size(), + [&](const char *decompressed_data_chunk, + size_t decompressed_data_chunk_size) { + decompressed_data.insert(decompressed_data.size(), + decompressed_data_chunk, + decompressed_data_chunk_size); + return true; + }); + ASSERT_TRUE(result); + } + ASSERT_EQ(original_text, decompressed_data); +} +#endif + // Sends a raw request to a server listening at HOST:PORT. static bool send_request(time_t read_timeout_sec, const std::string &req, std::string *resp = nullptr) { From fc54a3eb0ff228a5168fcb22b1e28acd3e21cf06 Mon Sep 17 00:00:00 2001 From: David Alonso de la Torre Date: Sat, 15 Mar 2025 09:59:06 +0100 Subject: [PATCH 2/9] Add zstd to CI tests --- .github/workflows/test.yaml | 4 +++- test/Makefile | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7763d379f5..d032cda79e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -67,6 +67,7 @@ jobs: sudo apt-get install -y libc6-dev${{ matrix.config.arch_suffix }} libstdc++-13-dev${{ matrix.config.arch_suffix }} \ libssl-dev${{ matrix.config.arch_suffix }} libcurl4-openssl-dev${{ matrix.config.arch_suffix }} \ zlib1g-dev${{ matrix.config.arch_suffix }} libbrotli-dev${{ matrix.config.arch_suffix }} + libzstd-dev${{ matrix.config.arch_suffix }} - name: build and run tests run: cd test && make EXTRA_CXXFLAGS="${{ matrix.config.arch_flags }}" - name: run fuzz test target @@ -126,7 +127,7 @@ jobs: - name: Setup msbuild on windows uses: microsoft/setup-msbuild@v2 - name: Install vcpkg dependencies - run: vcpkg install gtest curl zlib brotli + run: vcpkg install gtest curl zlib brotli zstd - name: Install OpenSSL if: ${{ matrix.config.with_ssl }} run: choco install openssl @@ -139,6 +140,7 @@ jobs: -DHTTPLIB_COMPILE=${{ matrix.config.compiled && 'ON' || 'OFF' }} -DHTTPLIB_REQUIRE_ZLIB=ON -DHTTPLIB_REQUIRE_BROTLI=ON + -DHTTPLIB_REQUIRE_ZSTD=ON -DHTTPLIB_REQUIRE_OPENSSL=${{ matrix.config.with_ssl && 'ON' || 'OFF' }} - name: Build ${{ matrix.config.name }} run: cmake --build build --config Release -- /v:m /clp:ShowCommandLine diff --git a/test/Makefile b/test/Makefile index 48cd3abb2e..db192f4067 100644 --- a/test/Makefile +++ b/test/Makefile @@ -18,7 +18,9 @@ ZLIB_SUPPORT = -DCPPHTTPLIB_ZLIB_SUPPORT -lz BROTLI_DIR = $(PREFIX)/opt/brotli BROTLI_SUPPORT = -DCPPHTTPLIB_BROTLI_SUPPORT -I$(BROTLI_DIR)/include -L$(BROTLI_DIR)/lib -lbrotlicommon -lbrotlienc -lbrotlidec -TEST_ARGS = gtest/src/gtest-all.cc gtest/src/gtest_main.cc -Igtest -Igtest/include $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT) -pthread -lcurl +ZSTD_SUPPORT = -DCPPHTTPLIB_ZSTD_SUPPORT -lzstd + +TEST_ARGS = gtest/src/gtest-all.cc gtest/src/gtest_main.cc -Igtest -Igtest/include $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT) $(ZSTD_SUPPORT) -pthread -lcurl # By default, use standalone_fuzz_target_runner. # This runner does no fuzzing, but simply executes the inputs From 17cf10c21aa23a5615d0c357dc5874177f8e49e8 Mon Sep 17 00:00:00 2001 From: David Alonso de la Torre Date: Sat, 15 Mar 2025 10:07:33 +0100 Subject: [PATCH 3/9] Use use zstd cmake target instead of ZSTD. Use cmake variable for found packages --- CMakeLists.txt | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a64a2b28b0..2ff07ec397 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -158,14 +158,11 @@ elseif(HTTPLIB_USE_BROTLI_IF_AVAILABLE) endif() if(HTTPLIB_REQUIRE_ZSTD) - find_package(ZSTD REQUIRED) + find_package(zstd REQUIRED) set(HTTPLIB_IS_USING_ZSTD TRUE) elseif(HTTPLIB_USE_ZSTD_IF_AVAILABLE) - find_package(ZSTD QUIET) - # FindZLIB doesn't have a ZLIB_FOUND variable, so check the target. - if(TARGET ZSTD::ZSTD) - set(HTTPLIB_IS_USING_ZSTD TRUE) - endif() + find_package(zstd QUIET) + set(HTTPLIB_IS_USING_ZSTD ${zstd_FOUND}) endif() # Used for default, common dirs that the end-user can change (if needed) @@ -242,7 +239,7 @@ target_link_libraries(${PROJECT_NAME} ${_INTERFACE_OR_PUBLIC} $<$:Brotli::encoder> $<$:Brotli::decoder> $<$:ZLIB::ZLIB> - $<$:ZSTD::ZSTD> + $<$:zstd> $<$:OpenSSL::SSL> $<$:OpenSSL::Crypto> ) From 9eab78cf16c0cb7929b9b9684f512ab9b86aebbb Mon Sep 17 00:00:00 2001 From: David Alonso de la Torre Date: Sat, 15 Mar 2025 10:08:29 +0100 Subject: [PATCH 4/9] Add missing comment for HTTPLIB_REQUIRE_ZSTD --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ff07ec397..c3e6256ebb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ * HTTPLIB_REQUIRE_OPENSSL (default off) * HTTPLIB_REQUIRE_ZLIB (default off) * HTTPLIB_REQUIRE_BROTLI (default off) + * HTTPLIB_REQUIRE_ZSTD (default off) * HTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN (default on) * HTTPLIB_COMPILE (default off) * HTTPLIB_INSTALL (default on) From 25a6e31e121a3257c939485e8a454fcf4041e640 Mon Sep 17 00:00:00 2001 From: David Alonso de la Torre Date: Sat, 15 Mar 2025 10:15:42 +0100 Subject: [PATCH 5/9] Fix test.yaml rebase error --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d032cda79e..296ad29b38 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -66,7 +66,7 @@ jobs: sudo apt-get update sudo apt-get install -y libc6-dev${{ matrix.config.arch_suffix }} libstdc++-13-dev${{ matrix.config.arch_suffix }} \ libssl-dev${{ matrix.config.arch_suffix }} libcurl4-openssl-dev${{ matrix.config.arch_suffix }} \ - zlib1g-dev${{ matrix.config.arch_suffix }} libbrotli-dev${{ matrix.config.arch_suffix }} + zlib1g-dev${{ matrix.config.arch_suffix }} libbrotli-dev${{ matrix.config.arch_suffix }} \ libzstd-dev${{ matrix.config.arch_suffix }} - name: build and run tests run: cd test && make EXTRA_CXXFLAGS="${{ matrix.config.arch_flags }}" From 9e81d40ede8a9483df4ff932283829db22798c90 Mon Sep 17 00:00:00 2001 From: David Alonso de la Torre Date: Sat, 15 Mar 2025 10:21:58 +0100 Subject: [PATCH 6/9] Use zstd::libzstd target --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3e6256ebb..0353b0ca38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -240,7 +240,7 @@ target_link_libraries(${PROJECT_NAME} ${_INTERFACE_OR_PUBLIC} $<$:Brotli::encoder> $<$:Brotli::decoder> $<$:ZLIB::ZLIB> - $<$:zstd> + $<$:zstd::libzstd> $<$:OpenSSL::SSL> $<$:OpenSSL::Crypto> ) From a3f87f439c213008e6f57ab855ddce21d3b005d7 Mon Sep 17 00:00:00 2001 From: David Alonso de la Torre Date: Sat, 15 Mar 2025 10:41:43 +0100 Subject: [PATCH 7/9] Add include and library paths to ZSTD args --- test/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index db192f4067..3107702beb 100644 --- a/test/Makefile +++ b/test/Makefile @@ -18,7 +18,8 @@ ZLIB_SUPPORT = -DCPPHTTPLIB_ZLIB_SUPPORT -lz BROTLI_DIR = $(PREFIX)/opt/brotli BROTLI_SUPPORT = -DCPPHTTPLIB_BROTLI_SUPPORT -I$(BROTLI_DIR)/include -L$(BROTLI_DIR)/lib -lbrotlicommon -lbrotlienc -lbrotlidec -ZSTD_SUPPORT = -DCPPHTTPLIB_ZSTD_SUPPORT -lzstd +ZSTD_DIR = $(PREFIX)/opt/zstd +ZSTD_SUPPORT = -DCPPHTTPLIB_ZSTD_SUPPORT -I$(ZSTD_DIR)/include -L$(ZSTD_DIR)/lib -lzstd TEST_ARGS = gtest/src/gtest-all.cc gtest/src/gtest_main.cc -Igtest -Igtest/include $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT) $(ZSTD_SUPPORT) -pthread -lcurl From c30268dca609cac346895865b64b4772a07f7b71 Mon Sep 17 00:00:00 2001 From: David Alonso de la Torre Date: Sat, 15 Mar 2025 10:48:11 +0100 Subject: [PATCH 8/9] Run clang-format --- httplib.h | 44 +++++++++++++++----------------------------- test/test.cc | 32 +++++++++++++++----------------- 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/httplib.h b/httplib.h index 90478b9d44..60686e3032 100644 --- a/httplib.h +++ b/httplib.h @@ -2449,7 +2449,7 @@ ssize_t send_socket(socket_t sock, const void *ptr, size_t size, int flags); ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags); -enum class EncodingType { None = 0, Gzip, Brotli, Zstd}; +enum class EncodingType { None = 0, Gzip, Brotli, Zstd }; EncodingType encoding_type(const Request &req, const Response &res); @@ -4201,63 +4201,49 @@ inline zstd_compressor::zstd_compressor() { ZSTD_CCtx_setParameter(ctx_, ZSTD_c_compressionLevel, ZSTD_fast); } -inline zstd_compressor::~zstd_compressor() { - ZSTD_freeCCtx(ctx_); -} +inline zstd_compressor::~zstd_compressor() { ZSTD_freeCCtx(ctx_); } inline bool zstd_compressor::compress(const char *data, size_t data_length, bool last, Callback callback) { std::array buff{}; ZSTD_EndDirective mode = last ? ZSTD_e_end : ZSTD_e_continue; - ZSTD_inBuffer input = { data, data_length, 0 }; + ZSTD_inBuffer input = {data, data_length, 0}; bool finished; do { - ZSTD_outBuffer output = { buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0 }; + ZSTD_outBuffer output = {buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0}; size_t const remaining = ZSTD_compressStream2(ctx_, &output, &input, mode); - if (ZSTD_isError(remaining)) { - return false; - } + if (ZSTD_isError(remaining)) { return false; } - if (!callback(buff.data(), output.pos)) { - return false; - } + if (!callback(buff.data(), output.pos)) { return false; } finished = last ? (remaining == 0) : (input.pos == input.size); - } while(!finished); + } while (!finished); return true; } -inline zstd_decompressor::zstd_decompressor() { - ctx_ = ZSTD_createDCtx(); -} +inline zstd_decompressor::zstd_decompressor() { ctx_ = ZSTD_createDCtx(); } -inline zstd_decompressor::~zstd_decompressor() { - ZSTD_freeDCtx(ctx_); -} +inline zstd_decompressor::~zstd_decompressor() { ZSTD_freeDCtx(ctx_); } inline bool zstd_decompressor::is_valid() const { return ctx_ != nullptr; } inline bool zstd_decompressor::decompress(const char *data, size_t data_length, Callback callback) { std::array buff{}; - ZSTD_inBuffer input = { data, data_length, 0 }; + ZSTD_inBuffer input = {data, data_length, 0}; while (input.pos < input.size) { - ZSTD_outBuffer output = { buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0 }; - size_t const remaining = ZSTD_decompressStream(ctx_, &output , &input); + ZSTD_outBuffer output = {buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0}; + size_t const remaining = ZSTD_decompressStream(ctx_, &output, &input); - if (ZSTD_isError(remaining)) { - return false; - } + if (ZSTD_isError(remaining)) { return false; } - if (!callback(buff.data(), output.pos)) { - return false; - } + if (!callback(buff.data(), output.pos)) { return false; } } return true; @@ -10506,4 +10492,4 @@ inline SSL_CTX *Client::ssl_context() const { } // namespace httplib -#endif // CPPHTTPLIB_HTTPLIB_H +#endif // CPPHTTPLIB_HTTPLIB_H \ No newline at end of file diff --git a/test/test.cc b/test/test.cc index f0455d1992..725af16551 100644 --- a/test/test.cc +++ b/test/test.cc @@ -688,7 +688,8 @@ TEST(ParseAcceptEncoding2, AcceptEncoding) { TEST(ParseAcceptEncoding3, AcceptEncoding) { Request req; - req.set_header("Accept-Encoding", "br;q=1.0, gzip;q=0.8, zstd;q=0.8, *;q=0.1"); + req.set_header("Accept-Encoding", + "br;q=1.0, gzip;q=0.8, zstd;q=0.8, *;q=0.1"); Response res; res.set_header("Content-Type", "text/plain"); @@ -3011,7 +3012,8 @@ class ServerTest : public ::testing::Test { const httplib::ContentReader &) { res.set_content("ok", "text/plain"); }) -#if defined(CPPHTTPLIB_ZLIB_SUPPORT) || defined(CPPHTTPLIB_BROTLI_SUPPORT) || defined(CPPHTTPLIB_ZSTD_SUPPORT) +#if defined(CPPHTTPLIB_ZLIB_SUPPORT) || defined(CPPHTTPLIB_BROTLI_SUPPORT) || \ + defined(CPPHTTPLIB_ZSTD_SUPPORT) .Get("/compress", [&](const Request & /*req*/, Response &res) { res.set_content( @@ -4991,12 +4993,10 @@ TEST_F(ServerTest, ZstdWithoutDecompressing) { cli_.set_decompress(false); auto res = cli_.Get("/compress", headers); - unsigned char compressed[26] = { - 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x64, 0x8d, 0x00, - 0x00, 0x50, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, - 0x37, 0x38, 0x39, 0x30, 0x01, 0x00, 0xd7, 0xa9, - 0x20, 0x01 - }; + unsigned char compressed[26] = {0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x64, 0x8d, + 0x00, 0x00, 0x50, 0x31, 0x32, 0x33, 0x34, + 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x01, + 0x00, 0xd7, 0xa9, 0x20, 0x01}; ASSERT_TRUE(res); EXPECT_EQ("zstd", res->get_header_value("Content-Encoding")); @@ -5004,7 +5004,8 @@ TEST_F(ServerTest, ZstdWithoutDecompressing) { EXPECT_EQ("26", res->get_header_value("Content-Length")); EXPECT_EQ(StatusCode::OK_200, res->status); ASSERT_EQ(26U, res->body.size()); - EXPECT_TRUE(std::memcmp(compressed, res->body.data(), sizeof(compressed)) == 0); + EXPECT_TRUE(std::memcmp(compressed, res->body.data(), sizeof(compressed)) == + 0); } TEST_F(ServerTest, ZstdWithContentReceiverWithoutAcceptEncoding) { @@ -5074,7 +5075,6 @@ TEST_F(ServerTest, MultipartFormDataZstd) { Headers headers; headers.emplace("Accept-Encoding", "zstd"); - cli_.set_compress(true); auto res = cli_.Post("/compress-multipart", headers, items); @@ -5085,7 +5085,7 @@ TEST_F(ServerTest, MultipartFormDataZstd) { TEST_F(ServerTest, PutWithContentProviderWithZstd) { Headers headers; headers.emplace("Accept-Encoding", "zstd"); - + cli_.set_compress(true); auto res = cli_.Put( "/put", headers, 3, @@ -5148,12 +5148,10 @@ TEST(ZstdDecompressor, ChunkedDecompression) { TEST(ZstdDecompressor, Decompress) { std::string original_text = "Compressed with ZSTD"; - unsigned char data[29] = { - 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x14, 0xa1, 0x00, - 0x00, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, - 0x73, 0x65, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, - 0x20, 0x5a, 0x53, 0x54, 0x44 - }; + unsigned char data[29] = {0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x14, 0xa1, 0x00, + 0x00, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, + 0x73, 0x65, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, + 0x20, 0x5a, 0x53, 0x54, 0x44}; std::string compressed_data(data, data + sizeof(data) / sizeof(data[0])); std::string decompressed_data; From be33a909ff30235ff3a0340468b9daea49ca122d Mon Sep 17 00:00:00 2001 From: David Alonso de la Torre Date: Sat, 15 Mar 2025 13:18:32 +0100 Subject: [PATCH 9/9] Add zstd to httplibConfig.cmake.in --- cmake/httplibConfig.cmake.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/httplibConfig.cmake.in b/cmake/httplibConfig.cmake.in index 93dff323de..918bea3d7e 100644 --- a/cmake/httplibConfig.cmake.in +++ b/cmake/httplibConfig.cmake.in @@ -35,6 +35,10 @@ if(@HTTPLIB_IS_USING_BROTLI@) find_dependency(Brotli COMPONENTS common encoder decoder) endif() +if(@HTTPLIB_IS_USING_ZSTD@) + find_dependency(zstd) +endif() + # Mildly useful for end-users # Not really recommended to be used though set_and_check(HTTPLIB_INCLUDE_DIR "@PACKAGE_CMAKE_INSTALL_FULL_INCLUDEDIR@") @@ -46,6 +50,7 @@ set_and_check(HTTPLIB_HEADER_PATH "@PACKAGE_CMAKE_INSTALL_FULL_INCLUDEDIR@/httpl set(httplib_OpenSSL_FOUND @HTTPLIB_IS_USING_OPENSSL@) set(httplib_ZLIB_FOUND @HTTPLIB_IS_USING_ZLIB@) set(httplib_Brotli_FOUND @HTTPLIB_IS_USING_BROTLI@) +set(httplib_zstd_FOUND @HTTPLIB_IS_USING_ZSTD@) check_required_components(httplib)