Use curl as optional client

.github/workflows/MainDistributionPipeline.yml

@@ -19,7 +19,6 @@ jobs:
       extension_name: httpfs
       duckdb_version: v1.3-ossivalis
       ci_tools_version: main
-      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads'
 
 
   duckdb-stable-deploy:
@@ -32,4 +31,3 @@ jobs:
       duckdb_version: v1.3-ossivalis
       ci_tools_version: main
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
-      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads'

.github/workflows/MinioTests.yml

@@ -10,15 +10,17 @@ defaults:
 jobs:
   minio-tests:
     name: Minio Tests
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     env:
       S3_TEST_SERVER_AVAILABLE: 1
       AWS_DEFAULT_REGION: eu-west-1
       AWS_ACCESS_KEY_ID: minio_duckdb_user
       AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
       DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
       DUCKDB_S3_USE_SSL: false
+      CORE_EXTENSIONS: 'parquet;json'
       GEN: ninja
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
       VCPKG_TARGET_TRIPLET: x64-linux
 
     steps:

CMakeLists.txt

@@ -9,16 +9,24 @@ add_extension_definitions()
 include_directories(extension/httpfs/include
                     ${DUCKDB_MODULE_BASE_DIR}/third_party/httplib)
 
+if (NOT EMSCRIPTEN)
+  set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_httplib_client.cpp extension/httpfs/httpfs_curl_client.cpp)
+  add_definitions(-DOVERRIDE_ENCRYPTION_UTILS=1)
+else()
+  set(EXTRA_SOURCES extension/httpfs/httpfs_client_wasm.cpp)
+endif()
+
 build_static_extension(
   httpfs
   extension/httpfs/hffs.cpp
   extension/httpfs/s3fs.cpp
   extension/httpfs/httpfs.cpp
-  extension/httpfs/httpfs_client.cpp
   extension/httpfs/http_state.cpp
   extension/httpfs/crypto.cpp
+  extension/httpfs/hash_functions.cpp
   extension/httpfs/create_secret_functions.cpp
-  extension/httpfs/httpfs_extension.cpp)
+  extension/httpfs/httpfs_extension.cpp
+  ${EXTRA_SOURCES} )
 
 set(PARAMETERS "-warnings")
 build_loadable_extension(
@@ -27,32 +35,41 @@ build_loadable_extension(
   extension/httpfs/hffs.cpp
   extension/httpfs/s3fs.cpp
   extension/httpfs/httpfs.cpp
-  extension/httpfs/httpfs_client.cpp
   extension/httpfs/http_state.cpp
   extension/httpfs/crypto.cpp
+  extension/httpfs/hash_functions.cpp
   extension/httpfs/create_secret_functions.cpp
-  extension/httpfs/httpfs_extension.cpp)
+  extension/httpfs/httpfs_extension.cpp
+  ${EXTRA_SOURCES} )
 
 if(MINGW)
   set(OPENSSL_USE_STATIC_LIBS TRUE)
 endif()
 
 find_package(OpenSSL REQUIRED)
+find_package(CURL REQUIRED)
 include_directories(${OPENSSL_INCLUDE_DIR})
+include_directories(${CURL_INCLUDE_DIRS})
 if(EMSCRIPTEN)
 
 else()
   target_link_libraries(httpfs_loadable_extension duckdb_mbedtls
                         ${OPENSSL_LIBRARIES})
   target_link_libraries(httpfs_extension duckdb_mbedtls ${OPENSSL_LIBRARIES})
 
+  # Link dependencies into extension
+  target_link_libraries(httpfs_loadable_extension ${CURL_LIBRARIES})
+  target_link_libraries(httpfs_extension ${CURL_LIBRARIES})
+
+
   if(MINGW)
     find_package(ZLIB)
     target_link_libraries(httpfs_loadable_extension ZLIB::ZLIB -lcrypt32)
     target_link_libraries(httpfs_extension ZLIB::ZLIB -lcrypt32)
   endif()
 endif()
 
+
 install(
   TARGETS httpfs_extension
   EXPORT "${DUCKDB_EXPORT_SET}"

extension/httpfs/crypto.cpp

@@ -1,35 +1,23 @@
 #include "crypto.hpp"
+#include "hash_functions.hpp"
 #include "mbedtls_wrapper.hpp"
 #include <iostream>
 #include "duckdb/common/common.hpp"
 #include <stdio.h>
 
 #define CPPHTTPLIB_OPENSSL_SUPPORT
-#include "httplib.hpp"
 
-namespace duckdb {
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/ssl.h>
+#include <openssl/x509v3.h>
+#include <openssl/rand.h>
 
-void sha256(const char *in, size_t in_len, hash_bytes &out) {
-	duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(in, in_len, (char *)out);
-}
+#if defined(_WIN32) && defined(OPENSSL_USE_APPLINK)
+#include <openssl/applink.c>
+#endif
 
-void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out) {
-	duckdb_mbedtls::MbedTlsWrapper::Hmac256(secret, secret_len, message.data(), message.size(), (char *)out);
-}
-
-void hmac256(std::string message, hash_bytes secret, hash_bytes &out) {
-	hmac256(message, (char *)secret, sizeof(hash_bytes), out);
-}
-
-void hex256(hash_bytes &in, hash_str &out) {
-	const char *hex = "0123456789abcdef";
-	unsigned char *pin = in;
-	unsigned char *pout = out;
-	for (; pin < in + sizeof(in); pout += 2, pin++) {
-		pout[0] = hex[(*pin >> 4) & 0xF];
-		pout[1] = hex[*pin & 0xF];
-	}
-}
+namespace duckdb {
 
 AESStateSSL::AESStateSSL(const std::string *key) : context(EVP_CIPHER_CTX_new()) {
 	if (!(context)) {

extension/httpfs/hash_functions.cpp

@@ -0,0 +1,28 @@
+#include "mbedtls_wrapper.hpp"
+#include "hash_functions.hpp"
+
+namespace duckdb {
+
+void sha256(const char *in, size_t in_len, hash_bytes &out) {
+	duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(in, in_len, (char *)out);
+}
+
+void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out) {
+	duckdb_mbedtls::MbedTlsWrapper::Hmac256(secret, secret_len, message.data(), message.size(), (char *)out);
+}
+
+void hmac256(std::string message, hash_bytes secret, hash_bytes &out) {
+	hmac256(message, (char *)secret, sizeof(hash_bytes), out);
+}
+
+void hex256(hash_bytes &in, hash_str &out) {
+	const char *hex = "0123456789abcdef";
+	unsigned char *pin = in;
+	unsigned char *pout = out;
+	for (; pin < in + sizeof(in); pout += 2, pin++) {
+		pout[0] = hex[(*pin >> 4) & 0xF];
+		pout[1] = hex[*pin & 0xF];
+	}
+}
+
+} // namespace duckdb

extension/httpfs/httpfs.cpp

@@ -727,4 +727,9 @@ void HTTPFileHandle::StoreClient(unique_ptr<HTTPClient> client) {
 HTTPFileHandle::~HTTPFileHandle() {
 	DUCKDB_LOG_FILE_SYSTEM_CLOSE((*this));
 };
+
+string HTTPFSUtil::GetName() const {
+	return "HTTPFS";
+}
+
 } // namespace duckdb

extension/httpfs/httpfs_client_wasm.cpp

@@ -0,0 +1,16 @@
+#include "httpfs_client.hpp"
+#include "http_state.hpp"
+
+namespace duckdb {
+
+unique_ptr<HTTPClient> HTTPFSUtil::InitializeClient(HTTPParams &http_params, const string &proto_host_port) {
+	throw InternalException("HTTPFSUtil::InitializeClient is not expected to be called");
+}
+
+unordered_map<string, string> HTTPFSUtil::ParseGetParameters(const string &text) {
+	unordered_map<string, string> result;
+	//TODO: HTTPFSUtil::ParseGetParameters is currently not implemented
+	return result;
+}
+
+} // namespace duckdb