Add support for webdav and Hetzner storage boxes

.github/workflows/IntegrationTests.yml

@@ -27,6 +27,11 @@ jobs:
       PYTHON_HTTP_SERVER_URL: http://localhost:8008
       PYTHON_HTTP_SERVER_DIR: /tmp/python_test_server
 
+      WEBDAV_TEST_SERVER_AVAILABLE: 1
+      WEBDAV_TEST_USERNAME: duckdb_webdav_user
+      WEBDAV_TEST_PASSWORD: duckdb_webdav_password
+      WEBDAV_TEST_BASE_URL: webdav://localhost:9100
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -83,6 +88,11 @@ jobs:
       - name: Start test server & run tests
         shell: bash
         run: |
+          # Minio S3 test server
           source ./scripts/run_s3_test_server.sh
           source ./scripts/set_s3_test_server_variables.sh
-          make test
+
+          # WebDav test server
+          ./scripts/run_webdav_test_server.sh
+
+          make test

scripts/run_webdav_test_server.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Note: DON'T run as root
+
+docker compose -f scripts/webdav.yml -p duckdb-webdav up -d
+
+# Get setup container name to monitor logs
+container_name=$(docker ps -a --format '{{.Names}}' | grep -m 1 "duckdb-webdav")
+echo $container_name
+
+# Wait for setup completion (up to 360 seconds like Minio)
+for i in $(seq 1 360);
+do
+  docker_finish_logs=$(docker logs $container_name 2>/dev/null | grep -m 1 'FINISHED SETTING UP WEBDAV' || echo '')
+  if [ ! -z "${docker_finish_logs}" ]; then
+    break
+  fi
+  sleep 1
+done
+
+export WEBDAV_TEST_SERVER_AVAILABLE=1
+export WEBDAV_TEST_BASE_URL="webdav://localhost:9100"

scripts/set_webdav_test_server_variables.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# Run this script with 'source' or the shorthand: '.':
+# i.e: source scripts/set_webdav_test_server_variables.sh
+
+# Enable the WebDAV tests to run
+export WEBDAV_TEST_SERVER_AVAILABLE=1
+
+export WEBDAV_TEST_USERNAME=duckdb_webdav_user
+export WEBDAV_TEST_PASSWORD=duckdb_webdav_password
+export WEBDAV_TEST_BASE_URL=webdav://localhost:9100

scripts/stop_webdav_test_server.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+echo "Stopping WebDAV test server..."
+docker compose -f scripts/webdav.yml -p duckdb-webdav down
+
+echo "WebDAV test server stopped."

scripts/webdav.yml

@@ -0,0 +1,74 @@
+services:
+  webdav:
+    image: bytemark/webdav
+    hostname: duckdb-webdav-test.local
+    ports:
+      - "9100:80"
+    environment:
+      - AUTH_TYPE=Basic
+      - USERNAME=duckdb_webdav_user
+      - PASSWORD=duckdb_webdav_password
+
+  webdav_setup:
+    image: alpine:latest
+    depends_on:
+      - webdav
+    links:
+      - webdav
+    entrypoint:
+      - /bin/sh
+      - -c
+      - |
+        apk add --no-cache curl;
+
+        until (
+          curl -u duckdb_webdav_user:duckdb_webdav_password -f http://webdav:80/ >/dev/null 2>&1
+        ) do
+          echo '...waiting for WebDAV server...' && sleep 1;
+        done;
+
+        echo 'WebDAV server is ready, creating test data...';
+
+        # Create directories using WebDAV MKCOL method
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X MKCOL http://webdav:80/test-dir/;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X MKCOL http://webdav:80/test-dir/upload-dir/;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X MKCOL http://webdav:80/test-dir/subdir1/;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X MKCOL http://webdav:80/test-dir/subdir2/;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X MKCOL http://webdav:80/glob-test/;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X MKCOL http://webdav:80/glob-test/year=2023/;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X MKCOL http://webdav:80/glob-test/year=2024/;
+
+        # Create temporary directory for test files
+        mkdir -p /tmp/webdav-test;
+
+        # Create test files
+        printf 'Hello from WebDAV' > /tmp/webdav-test/hello.txt;
+
+        echo 'id,name,value' > /tmp/webdav-test/test1.csv;
+        echo '1,Alice,100' >> /tmp/webdav-test/test1.csv;
+        echo '2,Bob,200' >> /tmp/webdav-test/test1.csv;
+
+        echo 'id,name,value' > /tmp/webdav-test/test2.csv;
+        echo '3,Charlie,300' >> /tmp/webdav-test/test2.csv;
+        echo '4,Diana,400' >> /tmp/webdav-test/test2.csv;
+
+        echo 'id,name,value' > /tmp/webdav-test/test3.csv;
+        echo '5,Eve,500' >> /tmp/webdav-test/test3.csv;
+        echo '6,Frank,600' >> /tmp/webdav-test/test3.csv;
+
+        echo 'id,year,data' > /tmp/webdav-test/data2023.csv;
+        echo '1,2023,test2023' >> /tmp/webdav-test/data2023.csv;
+
+        echo 'id,year,data' > /tmp/webdav-test/data2024.csv;
+        echo '2,2024,test2024' >> /tmp/webdav-test/data2024.csv;
+
+        # Upload test files using WebDAV PUT method
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X PUT http://webdav:80/hello.txt --data-binary @/tmp/webdav-test/hello.txt;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X PUT http://webdav:80/test-dir/test1.csv --data-binary @/tmp/webdav-test/test1.csv;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X PUT http://webdav:80/test-dir/subdir1/test2.csv --data-binary @/tmp/webdav-test/test2.csv;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X PUT http://webdav:80/test-dir/subdir2/test3.csv --data-binary @/tmp/webdav-test/test3.csv;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X PUT http://webdav:80/glob-test/year=2023/data.csv --data-binary @/tmp/webdav-test/data2023.csv;
+        curl -u duckdb_webdav_user:duckdb_webdav_password -X PUT http://webdav:80/glob-test/year=2024/data.csv --data-binary @/tmp/webdav-test/data2024.csv;
+
+        echo 'FINISHED SETTING UP WEBDAV';
+        exit 0;

src/CMakeLists.txt

@@ -2,6 +2,7 @@ set(HTTPFS_SOURCES
     hffs.cpp
     s3fs.cpp
     httpfs.cpp
+    webdavfs.cpp
     http_state.cpp
     crypto.cpp
     hash_functions.cpp

src/create_secret_functions.cpp

@@ -335,4 +335,55 @@ CreateBearerTokenFunctions::CreateHuggingFaceSecretFromCredentialChain(ClientCon
 	auto token = TryReadTokenFile("~/.cache/huggingface/token", "", false);
 	return CreateSecretFunctionInternal(context, input, token);
 }
+
+void CreateWebDAVSecretFunctions::Register(ExtensionLoader &loader) {
+	// WebDAV secret
+	SecretType secret_type_webdav;
+	secret_type_webdav.name = WEBDAV_TYPE;
+	secret_type_webdav.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
+	secret_type_webdav.default_provider = "config";
+	secret_type_webdav.extension = "httpfs";
+	loader.RegisterSecretType(secret_type_webdav);
+
+	// WebDAV config provider
+	CreateSecretFunction webdav_config_fun = {WEBDAV_TYPE, "config", CreateWebDAVSecretFromConfig};
+	webdav_config_fun.named_parameters["username"] = LogicalType::VARCHAR;
+	webdav_config_fun.named_parameters["password"] = LogicalType::VARCHAR;
+	loader.RegisterFunction(webdav_config_fun);
+}
+
+unique_ptr<BaseSecret> CreateWebDAVSecretFunctions::CreateSecretFunctionInternal(ClientContext &context,
+                                                                                 CreateSecretInput &input) {
+	// Set scope to user provided scope or the default
+	auto scope = input.scope;
+	if (scope.empty()) {
+		// Default scope includes webdav://, webdavs://, storagebox://, and Hetzner Storage Box URLs
+		scope.push_back("webdav://");
+		scope.push_back("webdavs://");
+		scope.push_back("storagebox://"); // Hetzner Storage Box shorthand
+		scope.push_back("https://");      // For Hetzner Storage Boxes and other HTTPS WebDAV servers
+	}
+	auto return_value = make_uniq<KeyValueSecret>(scope, input.type, input.provider, input.name);
+
+	//! Set key value map
+	for (const auto &named_param : input.options) {
+		auto lower_name = StringUtil::Lower(named_param.first);
+		if (lower_name == "username") {
+			return_value->secret_map["username"] = named_param.second.ToString();
+		} else if (lower_name == "password") {
+			return_value->secret_map["password"] = named_param.second.ToString();
+		}
+	}
+
+	//! Set redact keys
+	return_value->redact_keys = {"password"};
+
+	return std::move(return_value);
+}
+
+unique_ptr<BaseSecret> CreateWebDAVSecretFunctions::CreateWebDAVSecretFromConfig(ClientContext &context,
+                                                                                 CreateSecretInput &input) {
+	return CreateSecretFunctionInternal(context, input);
+}
+
 } // namespace duckdb

src/httpfs_curl_client.cpp

@@ -343,7 +343,15 @@ class HTTPFSCurlClient : public HTTPClient {
 		CURLcode res;
 		{
 			curl_easy_setopt(*curl, CURLOPT_URL, request_info->url.c_str());
-			curl_easy_setopt(*curl, CURLOPT_POST, 1L);
+
+			// Check if a custom HTTP method is specified in extra_headers
+			auto method_it = info.params.extra_headers.find("X-DuckDB-HTTP-Method");
+			if (method_it != info.params.extra_headers.end()) {
+				// Use custom HTTP method (e.g., PROPFIND for WebDAV)
+				curl_easy_setopt(*curl, CURLOPT_CUSTOMREQUEST, method_it->second.c_str());
+			} else {
+				curl_easy_setopt(*curl, CURLOPT_POST, 1L);
+			}
 
 			// Set POST body
 			curl_easy_setopt(*curl, CURLOPT_POSTFIELDS, const_char_ptr_cast(info.buffer_in));

src/httpfs_extension.cpp

@@ -5,6 +5,7 @@
 #include "duckdb.hpp"
 #include "s3fs.hpp"
 #include "hffs.hpp"
+#include "webdavfs.hpp"
 #ifdef OVERRIDE_ENCRYPTION_UTILS
 #include "crypto.hpp"
 #endif // OVERRIDE_ENCRYPTION_UTILS
@@ -41,6 +42,7 @@ static void LoadInternal(ExtensionLoader &loader) {
 	fs.RegisterSubSystem(make_uniq<HTTPFileSystem>());
 	fs.RegisterSubSystem(make_uniq<HuggingFaceFileSystem>());
 	fs.RegisterSubSystem(make_uniq<S3FileSystem>(BufferManager::GetBufferManager(instance)));
+	fs.RegisterSubSystem(make_uniq<WebDAVFileSystem>());
 
 	auto &config = DBConfig::GetConfig(instance);
 
@@ -137,6 +139,7 @@ static void LoadInternal(ExtensionLoader &loader) {
 
 	CreateS3SecretFunctions::Register(loader);
 	CreateBearerTokenFunctions::Register(loader);
+	CreateWebDAVSecretFunctions::Register(loader);
 
 #ifdef OVERRIDE_ENCRYPTION_UTILS
 	// set pointer to OpenSSL encryption state

src/httpfs_httplib_client.cpp

@@ -88,7 +88,16 @@ class HTTPFSClient : public HTTPClient {
 		}
 		// We use a custom Request method here, because there is no Post call with a contentreceiver in httplib
 		duckdb_httplib_openssl::Request req;
-		req.method = "POST";
+
+		// Check if a custom HTTP method is specified in extra_headers
+		auto method_it = info.params.extra_headers.find("X-DuckDB-HTTP-Method");
+		if (method_it != info.params.extra_headers.end()) {
+			// Use custom HTTP method (e.g., PROPFIND for WebDAV)
+			req.method = method_it->second;
+		} else {
+			req.method = "POST";
+		}
+
 		req.path = info.path;
 		req.headers = TransformHeaders(info.headers, info.params);
 		if (req.headers.find("Content-Type") == req.headers.end()) {

src/include/create_secret_functions.hpp

@@ -52,4 +52,18 @@ struct CreateBearerTokenFunctions {
 	                                                                         CreateSecretInput &input);
 };
 
+struct CreateWebDAVSecretFunctions {
+public:
+	static constexpr const char *WEBDAV_TYPE = "webdav";
+
+	//! Register all CreateSecretFunctions
+	static void Register(ExtensionLoader &loader);
+
+protected:
+	//! Internal function to create WebDAV secret
+	static unique_ptr<BaseSecret> CreateSecretFunctionInternal(ClientContext &context, CreateSecretInput &input);
+	//! Credential provider function
+	static unique_ptr<BaseSecret> CreateWebDAVSecretFromConfig(ClientContext &context, CreateSecretInput &input);
+};
+
 } // namespace duckdb

src/include/webdavfs.hpp

@@ -0,0 +1,105 @@
+#pragma once
+
+#include "httpfs.hpp"
+#include "duckdb/common/file_opener.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
+
+namespace duckdb {
+
+struct WebDAVAuthParams {
+	string username;
+	string password;
+
+	static WebDAVAuthParams ReadFrom(optional_ptr<FileOpener> opener, FileOpenerInfo &info);
+};
+
+struct ParsedWebDAVUrl {
+	string http_proto;
+	string host;
+	string path;
+
+	string GetHTTPUrl() const;
+};
+
+class WebDAVFileHandle : public HTTPFileHandle {
+	friend class WebDAVFileSystem;
+
+public:
+	WebDAVFileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpenFlags flags,
+	                 unique_ptr<HTTPParams> http_params_p, const WebDAVAuthParams &auth_params_p)
+	    : HTTPFileHandle(fs, file, flags, std::move(http_params_p)), auth_params(auth_params_p) {
+		if (flags.OpenForReading() && flags.OpenForWriting()) {
+			throw NotImplementedException("Cannot open a WebDAV file for both reading and writing");
+		} else if (flags.OpenForAppending()) {
+			throw NotImplementedException("Cannot open a WebDAV file for appending");
+		}
+	}
+	~WebDAVFileHandle() override;
+
+	WebDAVAuthParams auth_params;
+
+public:
+	void Close() override;
+	void Initialize(optional_ptr<FileOpener> opener) override;
+
+protected:
+	unique_ptr<HTTPClient> CreateClient() override;
+};
+
+class WebDAVFileSystem : public HTTPFileSystem {
+public:
+	WebDAVFileSystem() = default;
+
+	string GetName() const override;
+
+public:
+	// WebDAV-specific methods
+	duckdb::unique_ptr<HTTPResponse> PropfindRequest(FileHandle &handle, string url, HTTPHeaders header_map,
+	                                                 int depth = 1);
+	duckdb::unique_ptr<HTTPResponse> MkcolRequest(FileHandle &handle, string url, HTTPHeaders header_map);
+	duckdb::unique_ptr<HTTPResponse> CustomRequest(FileHandle &handle, string url, HTTPHeaders header_map,
+	                                               const string &method, char *buffer_in, idx_t buffer_in_len);
+
+	// Override standard methods for WebDAV support
+	duckdb::unique_ptr<HTTPResponse> HeadRequest(FileHandle &handle, string url, HTTPHeaders header_map) override;
+	duckdb::unique_ptr<HTTPResponse> GetRequest(FileHandle &handle, string url, HTTPHeaders header_map) override;
+	duckdb::unique_ptr<HTTPResponse> GetRangeRequest(FileHandle &handle, string url, HTTPHeaders header_map,
+	                                                 idx_t file_offset, char *buffer_out,
+	                                                 idx_t buffer_out_len) override;
+	duckdb::unique_ptr<HTTPResponse> PutRequest(FileHandle &handle, string url, HTTPHeaders header_map, char *buffer_in,
+	                                            idx_t buffer_in_len, string params = "") override;
+	duckdb::unique_ptr<HTTPResponse> DeleteRequest(FileHandle &handle, string url, HTTPHeaders header_map) override;
+
+	bool CanHandleFile(const string &fpath) override;
+	static bool IsWebDAVUrl(const string &url);
+	void RemoveFile(const string &filename, optional_ptr<FileOpener> opener = nullptr) override;
+	void MoveFile(const string &source, const string &target, optional_ptr<FileOpener> opener = nullptr) override;
+	void CreateDirectory(const string &directory, optional_ptr<FileOpener> opener = nullptr) override;
+	void RemoveDirectory(const string &directory, optional_ptr<FileOpener> opener = nullptr) override;
+	void FileSync(FileHandle &handle) override;
+	void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
+
+	bool OnDiskFile(FileHandle &handle) override {
+		return false;
+	}
+
+	bool DirectoryExists(const string &directory, optional_ptr<FileOpener> opener = nullptr) override;
+	vector<OpenFileInfo> Glob(const string &glob_pattern, FileOpener *opener = nullptr) override;
+	bool ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
+	               FileOpener *opener = nullptr) override;
+
+	static ParsedWebDAVUrl ParseUrl(const string &url);
+
+protected:
+	duckdb::unique_ptr<HTTPFileHandle> CreateHandle(const OpenFileInfo &file, FileOpenFlags flags,
+	                                                optional_ptr<FileOpener> opener) override;
+
+	HTTPException GetHTTPError(FileHandle &, const HTTPResponse &response, const string &url) override;
+
+private:
+	void AddAuthHeaders(HTTPHeaders &headers, const WebDAVAuthParams &auth_params);
+	string Base64Encode(const string &input);
+	string DirectPropfindRequest(const string &url, const WebDAVAuthParams &auth_params, int depth);
+};
+
+} // namespace duckdb