diff --git a/extension/httpfs/create_secret_functions.cpp b/extension/httpfs/create_secret_functions.cpp index c9ae0a7..ed2fc8b 100644 --- a/extension/httpfs/create_secret_functions.cpp +++ b/extension/httpfs/create_secret_functions.cpp @@ -114,6 +114,10 @@ unique_ptr CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli lower_name, named_param.second.type().ToString()); } secret->secret_map["requester_pays"] = Value::BOOLEAN(named_param.second.GetValue()); + } else if (lower_name == "bearer_token" && input.type == "gcs") { + secret->secret_map["bearer_token"] = named_param.second.ToString(); + // Mark it as sensitive + secret->redact_keys.insert("bearer_token"); } else { throw InvalidInputException("Unknown named parameter passed to CreateSecretFunctionInternal: " + lower_name); @@ -210,6 +214,10 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function, if (type == "r2") { function.named_parameters["account_id"] = LogicalType::VARCHAR; } + + if (type == "gcs") { + function.named_parameters["bearer_token"] = LogicalType::VARCHAR; + } } void CreateS3SecretFunctions::RegisterCreateSecretFunction(DatabaseInstance &instance, string type) { diff --git a/extension/httpfs/include/s3fs.hpp b/extension/httpfs/include/s3fs.hpp index dfa9619..1404120 100644 --- a/extension/httpfs/include/s3fs.hpp +++ b/extension/httpfs/include/s3fs.hpp @@ -31,6 +31,7 @@ struct S3AuthParams { bool use_ssl = true; bool s3_url_compatibility_mode = false; bool requester_pays = false; + string oauth2_bearer_token; // OAuth2 bearer token for GCS static S3AuthParams ReadFrom(optional_ptr opener, FileOpenerInfo &info); }; @@ -228,6 +229,7 @@ class S3FileSystem : public HTTPFileSystem { static string GetS3BadRequestError(S3AuthParams &s3_auth_params); static string GetS3AuthError(S3AuthParams &s3_auth_params); + static string GetGCSAuthError(S3AuthParams &s3_auth_params); static HTTPException GetS3Error(S3AuthParams &s3_auth_params, const HTTPResponse &response, const string &url); protected: diff --git a/extension/httpfs/s3fs.cpp b/extension/httpfs/s3fs.cpp index 0a54f93..e149e61 100644 --- a/extension/httpfs/s3fs.cpp +++ b/extension/httpfs/s3fs.cpp @@ -131,6 +131,11 @@ string S3FileSystem::UrlEncode(const string &input, bool encode_slash) { return StringUtil::URLEncode(input, encode_slash); } +static bool IsGCSRequest(const string &url) { + return StringUtil::StartsWith(url, "gcs://") || + StringUtil::StartsWith(url, "gs://"); +} + void AWSEnvironmentCredentialsProvider::SetExtensionOptionValue(string key, const char *env_var_name) { char *evar; @@ -209,6 +214,8 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr opener, FileOpenerI if (result.url_style.empty() || url_style_result.GetScope() != SettingScope::SECRET) { result.url_style = "path"; } + // Read bearer token for GCS + secret_reader.TryGetSecretKey("bearer_token", result.oauth2_bearer_token); } if (!result.region.empty() && (result.endpoint.empty() || result.endpoint == "s3.amazonaws.com")) { @@ -235,9 +242,13 @@ unique_ptr CreateSecret(vector &prefix_paths_p, string & return_value->secret_map["kms_key_id"] = params.kms_key_id; return_value->secret_map["s3_url_compatibility_mode"] = params.s3_url_compatibility_mode; return_value->secret_map["requester_pays"] = params.requester_pays; + return_value->secret_map["bearer_token"] = params.oauth2_bearer_token; //! Set redact keys return_value->redact_keys = {"secret", "session_token"}; + if (!params.oauth2_bearer_token.empty()) { + return_value->redact_keys.insert("bearer_token"); + } return return_value; } @@ -672,9 +683,19 @@ unique_ptr S3FileSystem::PostRequest(FileHandle &handle, string ur auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params, http_params); - auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); - auto headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "", - "", payload_hash, "application/octet-stream"); + + HTTPHeaders headers; + if (IsGCSRequest(url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + headers["Content-Type"] = "application/octet-stream"; + } else { + // Use existing S3 authentication + auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); + headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "", + "", payload_hash, "application/octet-stream"); + } return HTTPFileSystem::PostRequest(handle, http_url, headers, result, buffer_in, buffer_in_len); } @@ -685,10 +706,20 @@ unique_ptr S3FileSystem::PutRequest(FileHandle &handle, string url auto parsed_s3_url = S3UrlParse(url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params, http_params); auto content_type = "application/octet-stream"; - auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); - - auto headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "", - "", payload_hash, content_type); + + HTTPHeaders headers; + if (IsGCSRequest(url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + headers["Content-Type"] = content_type; + } else { + // Use existing S3 authentication + auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); + headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "", + "", payload_hash, content_type); + } + return HTTPFileSystem::PutRequest(handle, http_url, headers, buffer_in, buffer_in_len); } @@ -696,8 +727,18 @@ unique_ptr S3FileSystem::HeadRequest(FileHandle &handle, string s3 auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - auto headers = - create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "HEAD", auth_params, "", "", "", ""); + + HTTPHeaders headers; + if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + } else { + // Use existing S3 authentication + headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, + "s3", "HEAD", auth_params, "", "", "", ""); + } + return HTTPFileSystem::HeadRequest(handle, http_url, headers); } @@ -705,8 +746,18 @@ unique_ptr S3FileSystem::GetRequest(FileHandle &handle, string s3_ auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - auto headers = - create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", ""); + + HTTPHeaders headers; + if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + } else { + // Use existing S3 authentication + headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, + "s3", "GET", auth_params, "", "", "", ""); + } + return HTTPFileSystem::GetRequest(handle, http_url, headers); } @@ -715,8 +766,18 @@ unique_ptr S3FileSystem::GetRangeRequest(FileHandle &handle, strin auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - auto headers = - create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", ""); + + HTTPHeaders headers; + if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + } else { + // Use existing S3 authentication + headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, + "s3", "GET", auth_params, "", "", "", ""); + } + return HTTPFileSystem::GetRangeRequest(handle, http_url, headers, file_offset, buffer_out, buffer_out_len); } @@ -724,8 +785,18 @@ unique_ptr S3FileSystem::DeleteRequest(FileHandle &handle, string auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - auto headers = - create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "DELETE", auth_params, "", "", "", ""); + + HTTPHeaders headers; + if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + } else { + // Use existing S3 authentication + headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, + "s3", "DELETE", auth_params, "", "", "", ""); + } + return HTTPFileSystem::DeleteRequest(handle, http_url, headers); } @@ -774,7 +845,12 @@ void S3FileHandle::Initialize(optional_ptr opener) { } if (entry->second == "403") { // 403: FORBIDDEN - auto extra_text = S3FileSystem::GetS3AuthError(auth_params); + string extra_text; + if (IsGCSRequest(path)) { + extra_text = S3FileSystem::GetGCSAuthError(auth_params); + } else { + extra_text = S3FileSystem::GetS3AuthError(auth_params); + } throw Exception(error.Type(), error.RawMessage() + extra_text, extra_info); } } @@ -1036,6 +1112,24 @@ string S3FileSystem::GetS3AuthError(S3AuthParams &s3_auth_params) { return extra_text; } +string S3FileSystem::GetGCSAuthError(S3AuthParams &s3_auth_params) { + string extra_text = "\n\nAuthentication Failure - GCS authentication failed."; + if (s3_auth_params.oauth2_bearer_token.empty() && + s3_auth_params.secret_access_key.empty() && + s3_auth_params.access_key_id.empty()) { + extra_text += "\n* No credentials provided."; + extra_text += "\n* For OAuth2: CREATE SECRET (TYPE GCS, bearer_token 'your-token')"; + extra_text += "\n* For HMAC: CREATE SECRET (TYPE GCS, key_id 'key', secret 'secret')"; + } else if (!s3_auth_params.oauth2_bearer_token.empty()) { + extra_text += "\n* Bearer token was provided but authentication failed."; + extra_text += "\n* Ensure your OAuth2 token is valid and not expired."; + } else { + extra_text += "\n* HMAC credentials were provided but authentication failed."; + extra_text += "\n* Ensure your HMAC key_id and secret are correct."; + } + return extra_text; +} + HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const HTTPResponse &response, const string &url) { string extra_text; if (response.status == HTTPStatusCode::BadRequest_400) { @@ -1051,6 +1145,15 @@ HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const HTTPR HTTPException S3FileSystem::GetHTTPError(FileHandle &handle, const HTTPResponse &response, const string &url) { auto &s3_handle = handle.Cast(); + + // Use GCS-specific error for GCS URLs + if (IsGCSRequest(url) && response.status == HTTPStatusCode::Forbidden_403) { + string extra_text = GetGCSAuthError(s3_handle.auth_params); + auto status_message = HTTPFSUtil::GetStatusMessage(response.status); + throw HTTPException(response, "HTTP error on '%s' (HTTP %d %s)%s", url, + response.status, status_message, extra_text); + } + return GetS3Error(s3_handle.auth_params, response, url); } string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthParams &s3_auth_params, diff --git a/test/sql/secret/gcs_oauth.test b/test/sql/secret/gcs_oauth.test new file mode 100644 index 0000000..20ac9a4 --- /dev/null +++ b/test/sql/secret/gcs_oauth.test @@ -0,0 +1,89 @@ +# name: test/sql/secret/gcs_oauth.test +# description: Test GCS OAuth2 bearer token support +# group: [secret] + +require httpfs + +statement ok +PRAGMA enable_verification + +# Test creating a GCS secret with OAuth2 bearer token +statement ok +CREATE SECRET gcs_oauth_test ( + TYPE GCS, + bearer_token 'test_oauth2_token_12345' +); + +# Verify the secret was created +query I +SELECT COUNT(*) FROM duckdb_secrets() WHERE name = 'gcs_oauth_test' AND type = 'gcs'; +---- +1 + +# Verify bearer token is redacted +query I +SELECT COUNT(*) FROM duckdb_secrets() WHERE name = 'gcs_oauth_test' AND secret_string LIKE '%bearer_token=redacted%'; +---- +1 + +# Test creating a GCS secret with HMAC keys (backward compatibility) +statement ok +CREATE SECRET gcs_hmac_test ( + TYPE GCS, + key_id 'test_key_id', + secret 'test_secret' +); + +# Verify both secrets exist +query II +SELECT name, type FROM duckdb_secrets() WHERE name IN ('gcs_oauth_test', 'gcs_hmac_test') ORDER BY name; +---- +gcs_hmac_test gcs +gcs_oauth_test gcs + +# Test creating a GCS secret with both bearer token and HMAC (bearer token should take precedence) +statement ok +CREATE SECRET gcs_mixed_test ( + TYPE GCS, + bearer_token 'oauth_token', + key_id 'hmac_key', + secret 'hmac_secret' +); + +# Verify all three secrets exist +query I +SELECT COUNT(*) FROM duckdb_secrets() WHERE name LIKE 'gcs_%test'; +---- +3 + +# Clean up +statement ok +DROP SECRET gcs_oauth_test; + +statement ok +DROP SECRET gcs_hmac_test; + +statement ok +DROP SECRET gcs_mixed_test; + +# Verify all secrets are removed +query I +SELECT COUNT(*) FROM duckdb_secrets() WHERE name LIKE 'gcs_%test'; +---- +0 + +# Test that bearer_token parameter is not allowed for S3 secrets +statement error Unknown named parameter +CREATE SECRET s3_with_bearer ( + TYPE S3, + bearer_token 'should_not_work' +); +---- + +# Test that bearer_token parameter is not allowed for R2 secrets +statement error Unknown named parameter +CREATE SECRET r2_with_bearer ( + TYPE R2, + bearer_token 'should_not_work' +); +---- \ No newline at end of file