Skip to content

Add OAuth2 bearer token support for GCS #93

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+218 −16
Open

Add OAuth2 bearer token support for GCS #93

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions extension/httpfs/create_secret_functions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
lower_name, named_param.second.type().ToString());
}
secret->secret_map["requester_pays"] = Value::BOOLEAN(named_param.second.GetValue<bool>());
} else if (lower_name == "bearer_token" && input.type == "gcs") {
secret->secret_map["bearer_token"] = named_param.second.ToString();
// Mark it as sensitive
secret->redact_keys.insert("bearer_token");
} else {
throw InvalidInputException("Unknown named parameter passed to CreateSecretFunctionInternal: " +
lower_name);
Expand Down Expand Up @@ -210,6 +214,10 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
if (type == "r2") {
function.named_parameters["account_id"] = LogicalType::VARCHAR;
}

if (type == "gcs") {
function.named_parameters["bearer_token"] = LogicalType::VARCHAR;
}
}

void CreateS3SecretFunctions::RegisterCreateSecretFunction(DatabaseInstance &instance, string type) {
Expand Down
2 changes: 2 additions & 0 deletions extension/httpfs/include/s3fs.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ struct S3AuthParams {
bool use_ssl = true;
bool s3_url_compatibility_mode = false;
bool requester_pays = false;
string oauth2_bearer_token; // OAuth2 bearer token for GCS

static S3AuthParams ReadFrom(optional_ptr<FileOpener> opener, FileOpenerInfo &info);
};
Expand Down Expand Up @@ -228,6 +229,7 @@ class S3FileSystem : public HTTPFileSystem {

static string GetS3BadRequestError(S3AuthParams &s3_auth_params);
static string GetS3AuthError(S3AuthParams &s3_auth_params);
static string GetGCSAuthError(S3AuthParams &s3_auth_params);
static HTTPException GetS3Error(S3AuthParams &s3_auth_params, const HTTPResponse &response, const string &url);

protected:
Expand Down
135 changes: 119 additions & 16 deletions extension/httpfs/s3fs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,11 @@ string S3FileSystem::UrlEncode(const string &input, bool encode_slash) {
return StringUtil::URLEncode(input, encode_slash);
}

static bool IsGCSRequest(const string &url) {
return StringUtil::StartsWith(url, "gcs://") ||
StringUtil::StartsWith(url, "gs://");
}

void AWSEnvironmentCredentialsProvider::SetExtensionOptionValue(string key, const char *env_var_name) {
char *evar;

Expand Down Expand Up @@ -209,6 +214,8 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr<FileOpener> opener, FileOpenerI
if (result.url_style.empty() || url_style_result.GetScope() != SettingScope::SECRET) {
result.url_style = "path";
}
// Read bearer token for GCS
secret_reader.TryGetSecretKey("bearer_token", result.oauth2_bearer_token);
}

if (!result.region.empty() && (result.endpoint.empty() || result.endpoint == "s3.amazonaws.com")) {
Expand All @@ -235,9 +242,13 @@ unique_ptr<KeyValueSecret> CreateSecret(vector<string> &prefix_paths_p, string &
return_value->secret_map["kms_key_id"] = params.kms_key_id;
return_value->secret_map["s3_url_compatibility_mode"] = params.s3_url_compatibility_mode;
return_value->secret_map["requester_pays"] = params.requester_pays;
return_value->secret_map["bearer_token"] = params.oauth2_bearer_token;

//! Set redact keys
return_value->redact_keys = {"secret", "session_token"};
if (!params.oauth2_bearer_token.empty()) {
return_value->redact_keys.insert("bearer_token");
}

return return_value;
}
Expand Down Expand Up @@ -672,9 +683,19 @@ unique_ptr<HTTPResponse> S3FileSystem::PostRequest(FileHandle &handle, string ur
auto auth_params = handle.Cast<S3FileHandle>().auth_params;
auto parsed_s3_url = S3UrlParse(url, auth_params);
string http_url = parsed_s3_url.GetHTTPUrl(auth_params, http_params);
auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len);
auto headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "",
"", payload_hash, "application/octet-stream");

HTTPHeaders headers;
if (IsGCSRequest(url) && !auth_params.oauth2_bearer_token.empty()) {
// Use bearer token for GCS
headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
headers["Host"] = parsed_s3_url.host;
headers["Content-Type"] = "application/octet-stream";
} else {
// Use existing S3 authentication
auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len);
headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "",
"", payload_hash, "application/octet-stream");
}

return HTTPFileSystem::PostRequest(handle, http_url, headers, result, buffer_in, buffer_in_len);
}
Expand All @@ -685,28 +706,58 @@ unique_ptr<HTTPResponse> S3FileSystem::PutRequest(FileHandle &handle, string url
auto parsed_s3_url = S3UrlParse(url, auth_params);
string http_url = parsed_s3_url.GetHTTPUrl(auth_params, http_params);
auto content_type = "application/octet-stream";
auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len);

auto headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "",
"", payload_hash, content_type);

HTTPHeaders headers;
if (IsGCSRequest(url) && !auth_params.oauth2_bearer_token.empty()) {
// Use bearer token for GCS
headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
headers["Host"] = parsed_s3_url.host;
headers["Content-Type"] = content_type;
} else {
// Use existing S3 authentication
auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len);
headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "",
"", payload_hash, content_type);
}

return HTTPFileSystem::PutRequest(handle, http_url, headers, buffer_in, buffer_in_len);
}

unique_ptr<HTTPResponse> S3FileSystem::HeadRequest(FileHandle &handle, string s3_url, HTTPHeaders header_map) {
auto auth_params = handle.Cast<S3FileHandle>().auth_params;
auto parsed_s3_url = S3UrlParse(s3_url, auth_params);
string http_url = parsed_s3_url.GetHTTPUrl(auth_params);
auto headers =
create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "HEAD", auth_params, "", "", "", "");

HTTPHeaders headers;
if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) {
// Use bearer token for GCS
headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
headers["Host"] = parsed_s3_url.host;
} else {
// Use existing S3 authentication
headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host,
"s3", "HEAD", auth_params, "", "", "", "");
}

return HTTPFileSystem::HeadRequest(handle, http_url, headers);
}

unique_ptr<HTTPResponse> S3FileSystem::GetRequest(FileHandle &handle, string s3_url, HTTPHeaders header_map) {
auto auth_params = handle.Cast<S3FileHandle>().auth_params;
auto parsed_s3_url = S3UrlParse(s3_url, auth_params);
string http_url = parsed_s3_url.GetHTTPUrl(auth_params);
auto headers =
create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", "");

HTTPHeaders headers;
if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) {
// Use bearer token for GCS
headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
headers["Host"] = parsed_s3_url.host;
} else {
// Use existing S3 authentication
headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host,
"s3", "GET", auth_params, "", "", "", "");
}

return HTTPFileSystem::GetRequest(handle, http_url, headers);
}

Expand All @@ -715,17 +766,37 @@ unique_ptr<HTTPResponse> S3FileSystem::GetRangeRequest(FileHandle &handle, strin
auto auth_params = handle.Cast<S3FileHandle>().auth_params;
auto parsed_s3_url = S3UrlParse(s3_url, auth_params);
string http_url = parsed_s3_url.GetHTTPUrl(auth_params);
auto headers =
create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", "");

HTTPHeaders headers;
if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) {
// Use bearer token for GCS
headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
headers["Host"] = parsed_s3_url.host;
} else {
// Use existing S3 authentication
headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host,
"s3", "GET", auth_params, "", "", "", "");
}

return HTTPFileSystem::GetRangeRequest(handle, http_url, headers, file_offset, buffer_out, buffer_out_len);
}

unique_ptr<HTTPResponse> S3FileSystem::DeleteRequest(FileHandle &handle, string s3_url, HTTPHeaders header_map) {
auto auth_params = handle.Cast<S3FileHandle>().auth_params;
auto parsed_s3_url = S3UrlParse(s3_url, auth_params);
string http_url = parsed_s3_url.GetHTTPUrl(auth_params);
auto headers =
create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "DELETE", auth_params, "", "", "", "");

HTTPHeaders headers;
if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) {
// Use bearer token for GCS
headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
headers["Host"] = parsed_s3_url.host;
} else {
// Use existing S3 authentication
headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host,
"s3", "DELETE", auth_params, "", "", "", "");
}

return HTTPFileSystem::DeleteRequest(handle, http_url, headers);
}

Expand Down Expand Up @@ -774,7 +845,12 @@ void S3FileHandle::Initialize(optional_ptr<FileOpener> opener) {
}
if (entry->second == "403") {
// 403: FORBIDDEN
auto extra_text = S3FileSystem::GetS3AuthError(auth_params);
string extra_text;
if (IsGCSRequest(path)) {
extra_text = S3FileSystem::GetGCSAuthError(auth_params);
} else {
extra_text = S3FileSystem::GetS3AuthError(auth_params);
}
throw Exception(error.Type(), error.RawMessage() + extra_text, extra_info);
}
}
Expand Down Expand Up @@ -1036,6 +1112,24 @@ string S3FileSystem::GetS3AuthError(S3AuthParams &s3_auth_params) {
return extra_text;
}

string S3FileSystem::GetGCSAuthError(S3AuthParams &s3_auth_params) {
string extra_text = "\n\nAuthentication Failure - GCS authentication failed.";
if (s3_auth_params.oauth2_bearer_token.empty() &&
s3_auth_params.secret_access_key.empty() &&
s3_auth_params.access_key_id.empty()) {
extra_text += "\n* No credentials provided.";
extra_text += "\n* For OAuth2: CREATE SECRET (TYPE GCS, bearer_token 'your-token')";
extra_text += "\n* For HMAC: CREATE SECRET (TYPE GCS, key_id 'key', secret 'secret')";
} else if (!s3_auth_params.oauth2_bearer_token.empty()) {
extra_text += "\n* Bearer token was provided but authentication failed.";
extra_text += "\n* Ensure your OAuth2 token is valid and not expired.";
} else {
extra_text += "\n* HMAC credentials were provided but authentication failed.";
extra_text += "\n* Ensure your HMAC key_id and secret are correct.";
}
return extra_text;
}

HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const HTTPResponse &response, const string &url) {
string extra_text;
if (response.status == HTTPStatusCode::BadRequest_400) {
Expand All @@ -1051,6 +1145,15 @@ HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const HTTPR

HTTPException S3FileSystem::GetHTTPError(FileHandle &handle, const HTTPResponse &response, const string &url) {
auto &s3_handle = handle.Cast<S3FileHandle>();

// Use GCS-specific error for GCS URLs
if (IsGCSRequest(url) && response.status == HTTPStatusCode::Forbidden_403) {
string extra_text = GetGCSAuthError(s3_handle.auth_params);
auto status_message = HTTPFSUtil::GetStatusMessage(response.status);
throw HTTPException(response, "HTTP error on '%s' (HTTP %d %s)%s", url,
response.status, status_message, extra_text);
}

return GetS3Error(s3_handle.auth_params, response, url);
}
string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthParams &s3_auth_params,
Expand Down
89 changes: 89 additions & 0 deletions test/sql/secret/gcs_oauth.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
# name: test/sql/secret/gcs_oauth.test
# description: Test GCS OAuth2 bearer token support
# group: [secret]

require httpfs

statement ok
PRAGMA enable_verification

# Test creating a GCS secret with OAuth2 bearer token
statement ok
CREATE SECRET gcs_oauth_test (
TYPE GCS,
bearer_token 'test_oauth2_token_12345'
);

# Verify the secret was created
query I
SELECT COUNT(*) FROM duckdb_secrets() WHERE name = 'gcs_oauth_test' AND type = 'gcs';
----
1

# Verify bearer token is redacted
query I
SELECT COUNT(*) FROM duckdb_secrets() WHERE name = 'gcs_oauth_test' AND secret_string LIKE '%bearer_token=redacted%';
----
1

# Test creating a GCS secret with HMAC keys (backward compatibility)
statement ok
CREATE SECRET gcs_hmac_test (
TYPE GCS,
key_id 'test_key_id',
secret 'test_secret'
);

# Verify both secrets exist
query II
SELECT name, type FROM duckdb_secrets() WHERE name IN ('gcs_oauth_test', 'gcs_hmac_test') ORDER BY name;
----
gcs_hmac_test gcs
gcs_oauth_test gcs

# Test creating a GCS secret with both bearer token and HMAC (bearer token should take precedence)
statement ok
CREATE SECRET gcs_mixed_test (
TYPE GCS,
bearer_token 'oauth_token',
key_id 'hmac_key',
secret 'hmac_secret'
);

# Verify all three secrets exist
query I
SELECT COUNT(*) FROM duckdb_secrets() WHERE name LIKE 'gcs_%test';
----
3

# Clean up
statement ok
DROP SECRET gcs_oauth_test;

statement ok
DROP SECRET gcs_hmac_test;

statement ok
DROP SECRET gcs_mixed_test;

# Verify all secrets are removed
query I
SELECT COUNT(*) FROM duckdb_secrets() WHERE name LIKE 'gcs_%test';
----
0

# Test that bearer_token parameter is not allowed for S3 secrets
statement error Unknown named parameter
CREATE SECRET s3_with_bearer (
TYPE S3,
bearer_token 'should_not_work'
);
----

# Test that bearer_token parameter is not allowed for R2 secrets
statement error Unknown named parameter
CREATE SECRET r2_with_bearer (
TYPE R2,
bearer_token 'should_not_work'
);
----
Loading