diff --git a/extension/httpfs/create_secret_functions.cpp b/extension/httpfs/create_secret_functions.cpp index c9ae0a7..a5c6dd7 100644 --- a/extension/httpfs/create_secret_functions.cpp +++ b/extension/httpfs/create_secret_functions.cpp @@ -82,6 +82,10 @@ unique_ptr CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli secret->secret_map["use_ssl"] = Value::BOOLEAN(named_param.second.GetValue()); } else if (lower_name == "kms_key_id") { secret->secret_map["kms_key_id"] = named_param.second.ToString(); + } else if (lower_name == "sse_c_key") { + secret->secret_map["sse_c_key"] = named_param.second.ToString(); + } else if (lower_name == "sse_c_key_md5") { + secret->secret_map["sse_c_key_md5"] = named_param.second.ToString(); } else if (lower_name == "url_compatibility_mode") { if (named_param.second.type() != LogicalType::BOOLEAN) { throw InvalidInputException("Invalid type past to secret option: '%s', found '%s', expected: 'BOOLEAN'", @@ -120,6 +124,16 @@ unique_ptr CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli } } + bool has_sse_c_key = secret->secret_map.find("sse_c_key") != secret->secret_map.end(); + bool has_sse_c_key_md5 = secret->secret_map.find("sse_c_key_md5") != secret->secret_map.end(); + if (has_sse_c_key != has_sse_c_key_md5) { + throw InvalidInputException("Both `sse_c_key` and `sse_c_key_md5` must be set together, or neither should be set"); + } + bool has_kms_key_id = secret->secret_map.find("kms_key_id") != secret->secret_map.end(); + if (has_kms_key_id && has_sse_c_key) { + throw InvalidInputException("Cannot set `kms_key_id` and `sse_c_key` at the same time"); + } + return std::move(secret); } @@ -190,6 +204,8 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function, function.named_parameters["url_style"] = LogicalType::VARCHAR; function.named_parameters["use_ssl"] = LogicalType::BOOLEAN; function.named_parameters["kms_key_id"] = LogicalType::VARCHAR; + function.named_parameters["sse_c_key"] = LogicalType::VARCHAR; + function.named_parameters["sse_c_key_md5"] = LogicalType::VARCHAR; function.named_parameters["url_compatibility_mode"] = LogicalType::BOOLEAN; function.named_parameters["requester_pays"] = LogicalType::BOOLEAN; diff --git a/extension/httpfs/httpfs_extension.cpp b/extension/httpfs/httpfs_extension.cpp index 392d249..cee0013 100644 --- a/extension/httpfs/httpfs_extension.cpp +++ b/extension/httpfs/httpfs_extension.cpp @@ -68,6 +68,8 @@ static void LoadInternal(DatabaseInstance &instance) { config.AddExtensionOption("s3_url_style", "S3 URL style", LogicalType::VARCHAR, Value("vhost")); config.AddExtensionOption("s3_use_ssl", "S3 use SSL", LogicalType::BOOLEAN, Value(true)); config.AddExtensionOption("s3_kms_key_id", "S3 KMS Key ID", LogicalType::VARCHAR); + config.AddExtensionOption("s3_sse_c_key", "S3 SSE-C Key", LogicalType::VARCHAR); + config.AddExtensionOption("s3_sse_c_key_md5", "S3 SSE-C Key MD5", LogicalType::VARCHAR); config.AddExtensionOption("s3_url_compatibility_mode", "Disable Globs and Query Parameters on S3 URLs", LogicalType::BOOLEAN, Value(false)); config.AddExtensionOption("s3_requester_pays", "S3 use requester pays mode", LogicalType::BOOLEAN, Value(false)); diff --git a/extension/httpfs/include/s3fs.hpp b/extension/httpfs/include/s3fs.hpp index dfa9619..c100e03 100644 --- a/extension/httpfs/include/s3fs.hpp +++ b/extension/httpfs/include/s3fs.hpp @@ -27,6 +27,8 @@ struct S3AuthParams { string session_token; string endpoint; string kms_key_id; + string sse_c_key; + string sse_c_key_md5; string url_style; bool use_ssl = true; bool s3_url_compatibility_mode = false; @@ -44,9 +46,10 @@ struct AWSEnvironmentCredentialsProvider { static constexpr const char *DUCKDB_ENDPOINT_ENV_VAR = "DUCKDB_S3_ENDPOINT"; static constexpr const char *DUCKDB_USE_SSL_ENV_VAR = "DUCKDB_S3_USE_SSL"; static constexpr const char *DUCKDB_KMS_KEY_ID_ENV_VAR = "DUCKDB_S3_KMS_KEY_ID"; + static constexpr const char *DUCKDB_SSE_C_KEY_ENV_VAR = "DUCKDB_S3_SSE_C_KEY"; + static constexpr const char *DUCKDB_SSE_C_KEY_MD5_ENV_VAR = "DUCKDB_S3_SSE_C_KEY_MD5"; static constexpr const char *DUCKDB_REQUESTER_PAYS_ENV_VAR = "DUCKDB_S3_REQUESTER_PAYS"; - explicit AWSEnvironmentCredentialsProvider(DBConfig &config) : config(config) {}; DBConfig &config; diff --git a/extension/httpfs/s3fs.cpp b/extension/httpfs/s3fs.cpp index 0a54f93..03aa542 100644 --- a/extension/httpfs/s3fs.cpp +++ b/extension/httpfs/s3fs.cpp @@ -51,6 +51,7 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin // https://docs.aws.amazon.com/AmazonS3/latest/userguide/specifying-kms-encryption.html#sse-request-headers-kms bool use_sse_kms = auth_params.kms_key_id.length() > 0 && (method == "POST" || method == "PUT") && query.find("uploadId") == std::string::npos; + bool use_sse_c = auth_params.sse_c_key.length() > 0 && auth_params.sse_c_key_md5.length() > 0; res["x-amz-date"] = datetime_now; res["x-amz-content-sha256"] = payload_hash; @@ -61,13 +62,18 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin res["x-amz-server-side-encryption"] = "aws:kms"; res["x-amz-server-side-encryption-aws-kms-key-id"] = auth_params.kms_key_id; } + if (use_sse_c) { + res["x-amz-server-side-encryption-customer-algorithm"] = "AES256"; + res["x-amz-server-side-encryption-customer-key"] = auth_params.sse_c_key; + res["x-amz-server-side-encryption-customer-key-md5"] = auth_params.sse_c_key_md5; + } bool use_requester_pays = auth_params.requester_pays; if (use_requester_pays) { res["x-amz-request-payer"] = "requester"; } - string signed_headers = ""; + string signed_headers = ""; hash_bytes canonical_request_hash; hash_str canonical_request_hash_str; if (content_type.length() > 0) { @@ -80,10 +86,13 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin if (use_sse_kms) { signed_headers += ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id"; } + if (use_sse_c) { + signed_headers += ";x-amz-server-side-encryption-customer-algorithm;x-amz-server-side-encryption-customer-key;x-amz-server-side-encryption-customer-key-md5"; + } if (use_requester_pays) { signed_headers += ";x-amz-request-payer"; } - auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query; + auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query; if (content_type.length() > 0) { canonical_request += "\ncontent-type:" + content_type; } @@ -95,6 +104,11 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin canonical_request += "\nx-amz-server-side-encryption:aws:kms"; canonical_request += "\nx-amz-server-side-encryption-aws-kms-key-id:" + auth_params.kms_key_id; } + if (use_sse_c) { + canonical_request += "\nx-amz-server-side-encryption-customer-algorithm:AES256"; + canonical_request += "\nx-amz-server-side-encryption-customer-key:" + auth_params.sse_c_key; + canonical_request += "\nx-amz-server-side-encryption-customer-key-md5:" + auth_params.sse_c_key_md5; + } if (use_requester_pays) { canonical_request += "\nx-amz-request-payer:requester"; } @@ -154,6 +168,8 @@ void AWSEnvironmentCredentialsProvider::SetAll() { this->SetExtensionOptionValue("s3_endpoint", DUCKDB_ENDPOINT_ENV_VAR); this->SetExtensionOptionValue("s3_use_ssl", DUCKDB_USE_SSL_ENV_VAR); this->SetExtensionOptionValue("s3_kms_key_id", DUCKDB_KMS_KEY_ID_ENV_VAR); + this->SetExtensionOptionValue("s3_sse_c_key", DUCKDB_SSE_C_KEY_ENV_VAR); + this->SetExtensionOptionValue("s3_sse_c_key_md5", DUCKDB_SSE_C_KEY_MD5_ENV_VAR); this->SetExtensionOptionValue("s3_requester_pays", DUCKDB_REQUESTER_PAYS_ENV_VAR); } @@ -167,6 +183,8 @@ S3AuthParams AWSEnvironmentCredentialsProvider::CreateParams() { params.session_token = SESSION_TOKEN_ENV_VAR; params.endpoint = DUCKDB_ENDPOINT_ENV_VAR; params.kms_key_id = DUCKDB_KMS_KEY_ID_ENV_VAR; + params.sse_c_key = DUCKDB_SSE_C_KEY_ENV_VAR; + params.sse_c_key_md5 = DUCKDB_SSE_C_KEY_MD5_ENV_VAR; params.use_ssl = DUCKDB_USE_SSL_ENV_VAR; params.requester_pays = DUCKDB_REQUESTER_PAYS_ENV_VAR; @@ -192,6 +210,8 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr opener, FileOpenerI secret_reader.TryGetSecretKeyOrSetting("region", "s3_region", result.region); secret_reader.TryGetSecretKeyOrSetting("use_ssl", "s3_use_ssl", result.use_ssl); secret_reader.TryGetSecretKeyOrSetting("kms_key_id", "s3_kms_key_id", result.kms_key_id); + secret_reader.TryGetSecretKeyOrSetting("sse_c_key", "s3_sse_c_key", result.sse_c_key); + secret_reader.TryGetSecretKeyOrSetting("sse_c_key_md5", "s3_sse_c_key_md5", result.sse_c_key_md5); secret_reader.TryGetSecretKeyOrSetting("s3_url_compatibility_mode", "s3_url_compatibility_mode", result.s3_url_compatibility_mode); secret_reader.TryGetSecretKeyOrSetting("requester_pays", "s3_requester_pays", @@ -233,6 +253,8 @@ unique_ptr CreateSecret(vector &prefix_paths_p, string & return_value->secret_map["url_style"] = params.url_style; return_value->secret_map["use_ssl"] = params.use_ssl; return_value->secret_map["kms_key_id"] = params.kms_key_id; + return_value->secret_map["sse_c_key"] = params.sse_c_key; + return_value->secret_map["sse_c_key_md5"] = params.sse_c_key_md5; return_value->secret_map["s3_url_compatibility_mode"] = params.s3_url_compatibility_mode; return_value->secret_map["requester_pays"] = params.requester_pays;