@@ -7,10 +7,25 @@ namespace duckdb {
77
88void CreateS3SecretFunctions::Register (DatabaseInstance &instance) {
99 RegisterCreateSecretFunction (instance, " s3"
10+ RegisterCreateSecretFunction (instance, " aws"
1011 RegisterCreateSecretFunction (instance, " r2"
1112 RegisterCreateSecretFunction (instance, " gcs"
1213}
1314
15+ static Value MapToStruct (const Value &map) {
16+ auto children = MapValue::GetChildren (map);
17+
18+ child_list_t <Value> struct_fields;
19+ for (const auto &kv_child : children) {
20+ auto kv_pair = StructValue::GetChildren (kv_child);
21+ if (kv_pair.size () != 2 ) {
22+ throw InvalidInputException (" Invalid input passed to refresh_info"
23+ }
24+
25+ struct_fields.push_back ({kv_pair[0 ].ToString (), kv_pair[1 ]});
26+ }
27+ return Value::STRUCT (struct_fields);
28+ }
1429unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal (ClientContext &context,
1530 CreateSecretInput &input) {
1631 // Set scope to user provided scope or the default
@@ -25,6 +40,8 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
2540 } else if (input.type == " gcs"
2641 scope.push_back (" gcs://"
2742 scope.push_back (" gs://"
43+ } else if (input.type == " aws"
44+ scope.push_back (" "
2845 } else {
2946 throw InternalException (" Unknown secret type found in httpfs extension: '%s'" type );
3047 }
@@ -39,6 +56,8 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
3956 secret->secret_map [" url_style" " path"
4057 }
4158
59+ bool refresh = false ;
60+
4261 // apply any overridden settings
4362 for (const auto &named_param : input.options ) {
4463 auto lower_name = StringUtil::Lower (named_param.first );
@@ -61,6 +80,8 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
6180 lower_name, named_param.second .type ().ToString ());
6281 }
6382 secret->secret_map [" use_ssl" Value::BOOLEAN (named_param.second .GetValue <bool >());
83+ } else if (lower_name == " kms_key_id"
84+ secret->secret_map [" kms_key_id" second .ToString ();
6485 } else if (lower_name == " url_compatibility_mode"
6586 if (named_param.second .type () != LogicalType::BOOLEAN) {
6687 throw InvalidInputException (" Invalid type past to secret option: '%s', found '%s', expected: 'BOOLEAN'"
@@ -69,14 +90,86 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
6990 secret->secret_map [" url_compatibility_mode" Value::BOOLEAN (named_param.second .GetValue <bool >());
7091 } else if (lower_name == " account_id"
7192 continue ; // handled already
93+ } else if (lower_name == " refresh"
94+ if (refresh) {
95+ throw InvalidInputException (" Can not set `refresh` and `refresh_info` at the same time"
96+ }
97+ refresh = named_param.second .GetValue <string>() == " auto"
98+ secret->secret_map [" refresh" Value (" auto"
99+ child_list_t <Value> struct_fields;
100+ for (const auto &named_param : input.options ) {
101+ auto lower_name = StringUtil::Lower (named_param.first );
102+ struct_fields.push_back ({lower_name, named_param.second });
103+ }
104+ secret->secret_map [" refresh_info" Value::STRUCT (struct_fields);
105+ } else if (lower_name == " refresh_info"
106+ if (refresh) {
107+ throw InvalidInputException (" Can not set `refresh` and `refresh_info` at the same time"
108+ }
109+ refresh = true ;
110+ secret->secret_map [" refresh_info" MapToStruct (named_param.second );
72111 } else {
73- throw InternalException (" Unknown named parameter passed to CreateSecretFunctionInternal: "
112+ throw InvalidInputException (" Unknown named parameter passed to CreateSecretFunctionInternal: "
113+ lower_name);
74114 }
75115 }
76116
77117 return std::move (secret);
78118}
79119
120+ CreateSecretInput CreateS3SecretFunctions::GenerateRefreshSecretInfo (const SecretEntry &secret_entry,
121+ Value &refresh_info) {
122+ const auto &kv_secret = dynamic_cast <const KeyValueSecret &>(*secret_entry.secret );
123+
124+ CreateSecretInput result;
125+ result.on_conflict = OnCreateConflict::REPLACE_ON_CONFLICT;
126+ result.persist_type = SecretPersistType::TEMPORARY;
127+
128+ result.type = kv_secret.GetType ();
129+ result.name = kv_secret.GetName ();
130+ result.provider = kv_secret.GetProvider ();
131+ result.storage_type = secret_entry.storage_mode ;
132+ result.scope = kv_secret.GetScope ();
133+
134+ auto result_child_count = StructType::GetChildCount (refresh_info.type ());
135+ auto refresh_info_children = StructValue::GetChildren (refresh_info);
136+ D_ASSERT (refresh_info_children.size () == result_child_count);
137+ for (idx_t i = 0 ; i < result_child_count; i++) {
138+ auto &key = StructType::GetChildName (refresh_info.type (), i);
139+ auto &value = refresh_info_children[i];
140+ result.options [key] = value;
141+ }
142+
143+ return result;
144+ }
145+
146+ // ! Function that will automatically try to refresh a secret
147+ bool CreateS3SecretFunctions::TryRefreshS3Secret (ClientContext &context, const SecretEntry &secret_to_refresh) {
148+ const auto &kv_secret = dynamic_cast <const KeyValueSecret &>(*secret_to_refresh.secret );
149+
150+ Value refresh_info;
151+ if (!kv_secret.TryGetValue (" refresh_info"
152+ return false ;
153+ }
154+ auto &secret_manager = context.db ->GetSecretManager ();
155+ auto refresh_input = GenerateRefreshSecretInfo (secret_to_refresh, refresh_info);
156+
157+ // TODO: change SecretManager API to avoid requiring catching this exception
158+ try {
159+ auto res = secret_manager.CreateSecret (context, refresh_input);
160+ auto &new_secret = dynamic_cast <const KeyValueSecret &>(*res->secret );
161+ DUCKDB_LOG_INFO (context, " Successfully refreshed secret: %s, new key_id: %s"
162+ secret_to_refresh.secret ->GetName (), new_secret.TryGetValue (" key_id" ToString ());
163+ return true ;
164+ } catch (std::exception &ex) {
165+ ErrorData error (ex);
166+ string new_message = StringUtil::Format (" Exception thrown while trying to refresh secret %s. To fix this, "
167+ " please recreate or remove the secret and try again. Error: '%s'"
168+ secret_to_refresh.secret ->GetName (), error.Message ());
169+ throw Exception (error.Type (), new_message);
170+ }
171+ }
172+
80173unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateS3SecretFromConfig (ClientContext &context,
81174 CreateSecretInput &input) {
82175 return CreateSecretFunctionInternal (context, input);
@@ -90,8 +183,23 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
90183 function.named_parameters [" endpoint"
91184 function.named_parameters [" url_style"
92185 function.named_parameters [" use_ssl"
186+ function.named_parameters [" kms_key_id"
93187 function.named_parameters [" url_compatibility_mode"
94188
189+ // Whether a secret refresh attempt should be made when the secret appears to be incorrect
190+ function.named_parameters [" refresh"
191+
192+ // Refresh Modes
193+ // - auto
194+ // - disabled
195+ // - on_error
196+ // - on_timeout
197+
198+ // - on_use: every time a secret is used, it will refresh.
199+
200+ // Debugging/testing option: it allows specifying how the secret will be refreshed using a manually specfied MAP
201+ function.named_parameters [" refresh_info" LogicalType::MAP (LogicalType::VARCHAR, LogicalType::VARCHAR);
202+
95203 if (type == " r2"
96204 function.named_parameters [" account_id"
97205 }
@@ -103,6 +211,7 @@ void CreateS3SecretFunctions::RegisterCreateSecretFunction(DatabaseInstance &ins
103211 secret_type.name = type;
104212 secret_type.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
105213 secret_type.default_provider = " config"
214+ secret_type.extension = " httpfs"
106215
107216 ExtensionUtil::RegisterSecretType (instance, secret_type);
108217
@@ -117,6 +226,7 @@ void CreateBearerTokenFunctions::Register(DatabaseInstance &instance) {
117226 secret_type_hf.name = HUGGINGFACE_TYPE;
118227 secret_type_hf.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
119228 secret_type_hf.default_provider = " config"
229+ secret_type_hf.extension = " httpfs"
120230 ExtensionUtil::RegisterSecretType (instance, secret_type_hf);
121231
122232 // Huggingface config provider
0 commit comments