storage_cli_optional_flags config parameter

config/blobs.yml

@@ -1,7 +1,3 @@
-azure-storage-cli/azure-storage-cli-linux-amd64:
-  size: 7143608
-  object_id: ceb2994f-cbe8-4695-413c-33785a0b6322
-  sha: sha256:246944046f7e2f919965466055e957c30aeecf13012059c624d12ab04f48f822
 expat/expat-2.5.0.tar.bz2:
   size: 569205
   object_id: 970ccd16-75ac-4c55-5280-c00c4aa8f6cc
@@ -98,6 +94,10 @@ postgres/postgresql-11.22.tar.gz:
   size: 26826810
   object_id: d1f8d34c-b438-44e7-7672-5daea8a6da66
   sha: sha256:6445a4e1533c1e8bb616d4a3784bdc4c0226b541f6f0c8d996d9f27d581d49c3
+storage-cli/storage-cli-0.0.1-linux-amd64:
+  size: 59959431
+  object_id: 0bd00d1a-88a5-48e3-5908-6b0cc90d5f88
+  sha: sha256:302c54413358f07bd1a239aa27c0752167c21dc869ce8a9501e994969be49eea
 valkey/7.2.11.tar.gz:
   size: 3441067
   object_id: ac3c2823-80c0-41e9-78e4-e42014aad04a

jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb

@@ -156,6 +156,7 @@ storage_cli_config_file_droplets: /var/vcap/jobs/cc_deployment_updater/config/st
 storage_cli_config_file_buildpacks: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_buildpacks.json
 storage_cli_config_file_packages: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_packages.json
 storage_cli_config_file_resource_pool: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_resource_pool.json
+storage_cli_optional_flags: "<%= link("cloud_controller_internal").p("cc.storage_cli_optional_flags") %>"
 
 resource_pool:
   blobstore_type: <%= link("cloud_controller_internal").p("cc.resource_pool.blobstore_type") %>

jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb

@@ -13,7 +13,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds
 end
 
 # helper: add key only when value is present
-def add(h, key, val)
+def add_optional(h, key, val)
   return if val.nil?
   return if val.respond_to?(:empty?) && val.empty?
   h[key] = val
@@ -23,28 +23,66 @@ l = link("cloud_controller_internal")
 
 scope = "cc.buildpacks.connection_config"
 provider = l.p("cc.buildpacks.blobstore_provider", nil)
+options = {}
 
-if provider != "AzureRM"
-  options = {} # for now: all non-azure providers output an empty JSON object
-else
-  options = {}
+if provider == "AzureRM"
   options["provider"] = provider
   options["account_name"]   = l.p("#{scope}.azure_storage_account_name")
   options["container_name"] = l.p("#{scope}.container_name")
-  add(options, "account_key",        l.p("#{scope}.azure_storage_access_key"))
-  add(options, "environment",        l.p("#{scope}.environment", "AzureCloud"))
-  add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil))
-
-  # optional passthrough for extra storage-cli flags
-  begin
-    custom = l.p("#{scope}.custom", {})
-    if custom.respond_to?(:each)
-      custom.each { |k, v| add(options, k.to_s, v) }
-    end
-  rescue
-    # ignore if property not defined
-  end
+  options["account_key"] = l.p("#{scope}.azure_storage_access_key")
+  add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud"))
+  add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil))
   options = cli_cfg_with_default_timeout(options, 'storage_cli')
 end
+
+if provider == "Google"
+  options["provider"] = provider
+  options["credentials_source"] = "static"
+  options["json_key"] =  l.p("#{scope}.google_json_key_string")
+  options["bucket_name"] = l.p("#{scope}.bucket_name")
+  add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil))
+  add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil))
+end
+
+if provider == "AWS"
+  options["provider"] = provider
+  options["bucket_name"] = l.p("#{scope}.bucket_name")
+  options["credentials_source"] = "static"
+  options["access_key_id"] = l.p("#{scope}.aws_access_key_id")
+  options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key")
+  add_optional(options, "region", l.p("#{scope}.region", nil))
+  add_optional(options, "host", l.p("#{scope}.host", nil))
+  add_optional(options, "port", l.p("#{scope}.port", nil))
+  add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil))
+  add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil))
+  add_optional(options, "signature_version", l.p("#{scope}.signature_version", nil))
+  add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil))
+  add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil))
+  add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil))
+end
+
+if provider ==  "aliyun"
+  options["provider"] = provider
+  options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id")
+  options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret")
+  options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint")
+  options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket")
+end
+
+if provider == "webdav"
+  options["provider"] = provider
+  options["user"] = l.p("#{scope}.username")
+  options["password"] = l.p("#{scope}.password")
+  options["endpoint"] = l.p("#{scope}.public_endpoint")
+  add_optional(options, "secret", l.p("#{scope}.secret", nil))
+  add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil))
+
+  # TLS nested object with a Cert inside
+  ca_cert=l.p("#{scope}.ca_cert",nil)
+  unless ca_cert.empty?
+    options["tls"]={"cert"=>ca_cert}
+  end
+end
+
 -%>
 <%= JSON.pretty_generate(options) %>

jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb

@@ -13,7 +13,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds
 end
 
 # helper: add key only when value is present
-def add(h, key, val)
+def add_optional(h, key, val)
   return if val.nil?
   return if val.respond_to?(:empty?) && val.empty?
   h[key] = val
@@ -23,28 +23,66 @@ l = link("cloud_controller_internal")
 
 scope = "cc.droplets.connection_config"
 provider = l.p("cc.droplets.blobstore_provider", nil)
+options = {}
 
-if provider != "AzureRM"
-  options = {} # for now: all non-azure providers output an empty JSON object
-else
-  options = {}
+if provider == "AzureRM"
   options["provider"] = provider
   options["account_name"]   = l.p("#{scope}.azure_storage_account_name")
   options["container_name"] = l.p("#{scope}.container_name")
-  add(options, "account_key",        l.p("#{scope}.azure_storage_access_key"))
-  add(options, "environment",        l.p("#{scope}.environment", "AzureCloud"))
-  add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil))
-
-  # optional passthrough for extra storage-cli flags
-  begin
-    custom = l.p("cc.droplets.connection_config.custom", {})
-    if custom.respond_to?(:each)
-      custom.each { |k, v| add(options, k.to_s, v) }
-    end
-  rescue
-    # ignore if property not defined
-  end
+  options["account_key"] = l.p("#{scope}.azure_storage_access_key")
+  add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud"))
+  add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil))
   options = cli_cfg_with_default_timeout(options, 'storage_cli')
 end
+
+if provider == "Google"
+  options["provider"] = provider
+  options["credentials_source"] = "static"
+  options["json_key"] =  l.p("#{scope}.google_json_key_string")
+  options["bucket_name"] = l.p("#{scope}.bucket_name")
+  add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil))
+  add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil))
+end
+
+if provider == "AWS"
+  options["provider"] = provider
+  options["bucket_name"] = l.p("#{scope}.bucket_name")
+  options["credentials_source"] = "static"
+  options["access_key_id"] = l.p("#{scope}.aws_access_key_id")
+  options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key")
+  add_optional(options, "region", l.p("#{scope}.region", nil))
+  add_optional(options, "host", l.p("#{scope}.host", nil))
+  add_optional(options, "port", l.p("#{scope}.port", nil))
+  add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil))
+  add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil))
+  add_optional(options, "signature_version", l.p("#{scope}.signature_version", nil))
+  add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil))
+  add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil))
+  add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil))
+end
+
+if provider ==  "aliyun"
+  options["provider"] = provider
+  options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id")
+  options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret")
+  options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint")
+  options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket")
+end
+
+if provider == "webdav"
+  options["provider"] = provider
+  options["user"] = l.p("#{scope}.username")
+  options["password"] = l.p("#{scope}.password")
+  options["endpoint"] = l.p("#{scope}.public_endpoint")
+  add_optional(options, "secret", l.p("#{scope}.secret", nil))
+  add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil))
+
+  # TLS nested object with a Cert inside
+  ca_cert=l.p("#{scope}.ca_cert",nil)
+  unless ca_cert.empty?
+    options["tls"]={"cert"=>ca_cert}
+  end
+end
+
 -%>
 <%= JSON.pretty_generate(options) %>

jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb

@@ -13,7 +13,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds
 end
 
 # helper: add key only when value is present
-def add(h, key, val)
+def add_optional(h, key, val)
   return if val.nil?
   return if val.respond_to?(:empty?) && val.empty?
   h[key] = val
@@ -23,28 +23,66 @@ l = link("cloud_controller_internal")
 
 scope = "cc.packages.connection_config"
 provider = l.p("cc.packages.blobstore_provider", nil)
+options = {}
 
-if provider != "AzureRM"
-  options = {} # for now: all non-azure providers output an empty JSON object
-else
-  options = {}
+if provider == "AzureRM"
   options["provider"] = provider
   options["account_name"]   = l.p("#{scope}.azure_storage_account_name")
   options["container_name"] = l.p("#{scope}.container_name")
-  add(options, "account_key",        l.p("#{scope}.azure_storage_access_key"))
-  add(options, "environment",        l.p("#{scope}.environment", "AzureCloud"))
-  add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil))
-
-  # optional passthrough for extra storage-cli flags
-  begin
-    custom = l.p("#{scope}.custom", {})
-    if custom.respond_to?(:each)
-      custom.each { |k, v| add(options, k.to_s, v) }
-    end
-  rescue
-    # ignore if property not defined
-  end
+  options["account_key"] = l.p("#{scope}.azure_storage_access_key")
+  add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud"))
+  add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil))
   options = cli_cfg_with_default_timeout(options, 'storage_cli')
 end
+
+if provider == "Google"
+  options["provider"] = provider
+  options["credentials_source"] = "static"
+  options["json_key"] =  l.p("#{scope}.google_json_key_string")
+  options["bucket_name"] = l.p("#{scope}.bucket_name")
+  add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil))
+  add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil))
+end
+
+if provider == "AWS"
+  options["provider"] = provider
+  options["bucket_name"] = l.p("#{scope}.bucket_name")
+  options["credentials_source"] = "static"
+  options["access_key_id"] = l.p("#{scope}.aws_access_key_id")
+  options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key")
+  add_optional(options, "region", l.p("#{scope}.region", nil))
+  add_optional(options, "host", l.p("#{scope}.host", nil))
+  add_optional(options, "port", l.p("#{scope}.port", nil))
+  add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil))
+  add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil))
+  add_optional(options, "signature_version", l.p("#{scope}.signature_version", nil))
+  add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil))
+  add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil))
+  add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil))
+end
+
+if provider ==  "aliyun"
+  options["provider"] = provider
+  options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id")
+  options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret")
+  options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint")
+  options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket")
+end
+
+if provider == "webdav"
+  options["provider"] = provider
+  options["user"] = l.p("#{scope}.username")
+  options["password"] = l.p("#{scope}.password")
+  options["endpoint"] = l.p("#{scope}.public_endpoint")
+  add_optional(options, "secret", l.p("#{scope}.secret", nil))
+  add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil))
+
+  # TLS nested object with a Cert inside
+  ca_cert=l.p("#{scope}.ca_cert",nil)
+  unless ca_cert.empty?
+    options["tls"]={"cert"=>ca_cert}
+  end
+end
+
 -%>
 <%= JSON.pretty_generate(options) %>