From b116f03d70ae8a89f98d19fbe9b5dd4c4254d83a Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Thu, 30 Oct 2025 09:52:18 +0800 Subject: [PATCH 1/2] Added support for additional CLI arg/env var extension Signed-off-by: michaelawyu --- go.mod | 2 +- pkg/credentials/config.go | 81 ++++++++++-- pkg/credentials/config_test.go | 104 ++++++++++++++- .../controller/main.go} | 0 pkg/examples/kubelogin/main.go | 118 ++++++++++++++++++ 5 files changed, 287 insertions(+), 18 deletions(-) rename pkg/{controller_example.go => examples/controller/main.go} (100%) create mode 100644 pkg/examples/kubelogin/main.go diff --git a/go.mod b/go.mod index d131f71..3a1e189 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.24.0 require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.36.1 + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.33.0 k8s.io/apimachinery v0.33.0 k8s.io/client-go v0.33.0 @@ -53,7 +54,6 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.32.1 // indirect k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect diff --git a/pkg/credentials/config.go b/pkg/credentials/config.go index 84b2623..2c023c3 100644 --- a/pkg/credentials/config.go +++ b/pkg/credentials/config.go @@ -8,6 +8,7 @@ import ( "net/url" "os" + "gopkg.in/yaml.v3" "k8s.io/client-go/rest" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest" @@ -15,15 +16,32 @@ import ( "sigs.k8s.io/cluster-inventory-api/apis/v1alpha1" ) -// client.authentication.k8s.io/exec is a reserved extension key defined by the Kubernetes -// client authentication API (SIG Auth), not by the ClusterProfile API. -// Reference: -// https://kubernetes.io/docs/reference/config-api/client-authentication.v1beta1/#client-authentication-k8s-io-v1beta1-Cluster -const clusterExtensionKey = "client.authentication.k8s.io/exec" +const ( + // client.authentication.k8s.io/exec is a reserved extension key defined by the Kubernetes + // client authentication API (SIG Auth), not by the ClusterProfile API. + // Reference: + // https://kubernetes.io/docs/reference/config-api/client-authentication.v1beta1/#client-authentication-k8s-io-v1beta1-Cluster + clusterExecExtensionKey = "client.authentication.k8s.io/exec" + + // additionalCLIArgsExtensionKey and additionalEnvVarsExtensionKey are + // two reserved extensions defined in KEP 5339, which allows users to pass in (usually cluster-specific) + // additional command-line arguments and environment variables to the exec plugin from + // the ClusterProfile API side. + additionalCLIArgsExtensionKey = "multicluster.x-k8s.io/clusterprofiles/auth/exec/additional-args" + additionalEnvVarsExtensionKey = "multicluster.x-k8s.io/clusterprofiles/auth/exec/additional-envs" +) + +type AdditionalCLIArgEnvVarExtensionFlag int + +const ( + AdditionalCLIArgEnvVarExtensionFlagIgnore AdditionalCLIArgEnvVarExtensionFlag = iota + AdditionalCLIArgEnvVarExtensionFlagAllow +) type Provider struct { - Name string `json:"name"` - ExecConfig *clientcmdapi.ExecConfig `json:"execConfig"` + Name string `json:"name"` + ExecConfig *clientcmdapi.ExecConfig `json:"execConfig"` + AdditionalCLIArgEnvVarExtensionFlag AdditionalCLIArgEnvVarExtensionFlag `json:"additionalCLIArgEnvVarExtensionFlag,omitempty"` } type CredentialsProvider struct { @@ -68,11 +86,47 @@ func (cp *CredentialsProvider) BuildConfigFromCP(clusterprofile *v1alpha1.Cluste } // 2. Get Exec Config - execConfig := cp.getExecConfigFromConfig(clusterAccessor.Name) + execConfig, additionalCLIArgEnvVarsExtFlag := cp.getExecConfigAndFlagsFromConfig(clusterAccessor.Name) if execConfig == nil { return nil, fmt.Errorf("no exec credentials found for provider %q", clusterAccessor.Name) } + // 3. Add additional CLI arguments and environment variables from cluster extensions if allowed. + for idx := range clusterAccessor.Cluster.Extensions { + ext := &clusterAccessor.Cluster.Extensions[idx] + + switch { + case additionalCLIArgEnvVarsExtFlag == AdditionalCLIArgEnvVarExtensionFlagAllow && ext.Name == additionalCLIArgsExtensionKey: + var additionalArgs []string + if err := yaml.Unmarshal(ext.Extension.Raw, &additionalArgs); err != nil { + return nil, fmt.Errorf("failed to unmarshal additional CLI args extension: %w", err) + } + execConfig.Args = append(execConfig.Args, additionalArgs...) + case additionalCLIArgEnvVarsExtFlag == AdditionalCLIArgEnvVarExtensionFlagAllow && ext.Name == additionalEnvVarsExtensionKey: + var additionalEnvs map[string]string + if err := yaml.Unmarshal(ext.Extension.Raw, &additionalEnvs); err != nil { + return nil, fmt.Errorf("failed to unmarshal additional env vars extension: %w", err) + } + + // Update the value of existing env vars. + for idx := range execConfig.Env { + env := &execConfig.Env[idx] + if _, exists := additionalEnvs[env.Name]; exists { + env.Value = additionalEnvs[env.Name] + delete(additionalEnvs, env.Name) + } + } + + // Add new env vars. + for name, value := range additionalEnvs { + execConfig.Env = append(execConfig.Env, clientcmdapi.ExecEnvVar{ + Name: name, + Value: value, + }) + } + } + } + // 3. build resulting rest.Config config := &rest.Config{ Host: clusterAccessor.Cluster.Server, @@ -94,6 +148,7 @@ func (cp *CredentialsProvider) BuildConfigFromCP(clusterprofile *v1alpha1.Cluste Env: execConfig.Env, InteractiveMode: "Never", ProvideClusterInfo: execConfig.ProvideClusterInfo, + Config: execConfig.Config, } // Propagate reserved extension into ExecCredential.Spec.Cluster.Config if present @@ -101,18 +156,20 @@ func (cp *CredentialsProvider) BuildConfigFromCP(clusterprofile *v1alpha1.Cluste if err := clientcmdlatest.Scheme.Convert(&clusterAccessor.Cluster, internalCluster, nil); err != nil { return nil, fmt.Errorf("failed to convert v1 Cluster to internal: %w", err) } - config.ExecProvider.Config = internalCluster.Extensions[clusterExtensionKey] + if extData, ok := internalCluster.Extensions[clusterExecExtensionKey]; ok { + config.ExecProvider.Config = extData + } return config, nil } -func (cp *CredentialsProvider) getExecConfigFromConfig(providerName string) *clientcmdapi.ExecConfig { +func (cp *CredentialsProvider) getExecConfigAndFlagsFromConfig(providerName string) (*clientcmdapi.ExecConfig, AdditionalCLIArgEnvVarExtensionFlag) { for _, provider := range cp.Providers { if provider.Name == providerName { - return provider.ExecConfig + return provider.ExecConfig, provider.AdditionalCLIArgEnvVarExtensionFlag } } - return nil + return nil, AdditionalCLIArgEnvVarExtensionFlagIgnore } // getClusterAccessorFromClusterProfile returns the first AccessProvider from the ClusterProfile diff --git a/pkg/credentials/config_test.go b/pkg/credentials/config_test.go index 461ffdb..210cebf 100644 --- a/pkg/credentials/config_test.go +++ b/pkg/credentials/config_test.go @@ -25,7 +25,9 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" + "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" clientauthenticationv1 "k8s.io/client-go/pkg/apis/clientauthentication/v1" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1" @@ -65,6 +67,7 @@ var _ = ginkgo.Describe("CredentialsProvider", func() { Args: []string{"arg3"}, APIVersion: "client.authentication.k8s.io/v1beta1", }, + AdditionalCLIArgEnvVarExtensionFlag: AdditionalCLIArgEnvVarExtensionFlagAllow, }, } credentialsProvider = New(testProviders) @@ -132,6 +135,7 @@ var _ = ginkgo.Describe("CredentialsProvider", func() { Command: "gke-gcloud-auth-plugin", ProvideClusterInfo: true, }, + AdditionalCLIArgEnvVarExtensionFlag: AdditionalCLIArgEnvVarExtensionFlagIgnore, }, }, } @@ -150,6 +154,7 @@ var _ = ginkgo.Describe("CredentialsProvider", func() { gomega.Expect(cp.Providers).To(gomega.HaveLen(1)) gomega.Expect(cp.Providers[0].Name).To(gomega.Equal("gkeFleet")) gomega.Expect(cp.Providers[0].ExecConfig.Command).To(gomega.Equal("gke-gcloud-auth-plugin")) + gomega.Expect(cp.Providers[0].AdditionalCLIArgEnvVarExtensionFlag).To(gomega.Equal(AdditionalCLIArgEnvVarExtensionFlagIgnore)) }) ginkgo.It("should return an error when file does not exist", func() { @@ -185,26 +190,38 @@ var _ = ginkgo.Describe("CredentialsProvider", func() { ginkgo.Describe("getExecConfigFromConfig", func() { ginkgo.It("should return the correct ExecConfig for existing provider", func() { - execConfig := credentialsProvider.getExecConfigFromConfig("test-provider-1") + execConfig, additionalCLIArgEnvVarsExtFlag := credentialsProvider.getExecConfigAndFlagsFromConfig("test-provider-1") gomega.Expect(execConfig).NotTo(gomega.BeNil()) gomega.Expect(execConfig.Command).To(gomega.Equal("test-command-1")) gomega.Expect(execConfig.Args).To(gomega.Equal([]string{"arg1", "arg2"})) + gomega.Expect(additionalCLIArgEnvVarsExtFlag).To(gomega.Equal(AdditionalCLIArgEnvVarExtensionFlagIgnore)) + }) + + ginkgo.It("should return the correct ExecConfig for another existing provider", func() { + execConfig, additionalCLIArgEnvVarsExtFlag := credentialsProvider.getExecConfigAndFlagsFromConfig("test-provider-2") + gomega.Expect(execConfig).NotTo(gomega.BeNil()) + gomega.Expect(execConfig.Command).To(gomega.Equal("test-command-2")) + gomega.Expect(execConfig.Args).To(gomega.Equal([]string{"arg3"})) + gomega.Expect(additionalCLIArgEnvVarsExtFlag).To(gomega.Equal(AdditionalCLIArgEnvVarExtensionFlagAllow)) }) ginkgo.It("should return nil for non-existing provider", func() { - execConfig := credentialsProvider.getExecConfigFromConfig("non-existent-provider") + execConfig, additionalCLIArgEnvVarsExtFlag := credentialsProvider.getExecConfigAndFlagsFromConfig("non-existent-provider") gomega.Expect(execConfig).To(gomega.BeNil()) + gomega.Expect(additionalCLIArgEnvVarsExtFlag).To(gomega.Equal(AdditionalCLIArgEnvVarExtensionFlagIgnore)) }) ginkgo.It("should return nil for empty provider name", func() { - execConfig := credentialsProvider.getExecConfigFromConfig("") + execConfig, additionalCLIArgEnvVarsExtFlag := credentialsProvider.getExecConfigAndFlagsFromConfig("") gomega.Expect(execConfig).To(gomega.BeNil()) + gomega.Expect(additionalCLIArgEnvVarsExtFlag).To(gomega.Equal(AdditionalCLIArgEnvVarExtensionFlagIgnore)) }) ginkgo.It("should handle CredentialsProvider with no providers", func() { emptyCP := New([]Provider{}) - execConfig := emptyCP.getExecConfigFromConfig("any-provider") + execConfig, additionalCLIArgEnvVarsExtFlag := emptyCP.getExecConfigAndFlagsFromConfig("any-provider") gomega.Expect(execConfig).To(gomega.BeNil()) + gomega.Expect(additionalCLIArgEnvVarsExtFlag).To(gomega.Equal(AdditionalCLIArgEnvVarExtensionFlagIgnore)) }) }) @@ -285,6 +302,8 @@ var _ = ginkgo.Describe("CredentialsProvider", func() { ginkgo.Describe("BuildConfigFromCP", func() { var clusterProfile *v1alpha1.ClusterProfile + additionalCLIArgsData, _ := yaml.Marshal([]string{"--audience", "audience"}) + additionalEnvVarsData, _ := yaml.Marshal(map[string]string{"CLIENT_ID": "client-id", "TENANT_ID": "tenant-id"}) ginkgo.BeforeEach(func() { clusterProfile = &v1alpha1.ClusterProfile{ ObjectMeta: metav1.ObjectMeta{ @@ -298,6 +317,26 @@ var _ = ginkgo.Describe("CredentialsProvider", func() { Server: "https://test-server.com", CertificateAuthorityData: []byte("test-ca-data"), ProxyURL: "http://proxy.example.com", + Extensions: []clientcmdv1.NamedExtension{ + { + Name: clusterExecExtensionKey, + Extension: runtime.RawExtension{ + Raw: []byte("arbitrary-data"), + }, + }, + { + Name: additionalCLIArgsExtensionKey, + Extension: runtime.RawExtension{ + Raw: additionalCLIArgsData, + }, + }, + { + Name: additionalEnvVarsExtensionKey, + Extension: runtime.RawExtension{ + Raw: additionalEnvVarsData, + }, + }, + }, }, }, }, @@ -327,7 +366,7 @@ var _ = ginkgo.Describe("CredentialsProvider", func() { gomega.Expect(err.Error()).To(gomega.ContainSubstring("no exec credentials found for provider")) }) - ginkgo.It("should build config successfully", func() { + ginkgo.It("should build config successfully (no additional CLI args/env vars)", func() { cred := clientauthenticationv1.ExecCredential{ TypeMeta: metav1.TypeMeta{ APIVersion: "client.authentication.k8s.io/v1", @@ -355,6 +394,61 @@ var _ = ginkgo.Describe("CredentialsProvider", func() { gomega.Expect(config).NotTo(gomega.BeNil()) gomega.Expect(config.Host).To(gomega.Equal("https://test-server.com")) gomega.Expect(config.TLSClientConfig.CAData).To(gomega.Equal([]byte("test-ca-data"))) + gomega.Expect(config.ExecProvider.Command).To(gomega.Equal("cat")) + gomega.Expect(config.ExecProvider.Args).To(gomega.Equal([]string{testFile})) + }) + + ginkgo.It("should build config successfully (with additional CLI args/env vars)", func() { + cred := clientauthenticationv1.ExecCredential{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "client.authentication.k8s.io/v1", + Kind: "ExecCredential", + }, + Status: &clientauthenticationv1.ExecCredentialStatus{ + Token: "test-token", + }, + } + jsonData, err := json.Marshal(cred) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + testFile := filepath.Join(tempDir, "test-config.json") + err = os.WriteFile(testFile, jsonData, 0644) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + execCP := New([]Provider{ + { + Name: "test-provider-1", + ExecConfig: &clientcmdapi.ExecConfig{ + APIVersion: "client.authentication.k8s.io/v1", + Command: "cat", + Args: []string{testFile}, + Env: []clientcmdapi.ExecEnvVar{ + { + Name: "CLIENT_ID", + Value: "None", + }, + }, + }, + AdditionalCLIArgEnvVarExtensionFlag: AdditionalCLIArgEnvVarExtensionFlagAllow, + }, + }) + + config, err := execCP.BuildConfigFromCP(clusterProfile) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(config).NotTo(gomega.BeNil()) + gomega.Expect(config.Host).To(gomega.Equal("https://test-server.com")) + gomega.Expect(config.TLSClientConfig.CAData).To(gomega.Equal([]byte("test-ca-data"))) + gomega.Expect(config.ExecProvider.Command).To(gomega.Equal("cat")) + gomega.Expect(config.ExecProvider.Args).To(gomega.Equal([]string{testFile, "--audience", "audience"})) + gomega.Expect(len(config.ExecProvider.Env)).To(gomega.Equal(2)) + gomega.Expect(config.ExecProvider.Env).To(gomega.ContainElements( + clientcmdapi.ExecEnvVar{ + Name: "CLIENT_ID", + Value: "client-id", + }, + clientcmdapi.ExecEnvVar{ + Name: "TENANT_ID", + Value: "tenant-id", + }, + )) }) }) }) diff --git a/pkg/controller_example.go b/pkg/examples/controller/main.go similarity index 100% rename from pkg/controller_example.go rename to pkg/examples/controller/main.go diff --git a/pkg/examples/kubelogin/main.go b/pkg/examples/kubelogin/main.go new file mode 100644 index 0000000..ce47049 --- /dev/null +++ b/pkg/examples/kubelogin/main.go @@ -0,0 +1,118 @@ +package main + +import ( + "log" + + "gopkg.in/yaml.v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" + + "sigs.k8s.io/cluster-inventory-api/apis/v1alpha1" + "sigs.k8s.io/cluster-inventory-api/pkg/credentials" +) + +func main() { + providers := []credentials.Provider{ + { + Name: "aks-workload-identity", + ExecConfig: &clientcmdapi.ExecConfig{ + Command: "kubelogin", + Args: []string{ + "get-token", + "--login", + "workloadidentity", + "--federated-token-file", + // The well-known path where AKS mounts the service account token as a projected volume. + // + // This is an application-specific information and it can be configured to + // a different path if needed. + "/var/run/secrets/tokens/azure-identity-token", + }, + Env: []clientcmdapi.ExecEnvVar{}, + APIVersion: "client.authentication.k8s.io/v1beta1", + ProvideClusterInfo: false, + InteractiveMode: clientcmdapi.NeverExecInteractiveMode, + }, + AdditionalCLIArgEnvVarExtensionFlag: credentials.AdditionalCLIArgEnvVarExtensionFlagAllow, + }, + } + cps := credentials.New(providers) + + // The additional arguments are cluster-specific information. + additionalArgs := []string{ + "--tenant-id", "TENANT_ID", + "--authority-host", "https://login.microsoftonline.com/", + // The kubelogin plugin already knows the scopes for AKS; no need to specify it explicitly. + } + additionalArgsYAML, err := yaml.Marshal(additionalArgs) + if err != nil { + log.Fatalf("failed to marshal additional args") + } + + additionalEnvVars := map[string]string{ + "AZURE_CLIENT_ID": "CLIENT_ID", + } + additionalEnvVarsYAML, err := yaml.Marshal(additionalEnvVars) + if err != nil { + log.Fatalf("failed to marshal additional env vars") + } + + profile := &v1alpha1.ClusterProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bravelion", + Namespace: "fleet-system", + }, + Spec: v1alpha1.ClusterProfileSpec{ + DisplayName: "bravelion", + ClusterManager: v1alpha1.ClusterManager{ + Name: "kubefleet", + }, + }, + Status: v1alpha1.ClusterProfileStatus{ + CredentialProviders: []v1alpha1.CredentialProvider{ + { + Name: "aks-workload-identity", + Cluster: clientcmdapiv1.Cluster{ + Server: "https://bravelion.hcp.eastus.azmk8s.io:443", + CertificateAuthorityData: []byte(""), + Extensions: []clientcmdapiv1.NamedExtension{ + { + Name: "multicluster.x-k8s.io/clusterprofiles/auth/exec/additional-args", + Extension: runtime.RawExtension{ + Raw: additionalArgsYAML, + }, + }, + { + Name: "multicluster.x-k8s.io/clusterprofiles/auth/exec/additional-envs", + Extension: runtime.RawExtension{ + Raw: additionalEnvVarsYAML, + }, + }, + }, + }, + }, + }, + }, + } + + restConfig, err := cps.BuildConfigFromCP(profile) + if err != nil { + log.Fatalf("Failed to prepare REST config: %v", err) + } + + // The generated REST config can be used to build a Kubernetes client. + // + // It will invoke the kubelogin plugin as follows: + // + // kubelogin get-token \ + // --login workloadidentity \ + // --federated-token-file /var/run/secrets/tokens/azure-identity-token \ + // --tenant-id TENANT_ID \ + // --client-id CLIENT_ID \ + // --authority-host https://login.microsoftonline.com/ + log.Printf("Prepared REST config:\n%+v", restConfig) + log.Printf("CLI Args: %s", restConfig.ExecProvider.Args) + log.Printf("Env Vars: %+v", restConfig.ExecProvider.Env) +} From 03f11ddea4ab06fe62352f38c8c77814df8a5708 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Thu, 30 Oct 2025 10:11:57 +0800 Subject: [PATCH 2/2] Minor changes Signed-off-by: michaelawyu --- pkg/examples/kubelogin/main.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/examples/kubelogin/main.go b/pkg/examples/kubelogin/main.go index ce47049..adfe716 100644 --- a/pkg/examples/kubelogin/main.go +++ b/pkg/examples/kubelogin/main.go @@ -13,6 +13,13 @@ import ( "sigs.k8s.io/cluster-inventory-api/pkg/credentials" ) +// The example below showcases how to use Azure's kubelogin exec plugin to sign into an +// AKS cluster using the workload identity method. +// +// As the method requires cluster-specific information such as tenant ID and client ID, +// the example also demonstrates how to pass in additional command-line arguments +// and/or environment variables to the exec plugin via the ClusterProfile API using the +// reserved extensions, as defined in KEP 5339. func main() { providers := []credentials.Provider{ { @@ -51,6 +58,11 @@ func main() { log.Fatalf("failed to marshal additional args") } + // The additional environment variables are also cluster-specific information. + // + // kubelogin accepts client ID input also in the form of a CLI argument; the example + // here uses the environment variable form just to showcase the different ways of passing + // in additional information. additionalEnvVars := map[string]string{ "AZURE_CLIENT_ID": "CLIENT_ID", }