From ac1708e5975ddb4ed270d34b3bd8c220947ad802 Mon Sep 17 00:00:00 2001 From: kahirokunn Date: Wed, 27 Aug 2025 15:19:37 +0900 Subject: [PATCH] Add BuildConfigFromCP aligning with ClusterInventory Consumer "Push Model via Credentials in Secret" (KEP-4322) Signed-off-by: kahirokunn --- pkg/secrets/config.go | 123 ++++++++++++++++++++++++ pkg/secrets/config_test.go | 186 +++++++++++++++++++++++++++++++++++++ 2 files changed, 309 insertions(+) create mode 100644 pkg/secrets/config.go create mode 100644 pkg/secrets/config_test.go diff --git a/pkg/secrets/config.go b/pkg/secrets/config.go new file mode 100644 index 0000000..f1130ad --- /dev/null +++ b/pkg/secrets/config.go @@ -0,0 +1,123 @@ +package secrets + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/cluster-inventory-api/apis/v1alpha1" +) + +const ( + // Label key that must be set on the namespace hosting the secret + LabelConsumerKey = "x-k8s.io/cluster-inventory-consumer" + + // Label key that must be set on the secret to point to the ClusterProfile name + LabelClusterProfileName = "x-k8s.io/cluster-profile" + + // Optional label key on the secret to indicate the ClusterProfile namespace + // If absent, the ClusterProfile is assumed to be in the "default" namespace + LabelClusterProfileNamespace = "x-k8s.io/cluster-profile-namespace" + + // Data key within the secret that stores kubeconfig content + // Note: The KEP refers to this field as "Config" semantically, + // however, the conventional secret data key is lowercase "config" + // and is what client-go helpers expect in examples and tests. + SecretDataKeyKubeconfig = "config" +) + +// BuildConfigFromCP discovers a Secret that contains kubeconfig based on: +// - Namespace label: the hosting namespace MUST have label LabelConsumerKey= +// - Secret labels: +// - MUST have label LabelClusterProfileName= +// - MAY have label LabelClusterProfileNamespace= +// If absent, the ClusterProfile is assumed to be in namespace "default". +// The kubeconfig must be stored under the Secret data key "config". +// Returns a *rest.Config if exactly one matching secret is found; otherwise returns an error. +// Spec reference (KEP-4322 Secret format): +// https://github.com/kubernetes/enhancements/blob/master/keps/sig-multicluster/4322-cluster-inventory/README.md#secret-format +func BuildConfigFromCP( + ctx context.Context, + kubeClient kubernetes.Interface, + consumerName string, + clusterProfile *v1alpha1.ClusterProfile, +) (*rest.Config, error) { + if kubeClient == nil { + return nil, fmt.Errorf("kubeClient must not be nil") + } + if clusterProfile == nil { + return nil, fmt.Errorf("clusterProfile must not be nil") + } + if consumerName == "" { + return nil, fmt.Errorf("consumerName must be provided") + } + + cpName := clusterProfile.Name + if cpName == "" { + return nil, fmt.Errorf("clusterProfile.Name must be provided") + } + cpNamespace := clusterProfile.Namespace + if cpNamespace == "" { + cpNamespace = corev1.NamespaceDefault + } + + // 1) Find all namespaces for this consumer + nsSelector := labels.Set{LabelConsumerKey: consumerName}.AsSelector().String() + nsList, err := kubeClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{LabelSelector: nsSelector}) + if err != nil { + return nil, fmt.Errorf("failed to list namespaces for consumer %q: %w", consumerName, err) + } + if len(nsList.Items) == 0 { + return nil, fmt.Errorf("no namespaces found labeled %q=%q", LabelConsumerKey, consumerName) + } + + // 2) For each namespace, find secrets labeled for this ClusterProfile + var candidates []corev1.Secret + secSelector := labels.Set{LabelClusterProfileName: cpName}.AsSelector().String() + for _, ns := range nsList.Items { + sList, err := kubeClient.CoreV1().Secrets(ns.Name).List(ctx, metav1.ListOptions{LabelSelector: secSelector}) + if err != nil { + return nil, fmt.Errorf("failed to list secrets in namespace %s: %w", ns.Name, err) + } + for _, s := range sList.Items { + // If the secret specifies a CP namespace label, it must match cpNamespace. + // If it does not specify, it is considered to target the "default" namespace only. + labeledNs, hasNsLabel := s.Labels[LabelClusterProfileNamespace] + if hasNsLabel { + if labeledNs == cpNamespace { + candidates = append(candidates, s) + } + continue + } + // No namespace label present. Only accept if CP is in default namespace. + if cpNamespace == corev1.NamespaceDefault { + candidates = append(candidates, s) + } + } + } + + if len(candidates) == 0 { + return nil, fmt.Errorf("no secret found for ClusterProfile %s/%s for consumer %q", cpNamespace, cpName, consumerName) + } + if len(candidates) > 1 { + return nil, fmt.Errorf("multiple secrets found for ClusterProfile %s/%s for consumer %q; expected exactly one", cpNamespace, cpName, consumerName) + } + + // 3) Build rest.Config from the single matching secret + selected := candidates[0] + raw, ok := selected.Data[SecretDataKeyKubeconfig] + if !ok || len(raw) == 0 { + return nil, fmt.Errorf("secret %s/%s does not contain kubeconfig under key %q", selected.Namespace, selected.Name, SecretDataKeyKubeconfig) + } + restCfg, err := clientcmd.RESTConfigFromKubeConfig(raw) + if err != nil { + return nil, fmt.Errorf("failed to build rest.Config from kubeconfig in secret %s/%s: %w", selected.Namespace, selected.Name, err) + } + return restCfg, nil +} + diff --git a/pkg/secrets/config_test.go b/pkg/secrets/config_test.go new file mode 100644 index 0000000..211f1fc --- /dev/null +++ b/pkg/secrets/config_test.go @@ -0,0 +1,186 @@ +package secrets + +import ( + "context" + "testing" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/cluster-inventory-api/apis/v1alpha1" +) + +func TestSecrets(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, "Secrets Package Suite") +} + +var _ = ginkgo.Describe("BuildConfigFromCP", func() { + var ( + ctx context.Context + clientset *fake.Clientset + consumerName string + namespaceName string + cpName string + cpNamespace string + ) + + ginkgo.BeforeEach(func() { + ctx = context.TODO() + clientset = fake.NewSimpleClientset() + consumerName = "my-consumer" + namespaceName = "ci-consumer-ns" + cpName = "workload-cluster" + cpNamespace = "default" + + // Create consumer namespace with required label + _, err := clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + Labels: map[string]string{ + LabelConsumerKey: consumerName, + }, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Create secret containing kubeconfig and labels for ClusterProfile + kubeconfig := []byte(`apiVersion: v1 +kind: Config +clusters: +- cluster: + server: https://example.cluster.local + name: c1 +contexts: +- context: + cluster: c1 + user: u1 + name: ctx1 +current-context: ctx1 +users: +- name: u1 + user: + token: abcdefg +`) + _, err = clientset.CoreV1().Secrets(namespaceName).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-credentials", + Namespace: namespaceName, + Labels: map[string]string{ + LabelClusterProfileName: cpName, + LabelClusterProfileNamespace: cpNamespace, + }, + }, + Data: map[string][]byte{ + SecretDataKeyKubeconfig: kubeconfig, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("should discover secret by labels and build rest.Config successfully", func() { + cp := &v1alpha1.ClusterProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: cpName, + Namespace: cpNamespace, + }, + } + cfg, err := BuildConfigFromCP(ctx, clientset, consumerName, cp) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(cfg).NotTo(gomega.BeNil()) + gomega.Expect(cfg.Host).To(gomega.Equal("https://example.cluster.local")) + }) +}) + +var _ = ginkgo.Describe("BuildConfigFromCP - error handling", func() { + ginkgo.It("should return error when kubeClient is nil", func() { + cp := &v1alpha1.ClusterProfile{ObjectMeta: metav1.ObjectMeta{Name: "cp"}} + cfg, err := BuildConfigFromCP(context.TODO(), nil, "consumer", cp) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("kubeClient must not be nil")) + gomega.Expect(cfg).To(gomega.BeNil()) + }) + + ginkgo.It("should return error when consumerName is empty", func() { + clientset := fake.NewSimpleClientset() + cp := &v1alpha1.ClusterProfile{ObjectMeta: metav1.ObjectMeta{Name: "cp"}} + _, err := BuildConfigFromCP(context.TODO(), clientset, "", cp) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("consumerName must be provided")) + }) + + ginkgo.It("should return error when clusterProfile is nil", func() { + clientset := fake.NewSimpleClientset() + _, err := BuildConfigFromCP(context.TODO(), clientset, "consumer", nil) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("clusterProfile must not be nil")) + }) + + ginkgo.It("should return error when clusterProfile.Name is empty", func() { + clientset := fake.NewSimpleClientset() + cp := &v1alpha1.ClusterProfile{} + _, err := BuildConfigFromCP(context.TODO(), clientset, "consumer", cp) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("clusterProfile.Name must be provided")) + }) + + ginkgo.It("should return error when no matching secret exists", func() { + ctx := context.TODO() + clientset := fake.NewSimpleClientset() + // consumer namespace exists and labeled, but no secrets + _, err := clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns", Labels: map[string]string{LabelConsumerKey: "consumer"}}}, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + cp := &v1alpha1.ClusterProfile{ObjectMeta: metav1.ObjectMeta{Name: "cp", Namespace: "default"}} + _, err = BuildConfigFromCP(ctx, clientset, "consumer", cp) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no secret found for ClusterProfile")) + }) + + ginkgo.It("should return error when matching secret lacks kubeconfig data", func() { + ctx := context.TODO() + clientset := fake.NewSimpleClientset() + _, err := clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns", Labels: map[string]string{LabelConsumerKey: "consumer"}}}, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + _, err = clientset.CoreV1().Secrets("ns").Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "s", + Namespace: "ns", + Labels: map[string]string{ + LabelClusterProfileName: "cp", + }, + }, + Data: map[string][]byte{}, + }, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + cp := &v1alpha1.ClusterProfile{ObjectMeta: metav1.ObjectMeta{Name: "cp", Namespace: "default"}} + _, err = BuildConfigFromCP(ctx, clientset, "consumer", cp) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("does not contain kubeconfig under key")) + }) + + ginkgo.It("should return error when kubeconfig is invalid", func() { + ctx := context.TODO() + clientset := fake.NewSimpleClientset() + _, err := clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns", Labels: map[string]string{LabelConsumerKey: "consumer"}}}, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + _, err = clientset.CoreV1().Secrets("ns").Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "s", + Namespace: "ns", + Labels: map[string]string{ + LabelClusterProfileName: "cp", + }, + }, + Data: map[string][]byte{ + SecretDataKeyKubeconfig: []byte("not a valid kubeconfig"), + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + cp := &v1alpha1.ClusterProfile{ObjectMeta: metav1.ObjectMeta{Name: "cp", Namespace: "default"}} + _, err = BuildConfigFromCP(ctx, clientset, "consumer", cp) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to build rest.Config")) + }) +})