From f588f82da6ce7fa5dd68658deea1f26607be91f1 Mon Sep 17 00:00:00 2001 From: Marco Nenciarini Date: Fri, 16 May 2025 17:07:05 +0200 Subject: [PATCH] Add support for injecting CA certificates into Secrets Closes #264 Signed-off-by: Marco Nenciarini --- README.md | 6 +- pkg/controller/cabundleinjector/secret.go | 81 ++++++++++++++ pkg/controller/cabundleinjector/starter.go | 1 + test/e2e/e2e_test.go | 121 +++++++++++++++++++++ 4 files changed, 206 insertions(+), 3 deletions(-) create mode 100644 pkg/controller/cabundleinjector/secret.go diff --git a/README.md b/README.md index 2351c6b16..e4d288d67 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ This operator runs the following OpenShift controllers: * **serving cert signer:** * Issues a signed serving certificate/key pair to services annotated with 'service.beta.openshift.io/serving-cert-secret-name' via a secret. [See the current OKD documentation for usage.](https://docs.okd.io/latest/security/certificates/service-serving-certificate.html) -* **configmap cabundle injector:** - * Watches for configmaps annotated with 'service.beta.openshift.io/inject-cabundle=true' and adds or updates a data item (key "service-ca.crt") containing the PEM-encoded CA signing bundle. Consumers of the configmap can then trust service-ca.crt in their TLS client configuration, allowing connections to services that utilize service-serving certificates. - * Note: Explicitly referencing the "service-ca.crt" key in a volumeMount will prevent a pod from starting until the configMap has been injected with the CA bundle (https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#restrictions). This behavior helps ensure that pods start with the CA bundle data available. +* **configmap/secret cabundle injector:** + * Watches for configmaps and secrets annotated with 'service.beta.openshift.io/inject-cabundle=true' and adds or updates a data item (key "service-ca.crt") containing the PEM-encoded CA signing bundle. Consumers of the configmap/secret can then trust service-ca.crt in their TLS client configuration, allowing connections to services that utilize service-serving certificates. + * Note: Explicitly referencing the "service-ca.crt" key in a volumeMount will prevent a pod from starting until the configMap/secret has been injected with the CA bundle (https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#restrictions). This behavior helps ensure that pods start with the CA bundle data available. ``` $ oc create configmap foobar --from-literal=key1=foo diff --git a/pkg/controller/cabundleinjector/secret.go b/pkg/controller/cabundleinjector/secret.go new file mode 100644 index 000000000..a6270a3a7 --- /dev/null +++ b/pkg/controller/cabundleinjector/secret.go @@ -0,0 +1,81 @@ +package cabundleinjector + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kcoreclient "k8s.io/client-go/kubernetes/typed/core/v1" + listers "k8s.io/client-go/listers/core/v1" + "k8s.io/klog/v2" + + apiannotations "github.com/openshift/api/annotations" + "github.com/openshift/library-go/pkg/controller/factory" + "github.com/openshift/service-ca-operator/pkg/controller/api" +) + +type secretCABundleInjector struct { + client kcoreclient.SecretsGetter + lister listers.SecretLister + caBundle string + + filterFn func(secret *corev1.Secret) bool +} + +func newSecretInjectorConfig(config *caBundleInjectorConfig) controllerConfig { + informer := config.kubeInformers.Core().V1().Secrets() + + syncer := &secretCABundleInjector{ + client: config.kubeClient.CoreV1(), + lister: informer.Lister(), + caBundle: string(config.caBundle), + } + + return controllerConfig{ + name: "SecretCABundleInjector", + sync: syncer.Sync, + informer: informer.Informer(), + annotationsChecker: annotationsChecker( + api.InjectCABundleAnnotationName, + ), + namespaced: true, + } +} + +func (bi *secretCABundleInjector) Sync(ctx context.Context, syncCtx factory.SyncContext) error { + namespace, name := namespacedObjectFromQueueKey(syncCtx.QueueKey()) + + secret, err := bi.lister.Secrets(namespace).Get(name) + if apierrors.IsNotFound(err) { + return nil + } else if err != nil { + return err + } + + if bi.filterFn != nil && !bi.filterFn(secret) { + return nil + } + + // skip updating when the CA bundle is already there + if data, ok := secret.Data[api.InjectionDataKey]; ok && + string(data) == bi.caBundle && len(secret.Data) == 1 { + + return nil + } + + klog.Infof("updating secret %s/%s with the service signing CA bundle", secret.Namespace, secret.Name) + + // make a copy to avoid mutating cache state + secretCopy := secret.DeepCopy() + secretCopy.Data = map[string][]byte{api.InjectionDataKey: []byte(bi.caBundle)} + // set the owning-component unless someone else has claimed it. + if len(secretCopy.Annotations[apiannotations.OpenShiftComponent]) == 0 { + secretCopy.Annotations[apiannotations.OpenShiftComponent] = api.OwningJiraComponent + secretCopy.Annotations[apiannotations.OpenShiftDescription] = fmt.Sprintf("Secret is added/updated with a data item containing the CA signing bundle that can be used to verify service-serving certificates") + } + + _, err = bi.client.Secrets(secretCopy.Namespace).Update(ctx, secretCopy, metav1.UpdateOptions{}) + return err +} diff --git a/pkg/controller/cabundleinjector/starter.go b/pkg/controller/cabundleinjector/starter.go index 0112101ee..a5b2b9a88 100644 --- a/pkg/controller/cabundleinjector/starter.go +++ b/pkg/controller/cabundleinjector/starter.go @@ -89,6 +89,7 @@ func StartCABundleInjector(ctx context.Context, controllerContext *controllercmd configConstructors := []configBuilderFunc{ newAPIServiceInjectorConfig, newConfigMapInjectorConfig, + newSecretInjectorConfig, newCRDInjectorConfig, newMutatingWebhookInjectorConfig, newValidatingWebhookInjectorConfig, diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 2fb1553ca..a5d8b95bb 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -210,6 +210,18 @@ func createAnnotatedCABundleInjectionConfigMap(client *kubernetes.Clientset, con return err } +func createAnnotatedCABundleInjectionSecret(client *kubernetes.Clientset, secretName, namespace string) error { + obj := &v1.Secret{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + }, + } + setInjectionAnnotation(&obj.ObjectMeta) + _, err := client.CoreV1().Secrets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + return err +} + func pollForServiceServingSecret(client *kubernetes.Clientset, secretName, namespace string) error { return wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { _, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) @@ -236,6 +248,19 @@ func pollForCABundleInjectionConfigMap(client *kubernetes.Clientset, configMapNa }) } +func pollForCABundleInjectionSecret(client *kubernetes.Clientset, secretName, namespace string) error { + return wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { + _, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil + }) +} + func editServingSecretData(t *testing.T, client *kubernetes.Clientset, secretName, namespace, keyName string) error { sss, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) if err != nil { @@ -269,6 +294,24 @@ func editConfigMapCABundleInjectionData(t *testing.T, client *kubernetes.Clients return pollForConfigMapChange(t, client, cmcopy, "foo") } +func editSecretCABundleInjectionData(t *testing.T, client *kubernetes.Clientset, secretName, namespace string) error { + secret, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + return err + } + secretCopy := secret.DeepCopy() + if len(secretCopy.Data) != 1 { + return fmt.Errorf("ca bundle injection secret missing data") + } + secretCopy.Data["foo"] = []byte("blah") + _, err = client.CoreV1().Secrets(namespace).Update(context.TODO(), secretCopy, metav1.UpdateOptions{}) + if err != nil { + return err + } + + return pollForSecretChange(t, client, secretCopy, "foo") +} + func checkServiceServingCertSecretData(client *kubernetes.Clientset, secretName, namespace string) ([]byte, bool, error) { sss, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) if err != nil { @@ -312,6 +355,22 @@ func checkConfigMapCABundleInjectionData(client *kubernetes.Clientset, configMap return nil } +func checkSecretCABundleInjectionData(client *kubernetes.Clientset, secretName, namespace string) error { + cm, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + return err + } + if len(cm.Data) != 1 { + return fmt.Errorf("unexpected ca bundle injection secret data map length: %v", len(cm.Data)) + } + ok := true + _, ok = cm.Data[api.InjectionDataKey] + if !ok { + return fmt.Errorf("unexpected ca bundle injection secret data: %v", cm.Data) + } + return nil +} + func pollForConfigMapCAInjection(client *kubernetes.Clientset, configMapName, namespace string) error { return wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { cm, err := client.CoreV1().ConfigMaps(namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) @@ -1436,6 +1495,68 @@ func TestE2E(t *testing.T) { } }) + // test ca bundle injection secret + t.Run("ca-bundle-injection-secret", func(t *testing.T) { + ns, cleanup, err := createTestNamespace(t, adminClient, "test-"+randSeq(5)) + if err != nil { + t.Fatalf("could not create test namespace: %v", err) + } + defer cleanup() + + testSecretName := "test-secret-" + randSeq(5) + + err = createAnnotatedCABundleInjectionSecret(adminClient, testSecretName, ns.Name) + if err != nil { + t.Fatalf("error creating annotated secret: %v", err) + } + + err = pollForCABundleInjectionSecret(adminClient, testSecretName, ns.Name) + if err != nil { + t.Fatalf("error fetching ca bundle injection secret: %v", err) + } + + err = checkSecretCABundleInjectionData(adminClient, testSecretName, ns.Name) + if err != nil { + t.Fatalf("error when checking ca bundle injection secret: %v", err) + } + }) + + // test updated data in ca bundle injection secret will be stomped on + t.Run("ca-bundle-injection-secret-update", func(t *testing.T) { + ns, cleanup, err := createTestNamespace(t, adminClient, "test-"+randSeq(5)) + if err != nil { + t.Fatalf("could not create test namespace: %v", err) + } + defer cleanup() + + testSecretName := "test-secret-" + randSeq(5) + + err = createAnnotatedCABundleInjectionSecret(adminClient, testSecretName, ns.Name) + if err != nil { + t.Fatalf("error creating annotated secret: %v", err) + } + + err = pollForCABundleInjectionSecret(adminClient, testSecretName, ns.Name) + if err != nil { + t.Fatalf("error fetching ca bundle injection secret: %v", err) + } + + err = checkSecretCABundleInjectionData(adminClient, testSecretName, ns.Name) + if err != nil { + t.Fatalf("error when checking ca bundle injection secret: %v", err) + } + + err = editSecretCABundleInjectionData(t, adminClient, testSecretName, ns.Name) + if err != nil { + t.Fatalf("error editing ca bundle injection secret: %v", err) + } + + err = checkSecretCABundleInjectionData(adminClient, testSecretName, ns.Name) + if err != nil { + t.Fatalf("error when checking ca bundle injection secret: %v", err) + } + }) + t.Run("metrics", func(t *testing.T) { promClient, err := newPrometheusClientForConfig(adminConfig) if err != nil {