From 952a3805aa542e88a94b2b87880553a979a334b6 Mon Sep 17 00:00:00 2001 From: wangke19 Date: Fri, 16 Jan 2026 19:37:57 +0800 Subject: [PATCH 1/5] test/e2e: migrate refresh-CA test for OTE compatibility Migrates the refresh-CA test to the OTE (Openshift Test Extended) Ginkgo framework while maintaining dual-compatibility with traditional Go tests. Changes: - Add Ginkgo test context in e2e.go for OTE test discovery: * refresh-CA - Tests CA regeneration when deleted and recreated - Extract test logic into shared function with testing.TB interface: * testRefreshCA() - Verifies serving certs and configmaps update when CA secret is deleted and recreated - Add helper functions in e2e.go: * pollForCABundleInjectionConfigMapWithReturn() - Polls for configmap * pollForCARecreation() - Polls for CA secret recreation - Keep test runner in e2e_test.go that calls shared function - Remove duplicate helper functions from e2e_test.go Net change: +30 lines (132 added - 102 removed) Both test frameworks continue to work: - Standard Go test: go test -run "^TestE2E$/^refresh-CA$" - OTE: Ginkgo test discovery via service-ca-operator-tests-ext --- test/e2e/e2e.go | 127 +++++++++++++++++++++++++++++++++++++++++++ test/e2e/e2e_test.go | 107 ++---------------------------------- 2 files changed, 132 insertions(+), 102 deletions(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index e6adf2e2b..cfd48a66d 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -103,6 +103,12 @@ var _ = g.Describe("[sig-service-ca] service-ca-operator", func() { testServiceCAMetrics(g.GinkgoTB()) }) }) + + g.Context("refresh-CA", func() { + g.It("[Operator][Serial] should regenerate serving certs and configmaps when CA is deleted and recreated", func() { + testRefreshCA(g.GinkgoTB()) + }) + }) }) // testServingCertAnnotation checks that services with the serving-cert annotation @@ -1220,3 +1226,124 @@ func getSampleForPromQueryGinkgo(t testing.TB, promClient prometheusv1.API, quer } return res[0], nil } + +// testRefreshCA verifies that when the CA secret is deleted and recreated, +// all serving certs and configmaps get updated with the new CA. +// +// This test uses testing.TB interface for dual-compatibility with both +// standard Go tests and Ginkgo tests. +// +// This situation is temporary until we test the new e2e jobs with OTE. +// Eventually all tests will be run only as part of the OTE framework. +func testRefreshCA(t testing.TB) { + adminClient, err := getKubeClient() + if err != nil { + t.Fatalf("error getting kube client: %v", err) + } + + ns, cleanup, err := createTestNamespace(t, adminClient, "test-"+randSeq(5)) + if err != nil { + t.Fatalf("could not create test namespace: %v", err) + } + defer cleanup() + + // create secrets + testServiceName := "test-service-" + randSeq(5) + testSecretName := "test-secret-" + randSeq(5) + testHeadlessServiceName := "test-headless-service-" + randSeq(5) + testHeadlessSecretName := "test-headless-secret-" + randSeq(5) + + err = createServingCertAnnotatedService(adminClient, testSecretName, testServiceName, ns.Name, false) + if err != nil { + t.Fatalf("error creating annotated service: %v", err) + } + if err = createServingCertAnnotatedService(adminClient, testHeadlessSecretName, testHeadlessServiceName, ns.Name, true); err != nil { + t.Fatalf("error creating annotated headless service: %v", err) + } + + secret, err := pollForServiceServingSecretWithReturn(adminClient, testSecretName, ns.Name) + if err != nil { + t.Fatalf("error fetching created serving cert secret: %v", err) + } + secretCopy := secret.DeepCopy() + headlessSecret, err := pollForServiceServingSecretWithReturn(adminClient, testHeadlessSecretName, ns.Name) + if err != nil { + t.Fatalf("error fetching created serving cert secret: %v", err) + } + headlessSecretCopy := headlessSecret.DeepCopy() + + // create configmap + testConfigMapName := "test-configmap-" + randSeq(5) + + err = createAnnotatedCABundleInjectionConfigMap(adminClient, testConfigMapName, ns.Name) + if err != nil { + t.Fatalf("error creating annotated configmap: %v", err) + } + + configmap, err := pollForCABundleInjectionConfigMapWithReturn(adminClient, testConfigMapName, ns.Name) + if err != nil { + t.Fatalf("error fetching ca bundle injection configmap: %v", err) + } + configmapCopy := configmap.DeepCopy() + err = checkConfigMapCABundleInjectionData(adminClient, testConfigMapName, ns.Name) + if err != nil { + t.Fatalf("error when checking ca bundle injection configmap: %v", err) + } + + // delete ca secret + err = adminClient.CoreV1().Secrets("openshift-service-ca").Delete(context.TODO(), "signing-key", metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("error deleting signing key: %v", err) + } + + // make sure it's recreated + err = pollForCARecreation(adminClient) + if err != nil { + t.Fatalf("signing key was not recreated: %v", err) + } + + err = pollForConfigMapChange(t, adminClient, configmapCopy, api.InjectionDataKey) + if err != nil { + t.Fatalf("configmap bundle did not change: %v", err) + } + + err = pollForSecretChangeGinkgo(t, adminClient, secretCopy, v1.TLSCertKey, v1.TLSPrivateKeyKey) + if err != nil { + t.Fatalf("secret cert did not change: %v", err) + } + if err := pollForSecretChangeGinkgo(t, adminClient, headlessSecretCopy); err != nil { + t.Fatalf("headless secret cert did not change: %v", err) + } +} + +// pollForCABundleInjectionConfigMapWithReturn polls for a CA bundle injection configmap and returns it. +func pollForCABundleInjectionConfigMapWithReturn(client *kubernetes.Clientset, configMapName, namespace string) (*v1.ConfigMap, error) { + var configmap *v1.ConfigMap + err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { + cm, err := client.CoreV1().ConfigMaps(namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + configmap = cm + return true, nil + }) + return configmap, err +} + +// pollForCARecreation polls for the signing secret to be re-created in +// response to CA secret deletion. +func pollForCARecreation(client *kubernetes.Clientset) error { + return wait.PollImmediate(time.Second, rotationPollTimeout, func() (bool, error) { + _, err := client.CoreV1().Secrets("openshift-service-ca").Get(context.TODO(), "signing-key", metav1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil + }) +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 4fe9d75e6..e51a6c6b3 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -93,22 +93,6 @@ func editServingSecretData(t *testing.T, client *kubernetes.Clientset, secretNam return pollForSecretChange(t, client, scopy, keyName) } -func pollForCABundleInjectionConfigMapWithReturn(client *kubernetes.Clientset, configMapName, namespace string) (*v1.ConfigMap, error) { - var configmap *v1.ConfigMap - err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { - cm, err := client.CoreV1().ConfigMaps(namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - return false, nil - } - if err != nil { - return false, err - } - configmap = cm - return true, nil - }) - return configmap, err -} - func pollForSecretChange(t *testing.T, client *kubernetes.Clientset, secret *v1.Secret, keysToChange ...string) error { return wait.PollImmediate(pollInterval, rotationPollTimeout, func() (bool, error) { s, err := client.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{}) @@ -352,19 +336,6 @@ func pollForCARotation(t *testing.T, client *kubernetes.Clientset, caCertPEM, ca // pollForCARecreation polls for the signing secret to be re-created in // response to CA secret deletion. -func pollForCARecreation(client *kubernetes.Clientset) error { - return wait.PollImmediate(time.Second, rotationPollTimeout, func() (bool, error) { - _, err := client.CoreV1().Secrets(serviceCAControllerNamespace).Get(context.TODO(), signingKeySecretName, metav1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - return false, nil - } - if err != nil { - return false, err - } - return true, nil - }) -} - // pollForUpdatedServingCert returns the cert and key PEM if it changes from // that provided before the polling timeout. @@ -723,80 +694,12 @@ func TestE2E(t *testing.T) { }) }) + // test CA refresh - delete and recreate CA secret, verify serving certs and configmaps are updated + // NOTE: This test is also available in the OTE framework (test/e2e/e2e.go). + // This duplication is temporary until we fully migrate to OTE and validate the new e2e jobs. + // Eventually, all tests will run only through the OTE framework. t.Run("refresh-CA", func(t *testing.T) { - ns, cleanup, err := createTestNamespace(t, adminClient, "test-"+randSeq(5)) - if err != nil { - t.Fatalf("could not create test namespace: %v", err) - } - defer cleanup() - - // create secrets - testServiceName := "test-service-" + randSeq(5) - testSecretName := "test-secret-" + randSeq(5) - testHeadlessServiceName := "test-headless-service-" + randSeq(5) - testHeadlessSecretName := "test-headless-secret-" + randSeq(5) - - err = createServingCertAnnotatedService(adminClient, testSecretName, testServiceName, ns.Name, false) - if err != nil { - t.Fatalf("error creating annotated service: %v", err) - } - if err = createServingCertAnnotatedService(adminClient, testHeadlessSecretName, testHeadlessServiceName, ns.Name, true); err != nil { - t.Fatalf("error creating annotated headless service: %v", err) - } - - secret, err := pollForServiceServingSecretWithReturn(adminClient, testSecretName, ns.Name) - if err != nil { - t.Fatalf("error fetching created serving cert secret: %v", err) - } - secretCopy := secret.DeepCopy() - headlessSecret, err := pollForServiceServingSecretWithReturn(adminClient, testHeadlessSecretName, ns.Name) - if err != nil { - t.Fatalf("error fetching created serving cert secret: %v", err) - } - headlessSecretCopy := headlessSecret.DeepCopy() - - // create configmap - testConfigMapName := "test-configmap-" + randSeq(5) - - err = createAnnotatedCABundleInjectionConfigMap(adminClient, testConfigMapName, ns.Name) - if err != nil { - t.Fatalf("error creating annotated configmap: %v", err) - } - - configmap, err := pollForCABundleInjectionConfigMapWithReturn(adminClient, testConfigMapName, ns.Name) - if err != nil { - t.Fatalf("error fetching ca bundle injection configmap: %v", err) - } - configmapCopy := configmap.DeepCopy() - err = checkConfigMapCABundleInjectionData(adminClient, testConfigMapName, ns.Name) - if err != nil { - t.Fatalf("error when checking ca bundle injection configmap: %v", err) - } - - // delete ca secret - err = adminClient.CoreV1().Secrets(serviceCAControllerNamespace).Delete(context.TODO(), signingKeySecretName, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("error deleting signing key: %v", err) - } - - // make sure it's recreated - err = pollForCARecreation(adminClient) - if err != nil { - t.Fatalf("signing key was not recreated: %v", err) - } - - err = pollForConfigMapChange(t, adminClient, configmapCopy, api.InjectionDataKey) - if err != nil { - t.Fatalf("configmap bundle did not change: %v", err) - } - - err = pollForSecretChange(t, adminClient, secretCopy, v1.TLSCertKey, v1.TLSPrivateKeyKey) - if err != nil { - t.Fatalf("secret cert did not change: %v", err) - } - if err := pollForSecretChange(t, adminClient, headlessSecretCopy); err != nil { - t.Fatalf("headless secret cert did not change: %v", err) - } + testRefreshCA(t) }) // This test triggers rotation by updating the CA to have an From 830b8ebb3de6458a8306c78b3f15ab35d36d5151 Mon Sep 17 00:00:00 2001 From: wangke19 Date: Tue, 20 Jan 2026 18:54:30 +0800 Subject: [PATCH 2/5] test/e2e: fix Ginkgo context execution order Move headless-stateful-serving-cert-secret-delete-data context to correct position. It should run after serving-cert-secret-delete-data and before ca-bundle tests. This fixes the test execution sequence to match e2e_test.go order: 1. serving-cert-secret-delete-data 2. headless-stateful-serving-cert-secret-delete-data 3. ca-bundle-injection-configmap 4. ca-bundle-injection-configmap-update 5. vulnerable-legacy-ca-bundle-injection-configmap 6. metrics 7. refresh-CA --- test/e2e/e2e.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index cfd48a66d..5a51915af 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -70,6 +70,12 @@ var _ = g.Describe("[sig-service-ca] service-ca-operator", func() { }) }) + g.Context("headless-stateful-serving-cert-secret-delete-data", func() { + g.It("[Operator][Serial] should regenerate deleted serving cert secrets for StatefulSet with headless service", func() { + testHeadlessStatefulServingCertSecretDeleteData(g.GinkgoTB()) + }) + }) + g.Context("ca-bundle-injection-configmap", func() { g.It("[Operator][Serial] should inject CA bundle into annotated configmaps", func() { testCABundleInjectionConfigMap(g.GinkgoTB()) @@ -88,12 +94,6 @@ var _ = g.Describe("[sig-service-ca] service-ca-operator", func() { }) }) - g.Context("headless-stateful-serving-cert-secret-delete-data", func() { - g.It("[Operator][Serial] should regenerate deleted serving cert secrets for StatefulSet with headless service", func() { - testHeadlessStatefulServingCertSecretDeleteData(g.GinkgoTB()) - }) - }) - g.Context("metrics", func() { g.It("[Operator][Serial] should collect metrics from the operator", func() { testMetricsCollection(g.GinkgoTB()) From a2e78755a93d636c172bb8dea8e7efd88f8ceb60 Mon Sep 17 00:00:00 2001 From: wangke19 Date: Tue, 20 Jan 2026 20:24:15 +0800 Subject: [PATCH 3/5] test/e2e: add g.Ordered to enforce sequential test execution Add g.Ordered decorator to Ginkgo Describe block to guarantee tests run in declaration order. Without g.Ordered, Ginkgo randomizes test execution order by default, even with Parallelism: 1. This causes OTE tests to fail because the tests have state dependencies and must run in the exact order defined in e2e_test.go. The g.Ordered decorator ensures tests execute in this sequence: 1. serving-cert-annotation 2. serving-cert-secret-modify-bad-tlsCert 3. serving-cert-secret-add-data 4. serving-cert-secret-delete-data 5. headless-stateful-serving-cert-secret-delete-data 6. ca-bundle-injection-configmap 7. ca-bundle-injection-configmap-update 8. vulnerable-legacy-ca-bundle-injection-configmap 9. metrics 10. refresh-CA This matches the execution order in traditional Go tests. --- test/e2e/e2e.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 5a51915af..c13040ce4 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -39,7 +39,7 @@ const ( var characters = []rune("abcdefghijklmnopqrstuvwxyz0123456789") -var _ = g.Describe("[sig-service-ca] service-ca-operator", func() { +var _ = g.Describe("[sig-service-ca] service-ca-operator", g.Ordered, func() { g.Context("serving-cert-annotation", func() { for _, headless := range []bool{false, true} { g.It(fmt.Sprintf("[Operator][Serial] should provision certificates for services with headless=%v", headless), func() { From e60fe227d600432d08ac41397e22b5ff1b0a18ba Mon Sep 17 00:00:00 2001 From: Ke Wang Date: Wed, 11 Feb 2026 16:29:30 +0800 Subject: [PATCH 4/5] test/e2e: split OTE suite into stable and disruptive for refresh-CA The CI job e2e-aws-operator-serial-ote fails because the refresh-CA test deletes the cluster service CA signing key, causing cluster-wide TLS disruption that OTE monitors detect as failures. Split the single suite into a Stable suite (strict monitoring) and a Disruptive suite (relaxed thresholds) so monitors adjust expectations appropriately. Co-Authored-By: Claude Opus 4.6 --- cmd/service-ca-operator-tests-ext/main.go | 22 ++++++++++++++++------ test/e2e/e2e.go | 2 +- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/cmd/service-ca-operator-tests-ext/main.go b/cmd/service-ca-operator-tests-ext/main.go index 56eaa2440..ad6aba154 100644 --- a/cmd/service-ca-operator-tests-ext/main.go +++ b/cmd/service-ca-operator-tests-ext/main.go @@ -69,14 +69,24 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) { registry := oteextension.NewRegistry() extension := oteextension.NewExtension("openshift", "payload", "service-ca-operator") - // The following suite runs tests that verify the operator's behaviour. - // This suite is executed only on pull requests targeting this repository. - // Tests tagged with both [Operator] and [Serial] are included in this suite. + // Non-disruptive tests run with strict cluster health monitoring. extension.AddSuite(oteextension.Suite{ - Name: "openshift/service-ca-operator/operator/serial", - Parallelism: 1, + Name: "openshift/service-ca-operator/operator/serial", + Parallelism: 1, + ClusterStability: oteextension.ClusterStabilityStable, Qualifiers: []string{ - `name.contains("[Operator]") && name.contains("[Serial]")`, + `name.contains("[Operator]") && name.contains("[Serial]") && !name.contains("[Disruptive]")`, + }, + }) + + // Disruptive tests (e.g. CA rotation) that cause expected cluster-wide TLS disruption. + // Monitors will relax thresholds for this suite. + extension.AddSuite(oteextension.Suite{ + Name: "openshift/service-ca-operator/operator/serial-disruptive", + Parallelism: 1, + ClusterStability: oteextension.ClusterStabilityDisruptive, + Qualifiers: []string{ + `name.contains("[Operator]") && name.contains("[Serial]") && name.contains("[Disruptive]")`, }, }) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index c13040ce4..e1bf86cd9 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -105,7 +105,7 @@ var _ = g.Describe("[sig-service-ca] service-ca-operator", g.Ordered, func() { }) g.Context("refresh-CA", func() { - g.It("[Operator][Serial] should regenerate serving certs and configmaps when CA is deleted and recreated", func() { + g.It("[Operator][Serial][Disruptive] should regenerate serving certs and configmaps when CA is deleted and recreated", func() { testRefreshCA(g.GinkgoTB()) }) }) From 7b80d1f086612d1e1183b61a71498a7b8641ce03 Mon Sep 17 00:00:00 2001 From: wangke19 Date: Fri, 13 Feb 2026 22:05:14 +0800 Subject: [PATCH 5/5] test/e2e: address review feedback for OTE suite split - Remove g.Ordered decorator since OTE overrides it in CI and go test does not run Ginkgo specs - Remove explicit ClusterStabilityStable as it is the default when ClusterStability is unset --- cmd/service-ca-operator-tests-ext/main.go | 7 +++---- test/e2e/e2e.go | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/service-ca-operator-tests-ext/main.go b/cmd/service-ca-operator-tests-ext/main.go index ad6aba154..cee38eede 100644 --- a/cmd/service-ca-operator-tests-ext/main.go +++ b/cmd/service-ca-operator-tests-ext/main.go @@ -69,11 +69,10 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) { registry := oteextension.NewRegistry() extension := oteextension.NewExtension("openshift", "payload", "service-ca-operator") - // Non-disruptive tests run with strict cluster health monitoring. + // Non-disruptive tests run with default (Stable) cluster health monitoring. extension.AddSuite(oteextension.Suite{ - Name: "openshift/service-ca-operator/operator/serial", - Parallelism: 1, - ClusterStability: oteextension.ClusterStabilityStable, + Name: "openshift/service-ca-operator/operator/serial", + Parallelism: 1, Qualifiers: []string{ `name.contains("[Operator]") && name.contains("[Serial]") && !name.contains("[Disruptive]")`, }, diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index e1bf86cd9..4ca89965f 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -39,7 +39,7 @@ const ( var characters = []rune("abcdefghijklmnopqrstuvwxyz0123456789") -var _ = g.Describe("[sig-service-ca] service-ca-operator", g.Ordered, func() { +var _ = g.Describe("[sig-service-ca] service-ca-operator", func() { g.Context("serving-cert-annotation", func() { for _, headless := range []bool{false, true} { g.It(fmt.Sprintf("[Operator][Serial] should provision certificates for services with headless=%v", headless), func() {