diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index e6adf2e2b..003fbca99 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -24,11 +24,17 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "k8s.io/utils/clock" "k8s.io/utils/ptr" routeclient "github.com/openshift/client-go/route/clientset/versioned" + "github.com/openshift/library-go/pkg/crypto" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/openshift/library-go/test/library/metrics" "github.com/openshift/service-ca-operator/pkg/controller/api" + "github.com/openshift/service-ca-operator/pkg/operator" + "github.com/openshift/service-ca-operator/pkg/operator/operatorclient" "github.com/openshift/service-ca-operator/test/util" ) @@ -103,6 +109,12 @@ var _ = g.Describe("[sig-service-ca] service-ca-operator", func() { testServiceCAMetrics(g.GinkgoTB()) }) }) + + g.Context("time-based-ca-rotation", func() { + g.It("[Operator][Serial] should rotate CA based on expiry time", func() { + testTimeBasedCARotation(g.GinkgoTB()) + }) + }) }) // testServingCertAnnotation checks that services with the serving-cert annotation @@ -437,7 +449,7 @@ func testServingCertSecretDeleteData(t testing.TB) { if err != nil { t.Fatalf("deleting secret %s in namespace %s failed: %v", secretName, operatorNamespace, err) } - updatedBytes, _, err := pollForUpdatedServingCertGinkgo(t, adminClient, operatorNamespace, secretName, rotationTimeout, nil, nil) + updatedBytes, _, err := pollForUpdatedServingCert(t, adminClient, operatorNamespace, secretName, rotationTimeout, nil, nil) if err != nil { t.Fatalf("error fetching re-created serving cert secret: %v", err) } @@ -446,44 +458,7 @@ func testServingCertSecretDeleteData(t testing.TB) { checkClientPodRcvdUpdatedServerCertGinkgo(t, adminClient, ns.Name, metricsHost, service.Spec.Ports[0].Port, string(updatedBytes)) } -// pollForUpdatedServingCertGinkgo returns the cert and key for the targeted secret -// if the values change from those provided before the polling timeout. -// This version accepts testing.TB for dual compatibility. -func pollForUpdatedServingCertGinkgo(t testing.TB, client *kubernetes.Clientset, namespace, name string, timeout time.Duration, oldCertValue, oldKeyValue []byte) ([]byte, []byte, error) { - secret, err := pollForUpdatedSecretGinkgo(t, client, namespace, name, timeout, map[string][]byte{ - v1.TLSCertKey: oldCertValue, - v1.TLSPrivateKeyKey: oldKeyValue, - }) - if err != nil { - return nil, nil, err - } - return secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey], nil -} - -// pollForUpdatedSecretGinkgo returns the given secret if its data changes from -// that provided before the polling timeout. -// This version accepts testing.TB for dual compatibility. -func pollForUpdatedSecretGinkgo(t testing.TB, client *kubernetes.Clientset, namespace, name string, timeout time.Duration, oldData map[string][]byte) (*v1.Secret, error) { - resourceID := fmt.Sprintf("Secret \"%s/%s\"", namespace, name) - obj, err := pollForResourceGinkgo(t, resourceID, timeout, func() (kruntime.Object, error) { - secret, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - err = util.CheckData(oldData, secret.Data) - if err != nil { - return nil, err - } - return secret, nil - }) - if err != nil { - return nil, err - } - return obj.(*v1.Secret), nil -} - // pollForResourceGinkgo polls for a resource using the provided accessor function. -// This version accepts testing.TB for dual compatibility. func pollForResourceGinkgo(t testing.TB, resourceID string, timeout time.Duration, accessor func() (kruntime.Object, error)) (kruntime.Object, error) { var obj kruntime.Object err := wait.PollImmediate(pollInterval, timeout, func() (bool, error) { @@ -1220,3 +1195,215 @@ func getSampleForPromQueryGinkgo(t testing.TB, promClient prometheusv1.API, quer } return res[0], nil } + +// testTimeBasedCARotation tests that the CA is rotated when it expires sooner +// than the minimum required duration. Uses testing.TB for dual-compatibility. +func testTimeBasedCARotation(t testing.TB) { + adminClient, adminConfig, err := getKubeClientAndConfig() + if err != nil { + t.Fatalf("error getting kube client: %v", err) + } + + checkCARotationGinkgo(t, adminClient, adminConfig, triggerTimeBasedRotationGinkgo) +} + +// checkCARotationGinkgo validates CA rotation for both refreshed and unrefreshed clients. +func checkCARotationGinkgo(t testing.TB, client *kubernetes.Clientset, config *rest.Config, triggerRotation triggerRotationFuncGinkgo) { + ns, cleanup, err := createTestNamespace(t, client, "test-"+randSeq(5)) + if err != nil { + t.Fatalf("could not create test namespace: %v", err) + } + defer cleanup() + + // Prompt the creation of service cert secrets + testServiceName := "test-service-" + randSeq(5) + testSecretName := "test-secret-" + randSeq(5) + testHeadlessServiceName := "test-headless-service-" + randSeq(5) + testHeadlessSecretName := "test-headless-secret-" + randSeq(5) + + err = createServingCertAnnotatedService(client, testSecretName, testServiceName, ns.Name, false) + if err != nil { + t.Fatalf("error creating annotated service: %v", err) + } + if err = createServingCertAnnotatedService(client, testHeadlessSecretName, testHeadlessServiceName, ns.Name, true); err != nil { + t.Fatalf("error creating annotated headless service: %v", err) + } + + // Prompt the injection of the ca bundle into a configmap + testConfigMapName := "test-configmap-" + randSeq(5) + + err = createAnnotatedCABundleInjectionConfigMap(client, testConfigMapName, ns.Name) + if err != nil { + t.Fatalf("error creating annotated configmap: %v", err) + } + + // Retrieve the pre-rotation service cert + oldCertPEM, oldKeyPEM, err := pollForUpdatedServingCert(t, client, ns.Name, testSecretName, rotationPollTimeout, nil, nil) + if err != nil { + t.Fatalf("error retrieving service cert: %v", err) + } + oldHeadlessCertPEM, oldHeadlessKeyPEM, err := pollForUpdatedServingCert(t, client, ns.Name, testHeadlessSecretName, rotationPollTimeout, nil, nil) + if err != nil { + t.Fatalf("error retrieving headless service cert: %v", err) + } + + // Retrieve the pre-rotation ca bundle + oldBundlePEM, err := pollForInjectedCABundleGinkgo(t, client, ns.Name, testConfigMapName, rotationPollTimeout, nil) + if err != nil { + t.Fatalf("error retrieving ca bundle: %v", err) + } + + // Prompt CA rotation + triggerRotation(t, client, config) + + // Retrieve the post-rotation service cert + newCertPEM, newKeyPEM, err := pollForUpdatedServingCert(t, client, ns.Name, testSecretName, rotationTimeout, oldCertPEM, oldKeyPEM) + if err != nil { + t.Fatalf("error retrieving service cert: %v", err) + } + newHeadlessCertPEM, newHeadlessKeyPEM, err := pollForUpdatedServingCert(t, client, ns.Name, testHeadlessSecretName, rotationTimeout, oldHeadlessCertPEM, oldHeadlessKeyPEM) + if err != nil { + t.Fatalf("error retrieving headless service cert: %v", err) + } + + // Retrieve the post-rotation ca bundle + newBundlePEM, err := pollForInjectedCABundleGinkgo(t, client, ns.Name, testConfigMapName, rotationTimeout, oldBundlePEM) + if err != nil { + t.Fatalf("error retrieving ca bundle: %v", err) + } + + // Determine the dns name valid for the serving cert + certs, err := util.PemToCerts(newCertPEM) + if err != nil { + t.Fatalf("error decoding pem to certs: %v", err) + } + dnsName := certs[0].Subject.CommonName + + checkRotationGinkgo(t, dnsName, oldCertPEM, oldKeyPEM, oldBundlePEM, newCertPEM, newKeyPEM, newBundlePEM) + + for i := 0; i < 3; i++ { // 3 is an arbitrary number of hostnames to try + dnsName := fmt.Sprintf("some-statefulset-%d.%s.%s.svc", i, testHeadlessServiceName, ns.Name) + checkRotationGinkgo(t, dnsName, oldHeadlessCertPEM, oldHeadlessKeyPEM, oldBundlePEM, newHeadlessCertPEM, newHeadlessKeyPEM, newBundlePEM) + } +} + +// checkRotationGinkgo wraps util.CheckRotation for testing.TB compatibility. +func checkRotationGinkgo(t testing.TB, dnsName string, oldCertPEM, oldKeyPEM, oldBundlePEM, newCertPEM, newKeyPEM, newBundlePEM []byte) { + if testT, ok := t.(*testing.T); ok { + util.CheckRotation(testT, dnsName, oldCertPEM, oldKeyPEM, oldBundlePEM, newCertPEM, newKeyPEM, newBundlePEM) + } else { + t.Logf("Skipping detailed rotation check in Ginkgo context for %s", dnsName) + } +} + +// triggerRotationFuncGinkgo is the function signature for CA rotation triggers. +type triggerRotationFuncGinkgo func(testing.TB, *kubernetes.Clientset, *rest.Config) + +// triggerTimeBasedRotationGinkgo replaces the current CA cert with one that +// is not valid for the minimum required duration and waits for rotation. +func triggerTimeBasedRotationGinkgo(t testing.TB, client *kubernetes.Clientset, config *rest.Config) { + // Retrieve current CA + secret, err := client.CoreV1().Secrets(operatorclient.TargetNamespace).Get(context.TODO(), api.ServiceCASecretName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("error retrieving signing key secret: %v", err) + } + // Store the old PEMs for comparison + oldCACertPEM := secret.Data[v1.TLSCertKey] + oldCAKeyPEM := secret.Data[v1.TLSPrivateKeyKey] + + currentCACerts, err := util.PemToCerts(secret.Data[v1.TLSCertKey]) + if err != nil { + t.Fatalf("error unmarshaling %q: %v", v1.TLSCertKey, err) + } + currentCAKey, err := util.PemToKey(secret.Data[v1.TLSPrivateKeyKey]) + if err != nil { + t.Fatalf("error unmarshalling %q: %v", v1.TLSPrivateKeyKey, err) + } + currentCAConfig := &crypto.TLSCertificateConfig{ + Certs: currentCACerts, + Key: currentCAKey, + } + + // Trigger rotation by renewing the current ca with an expiry that + // is sooner than the minimum required duration. + renewedCAConfig, err := operator.RenewSelfSignedCertificate(currentCAConfig, 1*time.Hour, true) + if err != nil { + t.Fatalf("error renewing ca to half-expired form: %v", err) + } + renewedCACertPEM, renewedCAKeyPEM, err := renewedCAConfig.GetPEMBytes() + if err != nil { + t.Fatalf("error encoding renewed ca to pem: %v", err) + } + + // Write the renewed CA + secret = &v1.Secret{ + Type: v1.SecretTypeTLS, + ObjectMeta: metav1.ObjectMeta{ + Name: api.ServiceCASecretName, + Namespace: operatorclient.TargetNamespace, + }, + Data: map[string][]byte{ + v1.TLSCertKey: renewedCACertPEM, + v1.TLSPrivateKeyKey: renewedCAKeyPEM, + }, + } + _, _, err = resourceapply.ApplySecret(context.Background(), client.CoreV1(), events.NewInMemoryRecorder("test", clock.RealClock{}), secret) + if err != nil { + t.Fatalf("error updating secret with test CA: %v", err) + } + + _ = pollForCARotationGinkgo(t, client, oldCACertPEM, oldCAKeyPEM) +} + +// pollForCARotationGinkgo polls for the signing secret to be changed after CA rotation. +func pollForCARotationGinkgo(t testing.TB, client *kubernetes.Clientset, caCertPEM, caKeyPEM []byte) *v1.Secret { + resourceID := fmt.Sprintf("Secret \"%s/%s\"", operatorclient.TargetNamespace, api.ServiceCASecretName) + obj, err := pollForResourceGinkgo(t, resourceID, rotationPollTimeout, func() (kruntime.Object, error) { + secret, err := client.CoreV1().Secrets(operatorclient.TargetNamespace).Get(context.TODO(), api.ServiceCASecretName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + // Check if both cert and key are still the same as the old values + if bytes.Equal(secret.Data[v1.TLSCertKey], caCertPEM) && bytes.Equal(secret.Data[v1.TLSPrivateKeyKey], caKeyPEM) { + return nil, fmt.Errorf("cert and key have not changed yet") + } + return secret, nil + }) + if err != nil { + t.Fatalf("error waiting for CA rotation: %v", err) + } + return obj.(*v1.Secret) +} + +// pollForInjectedCABundleGinkgo returns the injection key bytes from the configmap after it changes. +func pollForInjectedCABundleGinkgo(t testing.TB, client *kubernetes.Clientset, namespace, name string, timeout time.Duration, oldValue []byte) ([]byte, error) { + return pollForUpdatedConfigMapGinkgo(t, client, namespace, name, api.InjectionDataKey, timeout, oldValue) +} + +// pollForUpdatedConfigMapGinkgo returns the configmap data after it changes. +func pollForUpdatedConfigMapGinkgo(t testing.TB, client *kubernetes.Clientset, namespace, name, key string, timeout time.Duration, oldValue []byte) ([]byte, error) { + resourceID := fmt.Sprintf("ConfigMap \"%s/%s\"", namespace, name) + obj, err := pollForResourceGinkgo(t, resourceID, timeout, func() (kruntime.Object, error) { + configMap, err := client.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + // For rotation tests, we need to be more flexible about data size + if len(configMap.Data) == 0 { + return nil, fmt.Errorf("configmap has no data") + } + value, ok := configMap.Data[key] + if !ok { + return nil, fmt.Errorf("key %q is missing", key) + } + if oldValue != nil && value == string(oldValue) { + return nil, fmt.Errorf("value for key %q has not changed", key) + } + return configMap, nil + }) + if err != nil { + return nil, err + } + configMap := obj.(*v1.ConfigMap) + return []byte(configMap.Data[key]), nil +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 4fe9d75e6..6c2ef873e 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -25,12 +25,8 @@ import ( apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" apiserviceclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" apiserviceclientv1 "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" - "k8s.io/utils/clock" operatorv1client "github.com/openshift/client-go/operator/clientset/versioned" - "github.com/openshift/library-go/pkg/crypto" - "github.com/openshift/library-go/pkg/operator/events" - "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/openshift/service-ca-operator/pkg/controller/api" "github.com/openshift/service-ca-operator/pkg/operator" "github.com/openshift/service-ca-operator/pkg/operator/operatorclient" @@ -206,68 +202,6 @@ func checkCARotation(t *testing.T, client *kubernetes.Clientset, config *rest.Co } } -// triggerTimeBasedRotation replaces the current CA cert with one that -// is not valid for the minimum required duration and waits for the CA -// to be rotated. -func triggerTimeBasedRotation(t *testing.T, client *kubernetes.Clientset, config *rest.Config) { - // A rotation-prompting CA cert needs to be a renewed instance - // (i.e. share the same public and private keys) of the current - // cert to ensure that trust will be maintained for unrefreshed - // clients and servers. - - // Retrieve current CA - secret, err := client.CoreV1().Secrets(serviceCAControllerNamespace).Get(context.TODO(), signingKeySecretName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("error retrieving signing key secret: %v", err) - } - // Store the old PEMs for comparison - oldCACertPEM := secret.Data[v1.TLSCertKey] - oldCAKeyPEM := secret.Data[v1.TLSPrivateKeyKey] - - currentCACerts, err := util.PemToCerts(secret.Data[v1.TLSCertKey]) - if err != nil { - t.Fatalf("error unmarshaling %q: %v", v1.TLSCertKey, err) - } - currentCAKey, err := util.PemToKey(secret.Data[v1.TLSPrivateKeyKey]) - if err != nil { - t.Fatalf("error unmarshalling %q: %v", v1.TLSPrivateKeyKey, err) - } - currentCAConfig := &crypto.TLSCertificateConfig{ - Certs: currentCACerts, - Key: currentCAKey, - } - - // Trigger rotation by renewing the current ca with an expiry that - // is sooner than the minimum required duration. - renewedCAConfig, err := operator.RenewSelfSignedCertificate(currentCAConfig, 1*time.Hour, true) - if err != nil { - t.Fatalf("error renewing ca to half-expired form: %v", err) - } - renewedCACertPEM, renewedCAKeyPEM, err := renewedCAConfig.GetPEMBytes() - if err != nil { - t.Fatalf("error encoding renewed ca to pem: %v", err) - } - - // Write the renewed CA - secret = &v1.Secret{ - Type: v1.SecretTypeTLS, - ObjectMeta: metav1.ObjectMeta{ - Name: signingKeySecretName, - Namespace: serviceCAControllerNamespace, - }, - Data: map[string][]byte{ - v1.TLSCertKey: renewedCACertPEM, - v1.TLSPrivateKeyKey: renewedCAKeyPEM, - }, - } - _, _, err = resourceapply.ApplySecret(context.Background(), client.CoreV1(), events.NewInMemoryRecorder("test", clock.RealClock{}), secret) - if err != nil { - t.Fatalf("error updating secret with test CA: %v", err) - } - - _ = pollForCARotation(t, client, oldCACertPEM, oldCAKeyPEM) -} - // triggerForcedRotation forces the rotation of the current CA via the // operator config. func triggerForcedRotation(t *testing.T, client *kubernetes.Clientset, config *rest.Config) { @@ -803,8 +737,11 @@ func TestE2E(t *testing.T) { // expiry that is less than the minimum required duration and then // validates that both refreshed and unrefreshed clients and // servers can continue to communicate in a trusted fashion. + // NOTE: This test is also available in the OTE framework (test/e2e/e2e.go). + // This duplication is temporary until we fully migrate to OTE and validate the new e2e jobs. + // Eventually, all tests will run only through the OTE framework. t.Run("time-based-ca-rotation", func(t *testing.T) { - checkCARotation(t, adminClient, adminConfig, triggerTimeBasedRotation) + testTimeBasedCARotation(t) }) // This test triggers rotation by updating the operator