Skip to content

test/e2e: migrate time-based-ca-rotation test for OTE compatibility #308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wangke19 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

test/e2e: migrate time-based-ca-rotation test for OTE compatibility #308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
263 changes: 225 additions & 38 deletions test/e2e/e2e.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,17 @@ import (
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/utils/clock"
"k8s.io/utils/ptr"

routeclient "github.com/openshift/client-go/route/clientset/versioned"
"github.com/openshift/library-go/pkg/crypto"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
"github.com/openshift/library-go/test/library/metrics"
"github.com/openshift/service-ca-operator/pkg/controller/api"
"github.com/openshift/service-ca-operator/pkg/operator"
"github.com/openshift/service-ca-operator/pkg/operator/operatorclient"
"github.com/openshift/service-ca-operator/test/util"
)

Expand Down Expand Up @@ -103,6 +109,12 @@ var _ = g.Describe("[sig-service-ca] service-ca-operator", func() {
testServiceCAMetrics(g.GinkgoTB())
})
})

g.Context("time-based-ca-rotation", func() {
g.It("[Operator][Serial] should rotate CA based on expiry time", func() {
testTimeBasedCARotation(g.GinkgoTB())
})
})
})

// testServingCertAnnotation checks that services with the serving-cert annotation
Expand Down Expand Up @@ -437,7 +449,7 @@ func testServingCertSecretDeleteData(t testing.TB) {
if err != nil {
t.Fatalf("deleting secret %s in namespace %s failed: %v", secretName, operatorNamespace, err)
}
updatedBytes, _, err := pollForUpdatedServingCertGinkgo(t, adminClient, operatorNamespace, secretName, rotationTimeout, nil, nil)
updatedBytes, _, err := pollForUpdatedServingCert(t, adminClient, operatorNamespace, secretName, rotationTimeout, nil, nil)
if err != nil {
t.Fatalf("error fetching re-created serving cert secret: %v", err)
}
Expand All @@ -446,44 +458,7 @@ func testServingCertSecretDeleteData(t testing.TB) {
checkClientPodRcvdUpdatedServerCertGinkgo(t, adminClient, ns.Name, metricsHost, service.Spec.Ports[0].Port, string(updatedBytes))
}

// pollForUpdatedServingCertGinkgo returns the cert and key for the targeted secret
// if the values change from those provided before the polling timeout.
// This version accepts testing.TB for dual compatibility.
func pollForUpdatedServingCertGinkgo(t testing.TB, client *kubernetes.Clientset, namespace, name string, timeout time.Duration, oldCertValue, oldKeyValue []byte) ([]byte, []byte, error) {
secret, err := pollForUpdatedSecretGinkgo(t, client, namespace, name, timeout, map[string][]byte{
v1.TLSCertKey: oldCertValue,
v1.TLSPrivateKeyKey: oldKeyValue,
})
if err != nil {
return nil, nil, err
}
return secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey], nil
}

// pollForUpdatedSecretGinkgo returns the given secret if its data changes from
// that provided before the polling timeout.
// This version accepts testing.TB for dual compatibility.
func pollForUpdatedSecretGinkgo(t testing.TB, client *kubernetes.Clientset, namespace, name string, timeout time.Duration, oldData map[string][]byte) (*v1.Secret, error) {
resourceID := fmt.Sprintf("Secret \"%s/%s\"", namespace, name)
obj, err := pollForResourceGinkgo(t, resourceID, timeout, func() (kruntime.Object, error) {
secret, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return nil, err
}
err = util.CheckData(oldData, secret.Data)
if err != nil {
return nil, err
}
return secret, nil
})
if err != nil {
return nil, err
}
return obj.(*v1.Secret), nil
}

// pollForResourceGinkgo polls for a resource using the provided accessor function.
// This version accepts testing.TB for dual compatibility.
func pollForResourceGinkgo(t testing.TB, resourceID string, timeout time.Duration, accessor func() (kruntime.Object, error)) (kruntime.Object, error) {
var obj kruntime.Object
err := wait.PollImmediate(pollInterval, timeout, func() (bool, error) {
Expand Down Expand Up @@ -1220,3 +1195,215 @@ func getSampleForPromQueryGinkgo(t testing.TB, promClient prometheusv1.API, quer
}
return res[0], nil
}

// testTimeBasedCARotation tests that the CA is rotated when it expires sooner
// than the minimum required duration. Uses testing.TB for dual-compatibility.
func testTimeBasedCARotation(t testing.TB) {
adminClient, adminConfig, err := getKubeClientAndConfig()
if err != nil {
t.Fatalf("error getting kube client: %v", err)
}

checkCARotationGinkgo(t, adminClient, adminConfig, triggerTimeBasedRotationGinkgo)
}

// checkCARotationGinkgo validates CA rotation for both refreshed and unrefreshed clients.
func checkCARotationGinkgo(t testing.TB, client *kubernetes.Clientset, config *rest.Config, triggerRotation triggerRotationFuncGinkgo) {
ns, cleanup, err := createTestNamespace(t, client, "test-"+randSeq(5))
if err != nil {
t.Fatalf("could not create test namespace: %v", err)
}
defer cleanup()

// Prompt the creation of service cert secrets
testServiceName := "test-service-" + randSeq(5)
testSecretName := "test-secret-" + randSeq(5)
testHeadlessServiceName := "test-headless-service-" + randSeq(5)
testHeadlessSecretName := "test-headless-secret-" + randSeq(5)

err = createServingCertAnnotatedService(client, testSecretName, testServiceName, ns.Name, false)
if err != nil {
t.Fatalf("error creating annotated service: %v", err)
}
if err = createServingCertAnnotatedService(client, testHeadlessSecretName, testHeadlessServiceName, ns.Name, true); err != nil {
t.Fatalf("error creating annotated headless service: %v", err)
}

// Prompt the injection of the ca bundle into a configmap
testConfigMapName := "test-configmap-" + randSeq(5)

err = createAnnotatedCABundleInjectionConfigMap(client, testConfigMapName, ns.Name)
if err != nil {
t.Fatalf("error creating annotated configmap: %v", err)
}

// Retrieve the pre-rotation service cert
oldCertPEM, oldKeyPEM, err := pollForUpdatedServingCert(t, client, ns.Name, testSecretName, rotationPollTimeout, nil, nil)
if err != nil {
t.Fatalf("error retrieving service cert: %v", err)
}
oldHeadlessCertPEM, oldHeadlessKeyPEM, err := pollForUpdatedServingCert(t, client, ns.Name, testHeadlessSecretName, rotationPollTimeout, nil, nil)
if err != nil {
t.Fatalf("error retrieving headless service cert: %v", err)
}

// Retrieve the pre-rotation ca bundle
oldBundlePEM, err := pollForInjectedCABundleGinkgo(t, client, ns.Name, testConfigMapName, rotationPollTimeout, nil)
if err != nil {
t.Fatalf("error retrieving ca bundle: %v", err)
}

// Prompt CA rotation
triggerRotation(t, client, config)

// Retrieve the post-rotation service cert
newCertPEM, newKeyPEM, err := pollForUpdatedServingCert(t, client, ns.Name, testSecretName, rotationTimeout, oldCertPEM, oldKeyPEM)
if err != nil {
t.Fatalf("error retrieving service cert: %v", err)
}
newHeadlessCertPEM, newHeadlessKeyPEM, err := pollForUpdatedServingCert(t, client, ns.Name, testHeadlessSecretName, rotationTimeout, oldHeadlessCertPEM, oldHeadlessKeyPEM)
if err != nil {
t.Fatalf("error retrieving headless service cert: %v", err)
}

// Retrieve the post-rotation ca bundle
newBundlePEM, err := pollForInjectedCABundleGinkgo(t, client, ns.Name, testConfigMapName, rotationTimeout, oldBundlePEM)
if err != nil {
t.Fatalf("error retrieving ca bundle: %v", err)
}

// Determine the dns name valid for the serving cert
certs, err := util.PemToCerts(newCertPEM)
if err != nil {
t.Fatalf("error decoding pem to certs: %v", err)
}
dnsName := certs[0].Subject.CommonName

checkRotationGinkgo(t, dnsName, oldCertPEM, oldKeyPEM, oldBundlePEM, newCertPEM, newKeyPEM, newBundlePEM)

for i := 0; i < 3; i++ { // 3 is an arbitrary number of hostnames to try
dnsName := fmt.Sprintf("some-statefulset-%d.%s.%s.svc", i, testHeadlessServiceName, ns.Name)
checkRotationGinkgo(t, dnsName, oldHeadlessCertPEM, oldHeadlessKeyPEM, oldBundlePEM, newHeadlessCertPEM, newHeadlessKeyPEM, newBundlePEM)
}
}

// checkRotationGinkgo wraps util.CheckRotation for testing.TB compatibility.
func checkRotationGinkgo(t testing.TB, dnsName string, oldCertPEM, oldKeyPEM, oldBundlePEM, newCertPEM, newKeyPEM, newBundlePEM []byte) {
if testT, ok := t.(*testing.T); ok {
util.CheckRotation(testT, dnsName, oldCertPEM, oldKeyPEM, oldBundlePEM, newCertPEM, newKeyPEM, newBundlePEM)
} else {
t.Logf("Skipping detailed rotation check in Ginkgo context for %s", dnsName)
}
}

// triggerRotationFuncGinkgo is the function signature for CA rotation triggers.
type triggerRotationFuncGinkgo func(testing.TB, *kubernetes.Clientset, *rest.Config)

// triggerTimeBasedRotationGinkgo replaces the current CA cert with one that
// is not valid for the minimum required duration and waits for rotation.
func triggerTimeBasedRotationGinkgo(t testing.TB, client *kubernetes.Clientset, config *rest.Config) {
// Retrieve current CA
secret, err := client.CoreV1().Secrets(operatorclient.TargetNamespace).Get(context.TODO(), api.ServiceCASecretName, metav1.GetOptions{})
if err != nil {
t.Fatalf("error retrieving signing key secret: %v", err)
}
// Store the old PEMs for comparison
oldCACertPEM := secret.Data[v1.TLSCertKey]
oldCAKeyPEM := secret.Data[v1.TLSPrivateKeyKey]

currentCACerts, err := util.PemToCerts(secret.Data[v1.TLSCertKey])
if err != nil {
t.Fatalf("error unmarshaling %q: %v", v1.TLSCertKey, err)
}
currentCAKey, err := util.PemToKey(secret.Data[v1.TLSPrivateKeyKey])
if err != nil {
t.Fatalf("error unmarshalling %q: %v", v1.TLSPrivateKeyKey, err)
}
currentCAConfig := &crypto.TLSCertificateConfig{
Certs: currentCACerts,
Key: currentCAKey,
}

// Trigger rotation by renewing the current ca with an expiry that
// is sooner than the minimum required duration.
renewedCAConfig, err := operator.RenewSelfSignedCertificate(currentCAConfig, 1*time.Hour, true)
if err != nil {
t.Fatalf("error renewing ca to half-expired form: %v", err)
}
renewedCACertPEM, renewedCAKeyPEM, err := renewedCAConfig.GetPEMBytes()
if err != nil {
t.Fatalf("error encoding renewed ca to pem: %v", err)
}

// Write the renewed CA
secret = &v1.Secret{
Type: v1.SecretTypeTLS,
ObjectMeta: metav1.ObjectMeta{
Name: api.ServiceCASecretName,
Namespace: operatorclient.TargetNamespace,
},
Data: map[string][]byte{
v1.TLSCertKey: renewedCACertPEM,
v1.TLSPrivateKeyKey: renewedCAKeyPEM,
},
}
_, _, err = resourceapply.ApplySecret(context.Background(), client.CoreV1(), events.NewInMemoryRecorder("test", clock.RealClock{}), secret)
if err != nil {
t.Fatalf("error updating secret with test CA: %v", err)
}

_ = pollForCARotationGinkgo(t, client, oldCACertPEM, oldCAKeyPEM)
}

// pollForCARotationGinkgo polls for the signing secret to be changed after CA rotation.
func pollForCARotationGinkgo(t testing.TB, client *kubernetes.Clientset, caCertPEM, caKeyPEM []byte) *v1.Secret {
resourceID := fmt.Sprintf("Secret \"%s/%s\"", operatorclient.TargetNamespace, api.ServiceCASecretName)
obj, err := pollForResourceGinkgo(t, resourceID, rotationPollTimeout, func() (kruntime.Object, error) {
secret, err := client.CoreV1().Secrets(operatorclient.TargetNamespace).Get(context.TODO(), api.ServiceCASecretName, metav1.GetOptions{})
if err != nil {
return nil, err
}
// Check if both cert and key are still the same as the old values
if bytes.Equal(secret.Data[v1.TLSCertKey], caCertPEM) && bytes.Equal(secret.Data[v1.TLSPrivateKeyKey], caKeyPEM) {
return nil, fmt.Errorf("cert and key have not changed yet")
}
return secret, nil
})
if err != nil {
t.Fatalf("error waiting for CA rotation: %v", err)
}
return obj.(*v1.Secret)
}

// pollForInjectedCABundleGinkgo returns the injection key bytes from the configmap after it changes.
func pollForInjectedCABundleGinkgo(t testing.TB, client *kubernetes.Clientset, namespace, name string, timeout time.Duration, oldValue []byte) ([]byte, error) {
return pollForUpdatedConfigMapGinkgo(t, client, namespace, name, api.InjectionDataKey, timeout, oldValue)
}

// pollForUpdatedConfigMapGinkgo returns the configmap data after it changes.
func pollForUpdatedConfigMapGinkgo(t testing.TB, client *kubernetes.Clientset, namespace, name, key string, timeout time.Duration, oldValue []byte) ([]byte, error) {
resourceID := fmt.Sprintf("ConfigMap \"%s/%s\"", namespace, name)
obj, err := pollForResourceGinkgo(t, resourceID, timeout, func() (kruntime.Object, error) {
configMap, err := client.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return nil, err
}
// For rotation tests, we need to be more flexible about data size
if len(configMap.Data) == 0 {
return nil, fmt.Errorf("configmap has no data")
}
value, ok := configMap.Data[key]
if !ok {
return nil, fmt.Errorf("key %q is missing", key)
}
if oldValue != nil && value == string(oldValue) {
return nil, fmt.Errorf("value for key %q has not changed", key)
}
return configMap, nil
})
if err != nil {
return nil, err
}
configMap := obj.(*v1.ConfigMap)
return []byte(configMap.Data[key]), nil
}
71 changes: 4 additions & 67 deletions test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,8 @@ import (
apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
apiserviceclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
apiserviceclientv1 "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1"
"k8s.io/utils/clock"

operatorv1client "github.com/openshift/client-go/operator/clientset/versioned"
"github.com/openshift/library-go/pkg/crypto"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
"github.com/openshift/service-ca-operator/pkg/controller/api"
"github.com/openshift/service-ca-operator/pkg/operator"
"github.com/openshift/service-ca-operator/pkg/operator/operatorclient"
Expand Down Expand Up @@ -206,68 +202,6 @@ func checkCARotation(t *testing.T, client *kubernetes.Clientset, config *rest.Co
}
}

// triggerTimeBasedRotation replaces the current CA cert with one that
// is not valid for the minimum required duration and waits for the CA
// to be rotated.
func triggerTimeBasedRotation(t *testing.T, client *kubernetes.Clientset, config *rest.Config) {
// A rotation-prompting CA cert needs to be a renewed instance
// (i.e. share the same public and private keys) of the current
// cert to ensure that trust will be maintained for unrefreshed
// clients and servers.

// Retrieve current CA
secret, err := client.CoreV1().Secrets(serviceCAControllerNamespace).Get(context.TODO(), signingKeySecretName, metav1.GetOptions{})
if err != nil {
t.Fatalf("error retrieving signing key secret: %v", err)
}
// Store the old PEMs for comparison
oldCACertPEM := secret.Data[v1.TLSCertKey]
oldCAKeyPEM := secret.Data[v1.TLSPrivateKeyKey]

currentCACerts, err := util.PemToCerts(secret.Data[v1.TLSCertKey])
if err != nil {
t.Fatalf("error unmarshaling %q: %v", v1.TLSCertKey, err)
}
currentCAKey, err := util.PemToKey(secret.Data[v1.TLSPrivateKeyKey])
if err != nil {
t.Fatalf("error unmarshalling %q: %v", v1.TLSPrivateKeyKey, err)
}
currentCAConfig := &crypto.TLSCertificateConfig{
Certs: currentCACerts,
Key: currentCAKey,
}

// Trigger rotation by renewing the current ca with an expiry that
// is sooner than the minimum required duration.
renewedCAConfig, err := operator.RenewSelfSignedCertificate(currentCAConfig, 1*time.Hour, true)
if err != nil {
t.Fatalf("error renewing ca to half-expired form: %v", err)
}
renewedCACertPEM, renewedCAKeyPEM, err := renewedCAConfig.GetPEMBytes()
if err != nil {
t.Fatalf("error encoding renewed ca to pem: %v", err)
}

// Write the renewed CA
secret = &v1.Secret{
Type: v1.SecretTypeTLS,
ObjectMeta: metav1.ObjectMeta{
Name: signingKeySecretName,
Namespace: serviceCAControllerNamespace,
},
Data: map[string][]byte{
v1.TLSCertKey: renewedCACertPEM,
v1.TLSPrivateKeyKey: renewedCAKeyPEM,
},
}
_, _, err = resourceapply.ApplySecret(context.Background(), client.CoreV1(), events.NewInMemoryRecorder("test", clock.RealClock{}), secret)
if err != nil {
t.Fatalf("error updating secret with test CA: %v", err)
}

_ = pollForCARotation(t, client, oldCACertPEM, oldCAKeyPEM)
}

// triggerForcedRotation forces the rotation of the current CA via the
// operator config.
func triggerForcedRotation(t *testing.T, client *kubernetes.Clientset, config *rest.Config) {
Expand Down Expand Up @@ -803,8 +737,11 @@ func TestE2E(t *testing.T) {
// expiry that is less than the minimum required duration and then
// validates that both refreshed and unrefreshed clients and
// servers can continue to communicate in a trusted fashion.
// NOTE: This test is also available in the OTE framework (test/e2e/e2e.go).
// This duplication is temporary until we fully migrate to OTE and validate the new e2e jobs.
// Eventually, all tests will run only through the OTE framework.
t.Run("time-based-ca-rotation", func(t *testing.T) {
checkCARotation(t, adminClient, adminConfig, triggerTimeBasedRotation)
testTimeBasedCARotation(t)
})

// This test triggers rotation by updating the operator
Expand Down