Introduce KMS encryption mode in encryption library

Author: ardaguclu

go.mod

@@ -34,6 +34,7 @@ require (
 	golang.org/x/net v0.43.0
 	golang.org/x/sys v0.36.0
 	golang.org/x/time v0.9.0
+	google.golang.org/grpc v1.72.1
 	gopkg.in/evanphx/json-patch.v4 v4.12.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	k8s.io/api v0.34.1
@@ -43,6 +44,7 @@ require (
 	k8s.io/client-go v0.34.1
 	k8s.io/component-base v0.34.1
 	k8s.io/klog/v2 v2.130.1
+	k8s.io/kms v0.34.1
 	k8s.io/kube-aggregator v0.34.1
 	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96
@@ -125,11 +127,9 @@ require (
 	golang.org/x/tools v0.36.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect
-	google.golang.org/grpc v1.72.1 // indirect
 	google.golang.org/protobuf v1.36.5 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/kms v0.34.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
 	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect

pkg/operator/encryption/controllers/key_controller.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/openshift/library-go/pkg/operator/encryption/kms"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -20,6 +21,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
 
+	configv1 "github.com/openshift/api/config/v1"
 	operatorv1 "github.com/openshift/api/operator/v1"
 	configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
 	configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1"
@@ -159,11 +161,20 @@ func (c *keyController) sync(ctx context.Context, syncCtx factory.SyncContext) (
 }
 
 func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext factory.SyncContext, encryptedGRs []schema.GroupResource) error {
-	currentMode, externalReason, err := c.getCurrentModeAndExternalReason(ctx)
+	currentMode, externalReason, kmsConfig, err := c.getCurrentModeAndExternalReason(ctx)
 	if err != nil {
 		return err
 	}
 
+	// Compute KMS hashes if using KMS mode
+	var kmsConfigHash, kmsKeyIDHash string
+	if currentMode == state.KMS && kmsConfig != nil {
+		kmsConfigHash, kmsKeyIDHash, err = c.getKMSHashes(ctx, kmsConfig)
+		if err != nil {
+			return err
+		}
+	}
+
 	currentConfig, desiredEncryptionState, secrets, isProgressingReason, err := statemachine.GetEncryptionConfigAndState(ctx, c.deployer, c.secretClient, c.encryptionSecretSelector, encryptedGRs)
 	if err != nil {
 		return err
@@ -191,7 +202,7 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact
 
 	var commonReason *string
 	for gr, grKeys := range desiredEncryptionState {
-		latestKeyID, internalReason, needed := needsNewKey(grKeys, currentMode, externalReason, encryptedGRs)
+		latestKeyID, internalReason, needed := needsNewKey(grKeys, currentMode, externalReason, encryptedGRs, kmsConfigHash, kmsKeyIDHash)
 		if !needed {
 			continue
 		}
@@ -218,7 +229,7 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact
 
 	sort.Sort(sort.StringSlice(reasons))
 	internalReason := strings.Join(reasons, ", ")
-	keySecret, err := c.generateKeySecret(newKeyID, currentMode, internalReason, externalReason)
+	keySecret, err := c.generateKeySecret(newKeyID, currentMode, internalReason, externalReason, kmsConfigHash, kmsKeyIDHash)
 	if err != nil {
 		return fmt.Errorf("failed to create key: %v", err)
 	}
@@ -255,7 +266,7 @@ func (c *keyController) validateExistingSecret(ctx context.Context, keySecret *c
 	return nil // we made this key earlier
 }
 
-func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, internalReason, externalReason string) (*corev1.Secret, error) {
+func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, internalReason, externalReason, kmsConfigHash, kmsKeyIDHash string) (*corev1.Secret, error) {
 	bs := crypto.ModeToNewKeyFunc[currentMode]()
 	ks := state.KeyState{
 		Key: apiserverv1.Key{
@@ -265,40 +276,69 @@ func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode,
 		Mode:           currentMode,
 		InternalReason: internalReason,
 		ExternalReason: externalReason,
+		KMSConfigHash:  kmsConfigHash,
+		KMSKeyIDHash:   kmsKeyIDHash,
 	}
 	return secrets.FromKeyState(c.instanceName, ks)
 }
 
-func (c *keyController) getCurrentModeAndExternalReason(ctx context.Context) (state.Mode, string, error) {
+func (c *keyController) getCurrentModeAndExternalReason(ctx context.Context) (state.Mode, string, *configv1.KMSConfig, error) {
 	apiServer, err := c.apiServerClient.Get(ctx, "cluster", metav1.GetOptions{})
 	if err != nil {
-		return "", "", err
+		return "", "", nil, err
 	}
 
 	operatorSpec, _, _, err := c.operatorClient.GetOperatorState()
 	if err != nil {
-		return "", "", err
+		return "", "", nil, err
 	}
 
 	encryptionConfig, err := structuredUnsupportedConfigFrom(operatorSpec.UnsupportedConfigOverrides.Raw, c.unsupportedConfigPrefix)
 	if err != nil {
-		return "", "", err
+		return "", "", nil, err
 	}
 
 	reason := encryptionConfig.Encryption.Reason
 	switch currentMode := state.Mode(apiServer.Spec.Encryption.Type); currentMode {
 	case state.AESCBC, state.AESGCM, state.Identity: // secretbox is disabled for now
-		return currentMode, reason, nil
+		return currentMode, reason, nil, nil
+	case state.KMS:
+		return currentMode, reason, apiServer.Spec.Encryption.KMS, nil
 	case "": // unspecified means use the default (which can change over time)
-		return state.DefaultMode, reason, nil
+		return state.DefaultMode, reason, nil, nil
 	default:
-		return "", "", fmt.Errorf("unknown encryption mode configured: %s", currentMode)
+		return "", "", nil, fmt.Errorf("unknown encryption mode configured: %s", currentMode)
+	}
+}
+
+func (c *keyController) getKMSHashes(ctx context.Context, kmsConfig *configv1.KMSConfig) (string, string, error) {
+	// Generate unix socket path from KMS config and get the hash
+	socketPath, configHash, err := kms.GenerateUnixSocketPath(kmsConfig)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to generate KMS unix socket path: %w", err)
+	}
+
+	kmsClient, err := kms.NewKMSClient(socketPath)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to create KMS client: %w", err)
+	}
+	defer kmsClient.Close()
+
+	statusResp, err := kmsClient.Status(ctx)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to call KMS Status endpoint: %w", err)
+	}
+
+	if statusResp.Healthz != "ok" {
+		return "", "", fmt.Errorf("KMS plugin is unhealthy: %s", statusResp.Healthz)
 	}
+
+	return configHash, kms.ComputeKMSKeyIDHash(statusResp.KeyID), nil
 }
 
 // needsNewKey checks whether a new key must be created for the given resource. If true, it also returns the latest
 // used key ID and a reason string.
-func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, externalReason string, encryptedGRs []schema.GroupResource) (uint64, string, bool) {
+func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, externalReason string, encryptedGRs []schema.GroupResource, kmsConfigHash, kmsKeyIDHash string) (uint64, string, bool) {
 	// we always need to have some encryption keys unless we are turned off
 	if len(grKeys.ReadKeys) == 0 {
 		return 0, "key-does-not-exist", currentMode != state.Identity
@@ -346,6 +386,21 @@ func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, extern
 		return latestKeyID, "external-reason-changed", true
 	}
 
+	// if we are using KMS, check if the KMS configuration or key ID hash has changed
+	if currentMode == state.KMS {
+		if latestKey.KMSConfigHash != kmsConfigHash && len(kmsConfigHash) != 0 {
+			return latestKeyID, "kms-config-changed", true
+		}
+
+		if latestKey.KMSKeyIDHash != kmsKeyIDHash && len(kmsKeyIDHash) != 0 {
+			return latestKeyID, "kms-key-id-changed", true
+		}
+
+		// For KMS mode, we don't do time-based rotation
+		// KMS keys are rotated externally by the KMS system
+		return 0, "", false
+	}
+
 	// we check for encryptionSecretMigratedTimestamp set by migration controller to determine when migration completed
 	// this also generates back pressure for key rotation when migration takes a long time or was recently completed
 	return latestKeyID, "rotation-interval-has-passed", time.Since(latestKey.Migrated.Timestamp) > encryptionSecretMigrationInterval

pkg/operator/encryption/controllers/key_controller_test.go

@@ -495,7 +495,7 @@ func TestGetCurrentModeAndExternalReason(t *testing.T) {
 
 			// act
 			target := keyController{unsupportedConfigPrefix: scenario.prefix, operatorClient: fakeOperatorClient, apiServerClient: fakeApiServerClient}
-			_, externalReason, err := target.getCurrentModeAndExternalReason(context.TODO())
+			_, externalReason, _, err := target.getCurrentModeAndExternalReason(context.TODO())
 
 			// validate
 			if err != nil {

pkg/operator/encryption/controllers/state_controller_test.go

@@ -707,6 +707,78 @@ func TestStateController(t *testing.T) {
 				encryptiontesting.ValidateOperatorClientConditions(ts, operatorClient, []operatorv1.OperatorCondition{expectedCondition})
 			},
 		},
+		{
+			name:            "secret with EncryptionConfig is created for KMS mode with proper KMS provider configuration",
+			targetNamespace: "kms",
+			targetGRs: []schema.GroupResource{
+				{Group: "", Resource: "secrets"},
+			},
+			initialResources: []runtime.Object{
+				encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"),
+				func() *corev1.Secret {
+					s := encryptiontesting.CreateEncryptionKeySecretNoDataWithMode("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 1, "kms")
+					// Set KMS-specific annotations with config hash and key ID hash
+					s.Annotations["encryption.apiserver.operator.openshift.io/kms-config-hash"] = "1234567890abcdef"
+					s.Annotations["encryption.apiserver.operator.openshift.io/kms-key-id-hash"] = "fedcba0987654321"
+					return s
+				}(),
+			},
+			expectedActions: []string{
+				"list:pods:kms",
+				"get:secrets:kms",
+				"list:secrets:openshift-config-managed",
+				"get:secrets:openshift-config-managed",
+				"create:secrets:openshift-config-managed",
+				"create:events:kms",
+				"create:events:kms",
+			},
+			expectedEncryptionCfg: func() *apiserverconfigv1.EncryptionConfiguration {
+				return &apiserverconfigv1.EncryptionConfiguration{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "EncryptionConfiguration",
+						APIVersion: "apiserver.config.k8s.io/v1",
+					},
+					Resources: []apiserverconfigv1.ResourceConfiguration{
+						{
+							Resources: []string{"secrets"},
+							Providers: []apiserverconfigv1.ProviderConfiguration{
+								{
+									Identity: &apiserverconfigv1.IdentityConfiguration{},
+								},
+								{
+									KMS: &apiserverconfigv1.KMSConfiguration{
+										APIVersion: "v2",
+										Name:       "kms-provider-fedcba0987654321-1", // kms-provider-{keyIDHash16}-{key ID}
+										Endpoint:   "unix://var/run/kms/kms-1234567890abcdef.sock",
+										Timeout: &metav1.Duration{
+											Duration: 10 * time.Second,
+										},
+									},
+								},
+							},
+						},
+					},
+				}
+			}(),
+			validateFunc: func(ts *testing.T, actions []clientgotesting.Action, destName string, expectedEncryptionCfg *apiserverconfigv1.EncryptionConfiguration) {
+				wasSecretValidated := false
+				for _, action := range actions {
+					if action.Matches("create", "secrets") {
+						createAction := action.(clientgotesting.CreateAction)
+						actualSecret := createAction.GetObject().(*corev1.Secret)
+						err := validateSecretWithEncryptionConfig(actualSecret, expectedEncryptionCfg, destName)
+						if err != nil {
+							ts.Fatalf("failed to verify the encryption config, due to %v", err)
+						}
+						wasSecretValidated = true
+						break
+					}
+				}
+				if !wasSecretValidated {
+					ts.Errorf("the secret wasn't created and validated")
+				}
+			},
+		},
 	}
 
 	for _, scenario := range scenarios {

pkg/operator/encryption/crypto/keys.go

@@ -12,6 +12,7 @@ var (
 		state.AESGCM:    NewAES256Key,
 		state.SecretBox: NewAES256Key, // secretbox requires a 32 byte key so we can reuse the same function here
 		state.Identity:  NewIdentityKey,
+		state.KMS:       NewIdentityKey, // this is not used in KMS
 	}
 )
 

pkg/operator/encryption/encryptionconfig/config.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"sort"
 
+	"github.com/openshift/library-go/pkg/operator/encryption/kms"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
@@ -106,6 +107,25 @@ func ToEncryptionState(encryptionConfig *apiserverconfigv1.EncryptionConfigurati
 					Mode: s,
 				}
 
+			case provider.KMS != nil:
+				configHash, keyIDHash, keyName, err := kms.ExtractKMSHashAndKeyName(provider)
+				if err != nil {
+					klog.Warningf("skipping invalid encryption KMS config for resource %v", provider)
+					continue // should never happen
+				}
+
+				ks = state.KeyState{
+					Key: apiserverconfigv1.Key{
+						Name: keyName,
+						// We set this unused secret just to align with what we set initially.
+						// This is unused.
+						Secret: base64.StdEncoding.EncodeToString(crypto.ModeToNewKeyFunc[state.KMS]()),
+					},
+					Mode:          state.KMS,
+					KMSConfigHash: configHash,
+					KMSKeyIDHash:  keyIDHash,
+				}
+
 			default:
 				klog.Infof("skipping invalid provider index %d for resource %s", i, resourceConfig.Resources[0])
 				continue // should never happen
@@ -192,6 +212,8 @@ func stateToProviders(desired state.GroupResourceState) []apiserverconfigv1.Prov
 					Keys: []apiserverconfigv1.Key{key.Key},
 				},
 			})
+		case state.KMS:
+			providers = append(providers, kms.GenerateKMSProviderConfigurationFromKey(key))
 		default:
 			// this should never happen because our input should always be valid
 			klog.Infof("skipping key %s as it has invalid mode %s", key.Key.Name, key.Mode)