test

Signed-off-by: Qi Wang <qiwan@redhat.com>

Author: QiWang19

pkg/controller/container-runtime-config/container_runtime_config_controller.go

@@ -3,6 +3,7 @@ package containerruntimeconfig
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
@@ -17,7 +18,9 @@ import (
 	configclientset "github.com/openshift/client-go/config/clientset/versioned"
 	configinformers "github.com/openshift/client-go/config/informers/externalversions"
 	cligoinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
+	cligoinformersv1alpha1 "github.com/openshift/client-go/config/informers/externalversions/config/v1alpha1"
 	cligolistersv1 "github.com/openshift/client-go/config/listers/config/v1"
+	cligolistersv1alpha1 "github.com/openshift/client-go/config/listers/config/v1alpha1"
 	runtimeutils "github.com/openshift/runtime-utils/pkg/registries"
 
 	operatorinformersv1alpha1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1alpha1"
@@ -84,6 +87,7 @@ type Controller struct {
 
 	syncHandler                   func(mcp string) error
 	syncImgHandler                func(mcp string) error
+	syncCRIOCPHandler             func(key string) error
 	enqueueContainerRuntimeConfig func(*mcfgv1.ContainerRuntimeConfig)
 
 	ccLister       mcfglistersv1.ControllerConfigLister
@@ -104,6 +108,10 @@ type Controller struct {
 	itmsLister       cligolistersv1.ImageTagMirrorSetLister
 	itmsListerSynced cache.InformerSynced
 
+	criocpLister         cligolistersv1alpha1.CRIOCredentialProviderConfigLister
+	criocpListerSynced   cache.InformerSynced
+	addedCRIOCPObservers bool
+
 	configInformerFactory          configinformers.SharedInformerFactory
 	clusterImagePolicyLister       cligolistersv1.ClusterImagePolicyLister
 	clusterImagePolicyListerSynced cache.InformerSynced
@@ -120,8 +128,9 @@ type Controller struct {
 
 	fgHandler ctrlcommon.FeatureGatesHandler
 
-	queue    workqueue.TypedRateLimitingInterface[string]
-	imgQueue workqueue.TypedRateLimitingInterface[string]
+	queue       workqueue.TypedRateLimitingInterface[string]
+	imgQueue    workqueue.TypedRateLimitingInterface[string]
+	criocpQueue workqueue.TypedRateLimitingInterface[string]
 }
 
 // New returns a new container runtime config controller
@@ -133,6 +142,7 @@ func New(
 	imgInformer cligoinformersv1.ImageInformer,
 	idmsInformer cligoinformersv1.ImageDigestMirrorSetInformer,
 	itmsInformer cligoinformersv1.ImageTagMirrorSetInformer,
+	criocpInformer cligoinformersv1alpha1.CRIOCredentialProviderConfigInformer,
 	configInformerFactory configinformers.SharedInformerFactory,
 	icspInformer operatorinformersv1alpha1.ImageContentSourcePolicyInformer,
 	clusterVersionInformer cligoinformersv1.ClusterVersionInformer,
@@ -186,8 +196,15 @@ func New(
 		DeleteFunc: ctrl.itmsConfDeleted,
 	})
 
+	criocpInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    ctrl.criocpConfAdded,
+		UpdateFunc: ctrl.criocpConfUpdated,
+		DeleteFunc: ctrl.criocpConfDeleted,
+	})
+
 	ctrl.syncHandler = ctrl.syncContainerRuntimeConfig
 	ctrl.syncImgHandler = ctrl.syncImageConfig
+	ctrl.syncCRIOCPHandler = ctrl.syncCRIOCredentialProviderConfig
 	ctrl.enqueueContainerRuntimeConfig = ctrl.enqueue
 
 	ctrl.mcpLister = mcpInformer.Lister()
@@ -211,6 +228,9 @@ func New(
 	ctrl.itmsLister = itmsInformer.Lister()
 	ctrl.itmsListerSynced = itmsInformer.Informer().HasSynced
 
+	ctrl.criocpLister = criocpInformer.Lister()
+	ctrl.criocpListerSynced = criocpInformer.Informer().HasSynced
+
 	ctrl.clusterVersionLister = clusterVersionInformer.Lister()
 	ctrl.clusterVersionListerSynced = clusterVersionInformer.Informer().HasSynced
 
@@ -226,6 +246,7 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
 	defer ctrl.queue.ShutDown()
 	defer ctrl.imgQueue.ShutDown()
+	defer ctrl.criocpQueue.ShutDown()
 	listerCaches := []cache.InformerSynced{ctrl.mcpListerSynced, ctrl.mccrListerSynced, ctrl.ccListerSynced,
 		ctrl.imgListerSynced, ctrl.icspListerSynced, ctrl.idmsListerSynced, ctrl.itmsListerSynced, ctrl.clusterVersionListerSynced}
 
@@ -237,6 +258,14 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
 		ctrl.addedPolicyObservers = true
 	}
 
+	if ctrl.criocpEnabled() {
+		ctrl.addCRIOCPObservers()
+		klog.Info("added CRIOCredentialProviderConfig observers with CRIOCredentialProviderConfig featuregate enabled")
+		ctrl.configInformerFactory.Start(stopCh)
+		listerCaches = append(listerCaches, ctrl.criocpListerSynced)
+		ctrl.addedCRIOCPObservers = true
+	}
+
 	if !cache.WaitForCacheSync(stopCh, listerCaches...) {
 		return
 	}
@@ -251,6 +280,9 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
 	// Just need one worker for the image config
 	go wait.Until(ctrl.imgWorker, time.Second, stopCh)
 
+	// Just need one worker for the CRIOCredentialProviderConfig
+	go wait.Until(ctrl.criocpWorker, time.Second, stopCh)
+
 	<-stopCh
 }
 
@@ -312,6 +344,28 @@ func (ctrl *Controller) itmsConfDeleted(_ interface{}) {
 	ctrl.imgQueue.Add("openshift-config")
 }
 
+func (ctrl *Controller) addCRIOCPObservers() {
+	ctrl.configInformerFactory.Config().V1alpha1().CRIOCredentialProviderConfigs().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    ctrl.criocpConfAdded,
+		UpdateFunc: ctrl.criocpConfUpdated,
+		DeleteFunc: ctrl.criocpConfDeleted,
+	})
+	ctrl.criocpLister = ctrl.configInformerFactory.Config().V1alpha1().CRIOCredentialProviderConfigs().Lister()
+	ctrl.criocpListerSynced = ctrl.configInformerFactory.Config().V1alpha1().CRIOCredentialProviderConfigs().Informer().HasSynced
+}
+
+func (ctrl *Controller) criocpConfAdded(_ interface{}) {
+	ctrl.criocpQueue.Add("openshift-config")
+}
+
+func (ctrl *Controller) criocpConfUpdated(_, _ interface{}) {
+	ctrl.criocpQueue.Add("openshift-config")
+}
+
+func (ctrl *Controller) criocpConfDeleted(_ interface{}) {
+	ctrl.criocpQueue.Add("openshift-config")
+}
+
 func (ctrl *Controller) addImagePolicyObservers() {
 	ctrl.configInformerFactory.Config().V1().ClusterImagePolicies().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc:    ctrl.clusterImagePolicyAdded,
@@ -358,6 +412,10 @@ func (ctrl *Controller) sigstoreAPIEnabled() bool {
 	return ctrl.fgHandler.Enabled(features.FeatureGateSigstoreImageVerification)
 }
 
+func (ctrl *Controller) criocpEnabled() bool {
+	return ctrl.fgHandler.Enabled(features.FeatureGateCRIOCredentialProviderConfig)
+}
+
 func (ctrl *Controller) updateContainerRuntimeConfig(oldObj, newObj interface{}) {
 	oldCtrCfg := oldObj.(*mcfgv1.ContainerRuntimeConfig)
 	newCtrCfg := newObj.(*mcfgv1.ContainerRuntimeConfig)
@@ -437,6 +495,11 @@ func (ctrl *Controller) imgWorker() {
 	}
 }
 
+func (ctrl *Controller) criocpWorker() {
+	for ctrl.processNextCRIOCPWorkItem() {
+	}
+}
+
 func (ctrl *Controller) processNextWorkItem() bool {
 	key, quit := ctrl.queue.Get()
 	if quit {
@@ -463,6 +526,19 @@ func (ctrl *Controller) processNextImgWorkItem() bool {
 	return true
 }
 
+func (ctrl *Controller) processNextCRIOCPWorkItem() bool {
+	key, quit := ctrl.criocpQueue.Get()
+	if quit {
+		return false
+	}
+	defer ctrl.criocpQueue.Done(key)
+
+	err := ctrl.syncCRIOCPHandler(key)
+	ctrl.handleCRIOCPErr(err, key)
+
+	return true
+}
+
 func (ctrl *Controller) handleErr(err error, key string) {
 	if err == nil {
 		ctrl.queue.Forget(key)
@@ -499,6 +575,24 @@ func (ctrl *Controller) handleImgErr(err error, key string) {
 	ctrl.imgQueue.AddAfter(key, 1*time.Minute)
 }
 
+func (ctrl *Controller) handleCRIOCPErr(err error, key string) {
+	if err == nil {
+		ctrl.criocpQueue.Forget(key)
+		return
+	}
+
+	if ctrl.criocpQueue.NumRequeues(key) < maxRetries {
+		klog.V(2).Infof("Error syncing CRIOCredentialProviderConfig %v: %v", key, err)
+		ctrl.criocpQueue.AddRateLimited(key)
+		return
+	}
+
+	utilruntime.HandleError(err)
+	klog.V(2).Infof("Dropping CRIOCredentialProviderConfig %q out of the queue: %v", key, err)
+	ctrl.criocpQueue.Forget(key)
+	ctrl.criocpQueue.AddAfter(key, 1*time.Minute)
+}
+
 // generateOriginalContainerRuntimeConfigs returns rendered default storage, registries and policy config files
 func generateOriginalContainerRuntimeConfigs(templateDir string, cc *mcfgv1.ControllerConfig, role string) (*ign3types.File, *ign3types.File, *ign3types.File, error) {
 	// Render the default templates
@@ -546,6 +640,53 @@ func generateOriginalContainerRuntimeConfigs(templateDir string, cc *mcfgv1.Cont
 	return gmcStorageConfig, gmcRegistriesConfig, gmcPolicyJSON, nil
 }
 
+func generateOriginalCredentialProviderConfig(templateDir string, cc *mcfgv1.ControllerConfig, role string) (*ign3types.File, error) {
+
+	// Render the default templates
+	rc := &mtmpl.RenderConfig{
+		ControllerConfigSpec: &cc.Spec,
+	}
+	generatedConfigs, err := mtmpl.GenerateMachineConfigsForRole(rc, role, templateDir)
+	if err != nil {
+		return nil, fmt.Errorf("generateMachineConfigsforRole failed with error %w", err)
+	}
+	// Find generated provider.yaml
+	var (
+		config, gmcCredProviderConfig *ign3types.File
+		errCredProvider               error
+		credProviderConfigPath        string
+	)
+
+	// Determine credential provider config path based on platform
+	// staying consistent with path used in pkg/controller/template/render.go
+	credProviderConfigPathFormat := filepath.FromSlash("/etc/kubernetes/credential-providers/%s-credential-provider.yaml")
+	switch cc.Spec.Infra.Status.PlatformStatus.Type {
+	case apicfgv1.AWSPlatformType:
+		credProviderConfigPath = fmt.Sprintf(credProviderConfigPathFormat, "ecr")
+	case apicfgv1.GCPPlatformType:
+		credProviderConfigPath = fmt.Sprintf(credProviderConfigPathFormat, "gcr")
+	case apicfgv1.AzurePlatformType:
+		credProviderConfigPath = fmt.Sprintf(credProviderConfigPathFormat, "acr")
+	default:
+		return nil, fmt.Errorf("unsupported platform type: %s", cc.Spec.Infra.Status.PlatformStatus.Type)
+	}
+	klog.Infof("credential provider config path set to: %s", credProviderConfigPath)
+
+	// Find credential provider config
+	for _, gmc := range generatedConfigs {
+		config, errCredProvider = findCredProviderConfig(gmc, credProviderConfigPath)
+		if errCredProvider != nil {
+			klog.V(4).Infof("could not find credential provider config in generated config %s: %v", gmc.Name, errCredProvider)
+			return nil, fmt.Errorf("could not generate original credential provider configs: %w", errCredProvider)
+		}
+
+		gmcCredProviderConfig = config
+
+	}
+
+	return gmcCredProviderConfig, nil
+}
+
 func (ctrl *Controller) syncStatusOnly(cfg *mcfgv1.ContainerRuntimeConfig, err error, args ...interface{}) error {
 	statusUpdateErr := retry.RetryOnConflict(updateBackoff, func() error {
 		newcfg, getErr := ctrl.mccrLister.Get(cfg.Name)
@@ -958,6 +1099,45 @@ func (ctrl *Controller) syncImageConfig(key string) error {
 	return nil
 }
 
+func (ctrl *Controller) syncCRIOCredentialProviderConfig(key string) error {
+	startTime := time.Now()
+	klog.V(4).Infof("Started syncing CRIOCredentialProvider config %q (%v)", key, startTime)
+	defer func() {
+		klog.V(4).Infof("Finished syncing CRIOCredentialProvider config %q (%v)", key, time.Since(startTime))
+	}()
+
+	// Get ControllerConfig
+	controllerConfig, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName)
+	if err != nil {
+		return fmt.Errorf("could not get ControllerConfig %w", err)
+	}
+
+	sel, err := metav1.LabelSelectorAsSelector(metav1.AddLabelToSelector(&metav1.LabelSelector{}, builtInLabelKey, ""))
+	if err != nil {
+		return err
+	}
+	// Find all the MCO built in MachineConfigPools
+	mcpPools, err := ctrl.mcpLister.List(sel)
+	if err != nil {
+		return err
+	}
+
+	for _, pool := range mcpPools {
+		role := pool.Name
+		credProviderConfigIgn, err := generateOriginalCredentialProviderConfig(ctrl.templatesDir, controllerConfig, role)
+		if err != nil {
+			return fmt.Errorf("could not generate original CRIO credential provider config for role %s: %w", role, err)
+		}
+		contents, err := ctrlcommon.DecodeIgnitionFileContents(credProviderConfigIgn.Contents.Source, credProviderConfigIgn.Contents.Compression)
+		if err != nil {
+			return fmt.Errorf("could not decode CRIO credential provider config for role %s: %w", role, err)
+		}
+		klog.Infof("Decoded CRIO credential provider config contents successfully for role %s: %s", role, string(contents))
+	}
+
+	return nil
+}
+
 func (ctrl *Controller) syncIgnitionConfig(managedKey string, ignFile *ign3types.Config, pool *mcfgv1.MachineConfigPool, ownerRef metav1.OwnerReference) (bool, error) {
 	rawIgn, err := json.Marshal(ignFile)
 	if err != nil {

pkg/controller/container-runtime-config/helpers.go

@@ -200,6 +200,21 @@ func findPolicyJSON(mc *mcfgv1.MachineConfig) (*ign3types.File, error) {
 	return nil, fmt.Errorf("could not find Policy JSON")
 }
 
+func findCredProviderConfig(mc *mcfgv1.MachineConfig, credProviderConfigPath string) (*ign3types.File, error) {
+	ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw)
+	if err != nil {
+		return nil, fmt.Errorf("parsing Credential Provider Ignition config failed with error: %w", err)
+	}
+	for _, c := range ignCfg.Storage.Files {
+		klog.Infof("Checking file path : %s", c.Path)
+		if c.Path == credProviderConfigPath {
+			c := c
+			return &c, nil
+		}
+	}
+	return nil, fmt.Errorf("could not find Credential Provider Config")
+}
+
 // Deprecated: use getManagedKeyCtrCfg
 func getManagedKeyCtrCfgDeprecated(pool *mcfgv1.MachineConfigPool) string {
 	return fmt.Sprintf("99-%s-%s-containerruntime", pool.Name, pool.ObjectMeta.UID)