feat: ConfigMap image mapping overrides for LLS Distro

Implements a mechanism for the Llama Stack Operator to read and apply
LLS Distribution image updates from a ConfigMap, enabling independent patching
for security fixes or bug fixes without requiring a new LLS Operator.

- Add ImageMappingOverrides field to LlamaStackDistributionReconciler
- Implement parseImageMappingOverrides() to read image-overrides from ConfigMap
- Support symbolic name mapping (e.g., `starter`) to specific images
- Included unit tests

The operator now reads image overrides from the 'image-overrides' key in the
operator ConfigMap, supporting YAML format with version-to-image mappings.
Overrides take precedence over default distribution images and are refreshed
on each reconciler initialization.

Closes: RHAIENG-1079
Signed-off-by: Derek Higgins <derekh@redhat.com>

Author: derekhiggins

README.md

@@ -134,6 +134,35 @@ kubectl apply -f feature-flags.yaml
 Within the next reconciliation loop the operator will begin creating a `<name>-network-policy` resource for each distribution.
 Set `enabled: false` (or remove the block) to turn the feature back off; the operator will delete the previously managed policies.
 
+## Image Mapping Overrides
+
+The operator supports ConfigMap-driven image updates for LLS Distribution images. This allows independent patching for security fixes or bug fixes without requiring a new operator version.
+
+### Configuration
+
+Create or update the operator ConfigMap with an `image-overrides` key:
+
+```yaml
+
+  image-overrides: |
+    starter-gpu: quay.io/custom/llama-stack:starter-gpu
+    starter: quay.io/custom/llama-stack:starter
+```
+
+### Configuration Format
+
+Use the distribution name directly as the key (e.g., `starter-gpu`, `starter`). The operator will apply these overrides automatically
+
+### Example Usage
+
+To update the LLS Distribution image for all `starter` distributions:
+
+```bash
+kubectl patch configmap llama-stack-operator-config -n llama-stack-k8s-operator-system --type merge -p '{"data":{"image-overrides":"starter: quay.io/opendatahub/llama-stack:latest"}}'
+```
+
+This will cause all LlamaStackDistribution resources using the `starter` distribution to restart with the new image.
+
 ## Developer Guide
 
 ### Prerequisites

controllers/llamastackdistribution_controller.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-containerregistry/pkg/name"
 	llamav1alpha1 "github.com/llamastack/llama-stack-k8s-operator/api/v1alpha1"
 	"github.com/llamastack/llama-stack-k8s-operator/pkg/cluster"
 	"github.com/llamastack/llama-stack-k8s-operator/pkg/deploy"
@@ -92,6 +93,8 @@ type LlamaStackDistributionReconciler struct {
 	Scheme *runtime.Scheme
 	// Feature flags
 	EnableNetworkPolicy bool
+	// Image mapping overrides
+	ImageMappingOverrides map[string]string
 	// Cluster info
 	ClusterInfo *cluster.ClusterInfo
 	httpClient  *http.Client
@@ -597,21 +600,40 @@ func (r *LlamaStackDistributionReconciler) configMapUpdatePredicate(e event.Upda
 		return false
 	}
 
-	// Parse the feature flags if the operator config ConfigMap has changed
+	// Check if this is the operator config ConfigMap
+	if r.handleOperatorConfigUpdate(newConfigMap) {
+		return true
+	}
+
+	// Handle referenced ConfigMap updates
+	return r.handleReferencedConfigMapUpdate(oldConfigMap, newConfigMap)
+}
+
+// handleOperatorConfigUpdate processes updates to the operator config ConfigMap.
+func (r *LlamaStackDistributionReconciler) handleOperatorConfigUpdate(configMap *corev1.ConfigMap) bool {
 	operatorNamespace, err := deploy.GetOperatorNamespace()
 	if err != nil {
 		return false
 	}
-	if newConfigMap.Name == operatorConfigData && newConfigMap.Namespace == operatorNamespace {
-		EnableNetworkPolicy, err := parseFeatureFlags(newConfigMap.Data)
-		if err != nil {
-			log.FromContext(context.Background()).Error(err, "Failed to parse feature flags")
-		} else {
-			r.EnableNetworkPolicy = EnableNetworkPolicy
-		}
-		return true
+
+	if configMap.Name != operatorConfigData || configMap.Namespace != operatorNamespace {
+		return false
 	}
 
+	// Update feature flags
+	EnableNetworkPolicy, err := parseFeatureFlags(configMap.Data)
+	if err != nil {
+		log.FromContext(context.Background()).Error(err, "Failed to parse feature flags")
+	} else {
+		r.EnableNetworkPolicy = EnableNetworkPolicy
+	}
+
+	r.ImageMappingOverrides = ParseImageMappingOverrides(configMap.Data)
+	return true
+}
+
+// handleReferencedConfigMapUpdate processes updates to referenced ConfigMaps.
+func (r *LlamaStackDistributionReconciler) handleReferencedConfigMapUpdate(oldConfigMap, newConfigMap *corev1.ConfigMap) bool {
 	// Only proceed if this ConfigMap is referenced by any LlamaStackDistribution
 	if !r.isConfigMapReferenced(newConfigMap) {
 		return false
@@ -783,7 +805,7 @@ func (r *LlamaStackDistributionReconciler) findLlamaStackDistributionsForConfigM
 
 	operatorNamespace, err := deploy.GetOperatorNamespace()
 	if err != nil {
-		log.FromContext(context.Background()).Error(err, "Failed to get operator namespace for config map event processing")
+		logger.Error(err, "Failed to get operator namespace for config map event processing")
 		return nil
 	}
 	// If the operator config was changed, we reconcile all LlamaStackDistributions
@@ -1672,53 +1694,103 @@ func NewLlamaStackDistributionReconciler(ctx context.Context, client client.Clie
 		return nil, fmt.Errorf("failed to get operator namespace: %w", err)
 	}
 
-	// Get the ConfigMap
-	// If the ConfigMap doesn't exist, create it with default feature flags
-	// If the ConfigMap exists, parse the feature flags from the Configmap
+	// Initialize operator config ConfigMap
+	configMap, err := initializeOperatorConfigMap(ctx, client, operatorNamespace)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse feature flags from ConfigMap
+	enableNetworkPolicy, err := parseFeatureFlags(configMap.Data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse feature flags: %w", err)
+	}
+
+	// Parse image mapping overrides from ConfigMap
+	imageMappingOverrides := ParseImageMappingOverrides(configMap.Data)
+
+	return &LlamaStackDistributionReconciler{
+		Client:                client,
+		Scheme:                scheme,
+		EnableNetworkPolicy:   enableNetworkPolicy,
+		ImageMappingOverrides: imageMappingOverrides,
+		ClusterInfo:           clusterInfo,
+		httpClient:            &http.Client{Timeout: 5 * time.Second},
+	}, nil
+}
+
+// initializeOperatorConfigMap gets or creates the operator config ConfigMap.
+func initializeOperatorConfigMap(ctx context.Context, c client.Client, operatorNamespace string) (*corev1.ConfigMap, error) {
 	configMap := &corev1.ConfigMap{}
 	configMapName := types.NamespacedName{
 		Name:      operatorConfigData,
 		Namespace: operatorNamespace,
 	}
 
-	if err = client.Get(ctx, configMapName, configMap); err != nil {
-		if !k8serrors.IsNotFound(err) {
-			return nil, fmt.Errorf("failed to get ConfigMap: %w", err)
-		}
+	err := c.Get(ctx, configMapName, configMap)
+	if err == nil {
+		return configMap, nil
+	}
 
-		// ConfigMap doesn't exist, create it with defaults
-		configMap, err = createDefaultConfigMap(configMapName)
-		if err != nil {
-			return nil, fmt.Errorf("failed to generate default configMap: %w", err)
+	if !k8serrors.IsNotFound(err) {
+		return nil, fmt.Errorf("failed to get ConfigMap: %w", err)
+	}
+
+	// ConfigMap doesn't exist, create it with defaults
+	configMap, err = createDefaultConfigMap(configMapName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate default configMap: %w", err)
+	}
+
+	if err = c.Create(ctx, configMap); err != nil {
+		return nil, fmt.Errorf("failed to create ConfigMap: %w", err)
+	}
+
+	return configMap, nil
+}
+
+func ParseImageMappingOverrides(configMapData map[string]string) map[string]string {
+	imageMappingOverrides := make(map[string]string)
+	logger := log.FromContext(context.Background())
+
+	// Look for the image-overrides key in the ConfigMap data
+	if overridesYAML, exists := configMapData["image-overrides"]; exists {
+		// Parse the YAML content
+		var overrides map[string]string
+		if err := yaml.Unmarshal([]byte(overridesYAML), &overrides); err != nil {
+			// Log error but continue with empty overrides
+			logger.V(1).Info("failed to parse image-overrides YAML", "error", err)
+			return imageMappingOverrides
 		}
 
-		if err = client.Create(ctx, configMap); err != nil {
-			return nil, fmt.Errorf("failed to create ConfigMap: %w", err)
+		// Validate and copy the parsed overrides to our result map
+		for version, image := range overrides {
+			// Validate the image reference format
+			if _, err := name.ParseReference(image); err != nil {
+				logger.V(1).Info(
+					"skipping invalid image override",
+					"version", version,
+					"image", image,
+					"error", err,
+				)
+				continue
+			}
+			imageMappingOverrides[version] = image
 		}
 	}
 
-	// Parse feature flags from ConfigMap
-	enableNetworkPolicy, err := parseFeatureFlags(configMap.Data)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse feature flags: %w", err)
-	}
-	return &LlamaStackDistributionReconciler{
-		Client:              client,
-		Scheme:              scheme,
-		EnableNetworkPolicy: enableNetworkPolicy,
-		ClusterInfo:         clusterInfo,
-		httpClient:          &http.Client{Timeout: 5 * time.Second},
-	}, nil
+	return imageMappingOverrides
 }
 
 // NewTestReconciler creates a reconciler for testing, allowing injection of a custom http client and feature flags.
 func NewTestReconciler(client client.Client, scheme *runtime.Scheme, clusterInfo *cluster.ClusterInfo,
 	httpClient *http.Client, enableNetworkPolicy bool) *LlamaStackDistributionReconciler {
 	return &LlamaStackDistributionReconciler{
-		Client:              client,
-		Scheme:              scheme,
-		ClusterInfo:         clusterInfo,
-		httpClient:          httpClient,
-		EnableNetworkPolicy: enableNetworkPolicy,
+		Client:                client,
+		Scheme:                scheme,
+		ClusterInfo:           clusterInfo,
+		httpClient:            httpClient,
+		EnableNetworkPolicy:   enableNetworkPolicy,
+		ImageMappingOverrides: make(map[string]string),
 	}
 }