feat: ConfigMap image mapping overrides for LLS Distro

Implements a mechanism for the Llama Stack Operator to read and apply
LLS Distribution image updates from a ConfigMap, enabling independent patching
for security fixes or bug fixes without requiring a new LLS Operator.

- Add ImageMappingOverrides field to LlamaStackDistributionReconciler
- Implement parseImageMappingOverrides() to read image-overrides from ConfigMap
- Support symbolic name mapping (e.g., `starter`) to specific images
- Included unit tests

The operator now reads image overrides from the 'image-overrides' key in the
operator ConfigMap, supporting YAML format with version-to-image mappings.
Overrides take precedence over default distribution images and are refreshed
on each reconciler initialization.

Closes: RHAIENG-1079
Signed-off-by: Derek Higgins <derekh@redhat.com>

Author: derekhiggins

README.md

@@ -134,6 +134,35 @@ kubectl apply -f feature-flags.yaml
 Within the next reconciliation loop the operator will begin creating a `<name>-network-policy` resource for each distribution.
 Set `enabled: false` (or remove the block) to turn the feature back off; the operator will delete the previously managed policies.
 
+## Image Mapping Overrides
+
+The operator supports ConfigMap-driven image updates for LLS Distribution images. This allows independent patching for security fixes or bug fixes without requiring a new operator version.
+
+### Configuration
+
+Create or update the operator ConfigMap with an `image-overrides` key:
+
+```yaml
+
+  image-overrides: |
+    starter-gpu: quay.io/custom/llama-stack:starter-gpu
+    starter: quay.io/custom/llama-stack:starter
+```
+
+### Configuration Format
+
+Use the distribution name directly as the key (e.g., `starter-gpu`, `starter`). The operator will apply these overrides automatically
+
+### Example Usage
+
+To update the LLS Distribution image for all `starter` distributions:
+
+```bash
+kubectl patch configmap llama-stack-operator-config -n llama-stack-k8s-operator-system --type merge -p '{"data":{"image-overrides":"starter: quay.io/opendatahub/llama-stack:latest"}}'
+```
+
+This will cause all LlamaStackDistribution resources using the `starter` distribution to restart with the new image.
+
 ## Developer Guide
 
 ### Prerequisites

controllers/llamastackdistribution_controller.go

@@ -92,6 +92,8 @@ type LlamaStackDistributionReconciler struct {
 	Scheme *runtime.Scheme
 	// Feature flags
 	EnableNetworkPolicy bool
+	// Image mapping overrides
+	ImageMappingOverrides map[string]string
 	// Cluster info
 	ClusterInfo *cluster.ClusterInfo
 	httpClient  *http.Client
@@ -597,21 +599,40 @@ func (r *LlamaStackDistributionReconciler) configMapUpdatePredicate(e event.Upda
 		return false
 	}
 
-	// Parse the feature flags if the operator config ConfigMap has changed
+	// Check if this is the operator config ConfigMap
+	if r.handleOperatorConfigUpdate(newConfigMap) {
+		return true
+	}
+
+	// Handle referenced ConfigMap updates
+	return r.handleReferencedConfigMapUpdate(oldConfigMap, newConfigMap)
+}
+
+// handleOperatorConfigUpdate processes updates to the operator config ConfigMap.
+func (r *LlamaStackDistributionReconciler) handleOperatorConfigUpdate(configMap *corev1.ConfigMap) bool {
 	operatorNamespace, err := deploy.GetOperatorNamespace()
 	if err != nil {
 		return false
 	}
-	if newConfigMap.Name == operatorConfigData && newConfigMap.Namespace == operatorNamespace {
-		EnableNetworkPolicy, err := parseFeatureFlags(newConfigMap.Data)
-		if err != nil {
-			log.FromContext(context.Background()).Error(err, "Failed to parse feature flags")
-		} else {
-			r.EnableNetworkPolicy = EnableNetworkPolicy
-		}
-		return true
+
+	if configMap.Name != operatorConfigData || configMap.Namespace != operatorNamespace {
+		return false
+	}
+
+	// Update feature flags
+	EnableNetworkPolicy, err := parseFeatureFlags(configMap.Data)
+	if err != nil {
+		log.FromContext(context.Background()).Error(err, "Failed to parse feature flags")
+	} else {
+		r.EnableNetworkPolicy = EnableNetworkPolicy
 	}
 
+	r.ImageMappingOverrides = ParseImageMappingOverrides(configMap.Data)
+	return true
+}
+
+// handleReferencedConfigMapUpdate processes updates to referenced ConfigMaps.
+func (r *LlamaStackDistributionReconciler) handleReferencedConfigMapUpdate(oldConfigMap, newConfigMap *corev1.ConfigMap) bool {
 	// Only proceed if this ConfigMap is referenced by any LlamaStackDistribution
 	if !r.isConfigMapReferenced(newConfigMap) {
 		return false
@@ -783,7 +804,7 @@ func (r *LlamaStackDistributionReconciler) findLlamaStackDistributionsForConfigM
 
 	operatorNamespace, err := deploy.GetOperatorNamespace()
 	if err != nil {
-		log.FromContext(context.Background()).Error(err, "Failed to get operator namespace for config map event processing")
+		logger.Error(err, "Failed to get operator namespace for config map event processing")
 		return nil
 	}
 	// If the operator config was changed, we reconcile all LlamaStackDistributions
@@ -1672,53 +1693,92 @@ func NewLlamaStackDistributionReconciler(ctx context.Context, client client.Clie
 		return nil, fmt.Errorf("failed to get operator namespace: %w", err)
 	}
 
-	// Get the ConfigMap
-	// If the ConfigMap doesn't exist, create it with default feature flags
-	// If the ConfigMap exists, parse the feature flags from the Configmap
+	// Initialize operator config ConfigMap
+	configMap, err := initializeOperatorConfigMap(ctx, client, operatorNamespace)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse feature flags from ConfigMap
+	enableNetworkPolicy, err := parseFeatureFlags(configMap.Data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse feature flags: %w", err)
+	}
+
+	// Parse image mapping overrides from ConfigMap
+	imageMappingOverrides := ParseImageMappingOverrides(configMap.Data)
+
+	return &LlamaStackDistributionReconciler{
+		Client:                client,
+		Scheme:                scheme,
+		EnableNetworkPolicy:   enableNetworkPolicy,
+		ImageMappingOverrides: imageMappingOverrides,
+		ClusterInfo:           clusterInfo,
+		httpClient:            &http.Client{Timeout: 5 * time.Second},
+	}, nil
+}
+
+// initializeOperatorConfigMap gets or creates the operator config ConfigMap.
+func initializeOperatorConfigMap(ctx context.Context, c client.Client, operatorNamespace string) (*corev1.ConfigMap, error) {
 	configMap := &corev1.ConfigMap{}
 	configMapName := types.NamespacedName{
 		Name:      operatorConfigData,
 		Namespace: operatorNamespace,
 	}
 
-	if err = client.Get(ctx, configMapName, configMap); err != nil {
-		if !k8serrors.IsNotFound(err) {
-			return nil, fmt.Errorf("failed to get ConfigMap: %w", err)
-		}
+	err := c.Get(ctx, configMapName, configMap)
+	if err == nil {
+		return configMap, nil
+	}
 
-		// ConfigMap doesn't exist, create it with defaults
-		configMap, err = createDefaultConfigMap(configMapName)
-		if err != nil {
-			return nil, fmt.Errorf("failed to generate default configMap: %w", err)
+	if !k8serrors.IsNotFound(err) {
+		return nil, fmt.Errorf("failed to get ConfigMap: %w", err)
+	}
+
+	// ConfigMap doesn't exist, create it with defaults
+	configMap, err = createDefaultConfigMap(configMapName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate default configMap: %w", err)
+	}
+
+	if err = c.Create(ctx, configMap); err != nil {
+		return nil, fmt.Errorf("failed to create ConfigMap: %w", err)
+	}
+
+	return configMap, nil
+}
+
+func ParseImageMappingOverrides(configMapData map[string]string) map[string]string {
+	imageMappingOverrides := make(map[string]string)
+
+	// Look for the image-overrides key in the ConfigMap data
+	if overridesYAML, exists := configMapData["image-overrides"]; exists {
+		// Parse the YAML content
+		var overrides map[string]string
+		if err := yaml.Unmarshal([]byte(overridesYAML), &overrides); err != nil {
+			// Log error but continue with empty overrides
+			fmt.Printf("failed to parse image-overrides YAML: %v\n", err)
+			return imageMappingOverrides
 		}
 
-		if err = client.Create(ctx, configMap); err != nil {
-			return nil, fmt.Errorf("failed to create ConfigMap: %w", err)
+		// Copy the parsed overrides to our result map
+		for version, image := range overrides {
+			imageMappingOverrides[version] = image
 		}
 	}
 
-	// Parse feature flags from ConfigMap
-	enableNetworkPolicy, err := parseFeatureFlags(configMap.Data)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse feature flags: %w", err)
-	}
-	return &LlamaStackDistributionReconciler{
-		Client:              client,
-		Scheme:              scheme,
-		EnableNetworkPolicy: enableNetworkPolicy,
-		ClusterInfo:         clusterInfo,
-		httpClient:          &http.Client{Timeout: 5 * time.Second},
-	}, nil
+	return imageMappingOverrides
 }
 
 // NewTestReconciler creates a reconciler for testing, allowing injection of a custom http client and feature flags.
 func NewTestReconciler(client client.Client, scheme *runtime.Scheme, clusterInfo *cluster.ClusterInfo,
 	httpClient *http.Client, enableNetworkPolicy bool) *LlamaStackDistributionReconciler {
 	return &LlamaStackDistributionReconciler{
-		Client:              client,
-		Scheme:              scheme,
-		ClusterInfo:         clusterInfo,
-		httpClient:          httpClient,
-		EnableNetworkPolicy: enableNetworkPolicy,
+		Client:                client,
+		Scheme:                scheme,
+		ClusterInfo:           clusterInfo,
+		httpClient:            httpClient,
+		EnableNetworkPolicy:   enableNetworkPolicy,
+		ImageMappingOverrides: make(map[string]string),
 	}
 }

controllers/llamastackdistribution_controller_test.go

@@ -796,3 +796,161 @@ InvalidCertificateDataThatIsNotValidX509
 			"error should indicate X.509 parsing failure")
 	})
 }
+
+func TestParseImageMappingOverrides_SingleOverride(t *testing.T) {
+	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
+
+	// Test data with single override
+	configMapData := map[string]string{
+		"image-overrides": "starter: quay.io/custom/llama-stack:starter",
+	}
+
+	// Call the function
+	result := controllers.ParseImageMappingOverrides(configMapData)
+
+	// Assertions
+	require.Len(t, result, 1, "Should have exactly one override")
+	require.Equal(t, "quay.io/custom/llama-stack:starter", result["starter"], "Override should match expected value")
+}
+
+func TestParseImageMappingOverrides_InvalidYAML(t *testing.T) {
+	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
+
+	// Test data with invalid YAML
+	configMapData := map[string]string{
+		"image-overrides": "invalid: yaml: content: [",
+	}
+
+	// Call the function
+	result := controllers.ParseImageMappingOverrides(configMapData)
+
+	// Assertions - should return empty map on error
+	require.Empty(t, result, "Should return empty map when YAML is invalid")
+}
+
+func TestNewLlamaStackDistributionReconciler_WithImageOverrides(t *testing.T) {
+	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
+
+	// Create operator namespace
+	operatorNamespace := createTestNamespace(t, "llama-stack-k8s-operator-system")
+	t.Setenv("OPERATOR_NAMESPACE", operatorNamespace.Name)
+
+	// Create test ConfigMap with image overrides
+	configMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "llama-stack-operator-config",
+			Namespace: operatorNamespace.Name,
+		},
+		Data: map[string]string{
+			"image-overrides": "starter: quay.io/custom/llama-stack:starter",
+			"featureFlags": `enableNetworkPolicy:
+    enabled: false`,
+		},
+	}
+	require.NoError(t, k8sClient.Create(t.Context(), configMap))
+
+	// Create test cluster info
+	clusterInfo := &cluster.ClusterInfo{
+		OperatorNamespace:  operatorNamespace.Name,
+		DistributionImages: map[string]string{"starter": "default-image"},
+	}
+
+	// Call the function
+	reconciler, err := controllers.NewLlamaStackDistributionReconciler(
+		t.Context(),
+		k8sClient,
+		scheme.Scheme,
+		clusterInfo,
+	)
+
+	// Assertions
+	require.NoError(t, err, "Should create reconciler successfully")
+	require.NotNil(t, reconciler, "Reconciler should not be nil")
+	require.Len(t, reconciler.ImageMappingOverrides, 1, "Should have one image override")
+	require.Equal(t, "quay.io/custom/llama-stack:starter",
+		reconciler.ImageMappingOverrides["starter"], "Override should match expected value")
+	require.False(t, reconciler.EnableNetworkPolicy, "Network policy should be disabled")
+}
+
+func TestConfigMapUpdateTriggersReconciliation(t *testing.T) {
+	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
+
+	// Create test namespace
+	namespace := createTestNamespace(t, "test-configmap-update")
+	operatorNamespace := createTestNamespace(t, "llama-stack-k8s-operator-system")
+	t.Setenv("OPERATOR_NAMESPACE", operatorNamespace.Name)
+
+	// Create initial ConfigMap
+	configMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "llama-stack-operator-config",
+			Namespace: operatorNamespace.Name,
+		},
+		Data: map[string]string{
+			"featureFlags": `enableNetworkPolicy:
+    enabled: false`,
+		},
+	}
+	require.NoError(t, k8sClient.Create(t.Context(), configMap))
+
+	// Create LlamaStackDistribution instance using starter
+	instance := NewDistributionBuilder().
+		WithName("test-configmap-update").
+		WithNamespace(namespace.Name).
+		WithDistribution("starter").
+		Build()
+	require.NoError(t, k8sClient.Create(t.Context(), instance))
+
+	// Create reconciler with initial overrides
+	clusterInfo := &cluster.ClusterInfo{
+		OperatorNamespace:  operatorNamespace.Name,
+		DistributionImages: map[string]string{"starter": "default-starter-image"},
+	}
+
+	reconciler, err := controllers.NewLlamaStackDistributionReconciler(
+		t.Context(),
+		k8sClient,
+		scheme.Scheme,
+		clusterInfo,
+	)
+	require.NoError(t, err)
+
+	// Initial reconciliation
+	_, err = reconciler.Reconcile(t.Context(), ctrl.Request{
+		NamespacedName: types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace},
+	})
+	require.NoError(t, err)
+
+	// Get initial deployment and verify it uses the first override
+	deployment := &appsv1.Deployment{}
+	waitForResource(t, k8sClient, instance.Namespace, instance.Name, deployment)
+	initialImage := deployment.Spec.Template.Spec.Containers[0].Image
+	require.Equal(t, "default-starter-image", initialImage,
+		"Initial deployment should use distribution image")
+
+	// Update ConfigMap with new overrides
+	configMap.Data["image-overrides"] = "starter: quay.io/custom/llama-stack:starter"
+	require.NoError(t, k8sClient.Update(t.Context(), configMap))
+
+	// Simulate ConfigMap update by recreating reconciler (in real scenario this would be triggered by watch)
+	updatedReconciler, err := controllers.NewLlamaStackDistributionReconciler(
+		t.Context(),
+		k8sClient,
+		scheme.Scheme,
+		clusterInfo,
+	)
+	require.NoError(t, err)
+
+	// Reconcile with updated overrides
+	_, err = updatedReconciler.Reconcile(t.Context(), ctrl.Request{
+		NamespacedName: types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace},
+	})
+	require.NoError(t, err)
+
+	// Verify deployment was updated with new image
+	waitForResourceWithKeyAndCondition(
+		t, k8sClient, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace},
+		deployment, func() bool {
+			return deployment.Spec.Template.Spec.Containers[0].Image == "quay.io/custom/llama-stack:starter"
+		}, "Deployment should be updated with new image")
+}