TEST - will close immediately

.github/hack/cleanup-odh.sh

@@ -1,14 +1,16 @@
 #!/bin/bash
 #
-# cleanup-odh.sh - Remove OpenDataHub operator and all related resources
+# cleanup-odh.sh - Remove OpenDataHub/RHOAI MaaS resources and related operators
 #
 # This script removes:
 # - DataScienceCluster and DSCInitialization custom resources
 # - ODH operator Subscription and CSV
 # - Custom CatalogSource (odh-custom-catalog)
 # - ODH operator namespace (odh-operator)
 # - OpenDataHub application namespace (opendatahub)
+# - MaaS resources from RHOAI namespace (redhat-ods-applications)
 # - MaaS subscription namespace (models-as-a-service)
+# - Policy engine artifacts (Kuadrant/RHCL OLM resources, AuthConfig CRs)
 # - Keycloak identity provider (if deployed)
 # - ODH CRDs (optional)
 #
@@ -40,6 +42,19 @@ fi
 echo "Connected to cluster. Starting cleanup..."
 echo ""
 
+# Detect operator type to find the right application namespace
+MAAS_APP_NAMESPACE=""
+if kubectl get subscription rhods-operator -n redhat-ods-operator &>/dev/null; then
+    MAAS_APP_NAMESPACE="redhat-ods-applications"
+    echo "Detected RHOAI operator (application namespace: $MAAS_APP_NAMESPACE)"
+elif kubectl get subscription opendatahub-operator -A &>/dev/null; then
+    MAAS_APP_NAMESPACE="opendatahub"
+    echo "Detected ODH operator (application namespace: $MAAS_APP_NAMESPACE)"
+else
+    echo "No operator detected, will clean both namespaces"
+fi
+echo ""
+
 # 1. Delete DataScienceCluster instances
 echo "1. Deleting DataScienceCluster instances..."
 kubectl delete datasciencecluster --all -A --ignore-not-found --timeout=120s 2>/dev/null || true
@@ -82,6 +97,41 @@ kubectl delete ns odh-operator --ignore-not-found --timeout=120s 2>/dev/null ||
 echo "8. Deleting opendatahub namespace..."
 kubectl delete ns opendatahub --ignore-not-found --timeout=120s 2>/dev/null || true
 
+# 8b. Clean MaaS resources from RHOAI application namespace
+# On RHOAI clusters, MaaS resources live in redhat-ods-applications which is
+# operator-managed. We delete MaaS resources individually instead of the namespace.
+cleanup_maas_resources() {
+    local ns=$1
+    if ! kubectl get namespace "$ns" &>/dev/null; then
+        echo "   $ns not found, skipping"
+        return 0
+    fi
+
+    echo "   Cleaning MaaS resources from $ns..."
+    kubectl delete deployment maas-api maas-controller postgres -n "$ns" --ignore-not-found 2>/dev/null || true
+    kubectl delete service maas-api postgres -n "$ns" --ignore-not-found 2>/dev/null || true
+    kubectl delete secret maas-db-config postgres-creds -n "$ns" --ignore-not-found 2>/dev/null || true
+    kubectl delete authpolicy maas-api-auth-policy -n "$ns" --ignore-not-found 2>/dev/null || true
+    kubectl delete httproute maas-api-route -n "$ns" --ignore-not-found 2>/dev/null || true
+    kubectl delete destinationrule maas-api-backend-tls -n "$ns" --ignore-not-found 2>/dev/null || true
+    kubectl delete networkpolicy maas-api-cleanup-restrict maas-authorino-allow -n "$ns" --ignore-not-found 2>/dev/null || true
+    kubectl delete cronjob maas-api-key-cleanup -n "$ns" --ignore-not-found 2>/dev/null || true
+    kubectl delete role maas-api-db-secret maas-controller-leader-election-role -n "$ns" --ignore-not-found 2>/dev/null || true
+    kubectl delete rolebinding maas-api-db-secret maas-controller-leader-election-rolebinding -n "$ns" --ignore-not-found 2>/dev/null || true
+    kubectl delete serviceaccount maas-api maas-controller -n "$ns" --ignore-not-found 2>/dev/null || true
+    echo "   ✅ MaaS resources cleaned from $ns"
+}
+
+if [[ "$MAAS_APP_NAMESPACE" == "redhat-ods-applications" ]]; then
+    echo "8b. Cleaning MaaS resources from RHOAI namespace..."
+    cleanup_maas_resources "redhat-ods-applications"
+elif [[ -z "$MAAS_APP_NAMESPACE" ]]; then
+    # No operator detected, clean both just in case
+    echo "8b. Cleaning MaaS resources from both possible namespaces..."
+    cleanup_maas_resources "redhat-ods-applications"
+    cleanup_maas_resources "opendatahub"
+fi
+
 force_delete_namespace() {
     local ns=$1
     shift
@@ -172,6 +222,11 @@ if kubectl get namespace rh-connectivity-link &>/dev/null; then
     echo "   ✅ RHCL OLM resources cleaned"
 fi
 
+# 11b. Delete AuthConfig CRs cluster-wide
+# Old AuthConfig CRs can block new policy engine installs if the CRD schema changes.
+echo "11b. Deleting AuthConfig CRs..."
+kubectl delete authconfig --all --all-namespaces --ignore-not-found 2>/dev/null || true
+
 # 12. Delete policy engine namespaces (Kuadrant or RHCL)
 for policy_ns in kuadrant-system rh-connectivity-link; do
     echo "12. Deleting $policy_ns namespace (if installed)..."
@@ -210,21 +265,30 @@ kubectl delete envoyfilter kuadrant-auth-tls-fix -n openshift-ingress --ignore-n
 kubectl delete authpolicy -n openshift-ingress --all --ignore-not-found 2>/dev/null || true
 kubectl delete ratelimitpolicy -n openshift-ingress --all --ignore-not-found 2>/dev/null || true
 kubectl delete tokenratelimitpolicy -n openshift-ingress --all --ignore-not-found 2>/dev/null || true
+kubectl delete gatewayclass openshift-default --ignore-not-found 2>/dev/null || true
 
 # 16. Delete MaaS RBAC (ClusterRoles, ClusterRoleBindings - can conflict with other managers)
 echo "16. Deleting MaaS RBAC..."
 kubectl delete clusterrolebinding maas-api maas-controller-rolebinding --ignore-not-found 2>/dev/null || true
 kubectl delete clusterrole maas-api maas-controller-role --ignore-not-found 2>/dev/null || true
 
-# 17. Optionally delete CRDs
+# 17. Delete CRDs
+# Always delete KServe/MaaS CRDs to prevent storedVersions schema conflicts on reinstall.
+# ODH-internal CRDs are only deleted with --include-crds.
+echo "17. Deleting KServe/MaaS CRDs (always removed to prevent version conflicts)..."
+for crd in $(kubectl get crd -o name 2>/dev/null | grep -E 'serving\.kserve\.io|maas\.opendatahub\.io'); do
+    echo "   Deleting $crd"
+    kubectl delete "$crd" --ignore-not-found --timeout=30s 2>/dev/null || true
+done
+
 if $INCLUDE_CRDS; then
-    echo "17. Deleting ODH CRDs..."
-    kubectl delete crd datascienceclusters.datasciencecluster.opendatahub.io --ignore-not-found 2>/dev/null || true
-    kubectl delete crd dscinitializations.dscinitialization.opendatahub.io --ignore-not-found 2>/dev/null || true
-    kubectl delete crd datasciencepipelinesapplications.datasciencepipelinesapplications.opendatahub.io --ignore-not-found 2>/dev/null || true
-    # Add more CRDs as needed
+    echo "17b. Deleting all ODH CRDs..."
+    for crd in $(kubectl get crd -o name 2>/dev/null | grep -E 'opendatahub\.io|trustyai\.opendatahub'); do
+        echo "   Deleting $crd"
+        kubectl delete "$crd" --ignore-not-found --timeout=30s 2>/dev/null || true
+    done
 else
-    echo "17. Skipping CRD deletion (use --include-crds to remove CRDs)"
+    echo "17b. Skipping ODH-internal CRD deletion (use --include-crds to remove all)"
 fi
 
 echo ""
@@ -233,4 +297,5 @@ echo ""
 echo "Verify cleanup with:"
 echo "  kubectl get subscription -A | grep -i odh"
 echo "  kubectl get csv -A | grep -i odh"
-echo "  kubectl get ns | grep -E 'odh|opendatahub|models-as-a-service|kuadrant|rh-connectivity-link|keycloak-system|llm'"
+echo "  kubectl get ns | grep -E 'odh|opendatahub|models-as-a-service|kuadrant|rh-connectivity-link|keycloak-system|llm'
+  kubectl get deployment maas-api maas-controller postgres -n redhat-ods-applications 2>/dev/null || echo '  (no MaaS resources in redhat-ods-applications)'"

.github/hack/install-odh.sh

@@ -12,6 +12,8 @@
 #   OPERATOR_INSTALL_PLAN_APPROVAL - Manual (default) or Automatic; use "-" to omit.
 #     Manual: blocks auto-upgrades; this script auto-approves only the first InstallPlan so install does not stall.
 #   OPERATOR_IMAGE   - Custom operator image to patch into CSV (optional)
+#   OPERATOR_OPERANDS_MAP - Path to operands-map.yaml for RELATED_IMAGE env var injection (optional)
+#                           Used with OPERATOR_IMAGE to ensure component images match the operator.
 #
 # Usage: ./install-odh.sh
 
@@ -59,6 +61,51 @@ patch_operator_csv_if_needed() {
     {\"op\": \"replace\", \"path\": \"/spec/install/spec/deployments/0/spec/template/spec/containers/0/image\", \"value\": \"$OPERATOR_IMAGE\"}
   ]"
   log_info "CSV $csv_name patched with image $OPERATOR_IMAGE"
+
+  # When using a custom operator image, the community CSV may lack RELATED_IMAGE env vars
+  # that the operator needs to deploy the correct component versions.
+  # If OPERATOR_OPERANDS_MAP points to a local operands-map.yaml, inject its env vars into the CSV.
+  if [[ -n "${OPERATOR_OPERANDS_MAP:-}" && -f "$OPERATOR_OPERANDS_MAP" ]]; then
+    log_info "Injecting RELATED_IMAGE env vars from $OPERATOR_OPERANDS_MAP into CSV"
+    local env_patches="["
+    local first=true
+    while IFS= read -r line; do
+      local name value
+      name=$(echo "$line" | sed -n 's/.*name: \(RELATED_IMAGE_[^ ]*\)/\1/p')
+      if [[ -n "$name" ]]; then
+        read -r value_line
+        value=$(echo "$value_line" | sed -n 's/.*value: \(.*\)/\1/p')
+        if [[ -n "$value" ]]; then
+          $first || env_patches+=","
+          first=false
+          env_patches+="{\"name\":\"$name\",\"value\":\"$value\"}"
+        fi
+      fi
+    done < "$OPERATOR_OPERANDS_MAP"
+
+    if [[ "$env_patches" != "[" ]]; then
+      env_patches+="]"
+      local container_path="/spec/install/spec/deployments/0/spec/template/spec/containers/0"
+      local existing_env
+      existing_env=$(kubectl get csv "$csv_name" -n "$namespace" -o jsonpath="{${container_path}.env}" 2>/dev/null || echo "[]")
+
+      local merged_env
+      merged_env=$(python3 -c "
+import json, sys
+existing = json.loads('${existing_env}')
+new_envs = json.loads(sys.stdin.read())
+existing_names = {e['name'] for e in existing}
+for e in new_envs:
+    if e['name'] not in existing_names:
+        existing.append(e)
+print(json.dumps(existing))
+" <<< "$env_patches")
+
+      kubectl patch csv "$csv_name" -n "$namespace" --type='json' \
+        -p="[{\"op\": \"replace\", \"path\": \"${container_path}/env\", \"value\": ${merged_env}}]"
+      log_info "CSV env vars patched with RELATED_IMAGE entries"
+    fi
+  fi
 }
 
 echo "=== Installing OpenDataHub operator ==="