feat(RHOAIENG-29330):Deny RayCluster creation with Ray Version mismatches

Author: LilyLinh

docs/sphinx/user-docs/cluster-configuration.rst

@@ -44,6 +44,79 @@ requirements for creating the Ray Cluster.
   documentation on building a custom image
   `here <https://github.com/opendatahub-io/distributed-workloads/tree/main/images/runtime/examples>`__.
 
+Ray Version Compatibility
+-------------------------
+
+ The CodeFlare SDK requires that the Ray version in your runtime image matches the Ray version used by the SDK itself.When you specify a custom runtime image, the SDK will automatically validate that the Ray version in the image matches this requirement.
+
+Version Validation Behavior
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The SDK performs the following validation when creating a cluster:
+
+1. **Compatible versions**: If the runtime image contains Ray 2.47.1, the cluster will be created successfully.
+
+2. **Version mismatch**: If the runtime image contains a different Ray version, cluster creation will fail with a detailed error message explaining the mismatch.
+
+3. **Unknown versions**: If the SDK cannot determine the Ray version from the image name (e.g., SHA-based tags), a warning will be issued but cluster creation will continue.
+
+Examples
+~~~~~~~~
+
+**Compatible image (recommended)**:
+
+.. code:: python
+
+   # This will work - versions match
+   cluster = Cluster(ClusterConfiguration(
+       name='ray-example',
+       image='quay.io/modh/ray:2.47.1-py311-cu121'
+   ))
+
+**Incompatible image (will fail)**:
+
+.. code:: python
+
+   # This will fail with a version mismatch error
+   cluster = Cluster(ClusterConfiguration(
+       name='ray-example',
+       image='ray:2.46.0'  # Different version!
+   ))
+
+**SHA-based image (will warn)**:
+
+.. code:: python
+
+   # This will issue a warning but continue
+   cluster = Cluster(ClusterConfiguration(
+       name='ray-example',
+       image='quay.io/modh/ray@sha256:abc123...'
+   ))
+
+Best Practices
+~~~~~~~~~~~~~~
+
+- **Use versioned tags**: Always use semantic version tags (e.g., `ray:2.47.1`) rather than `latest` or SHA-based tags for better version detection.
+
+- **Test compatibility**: When building custom images, test them with the CodeFlare SDK to ensure compatibility.
+
+- **Check SDK version**: You can check the Ray version used by the SDK with:
+
+.. code:: python
+
+   from codeflare_sdk.common.utils.constants import RAY_VERSION
+   print(f"CodeFlare SDK uses Ray version: {RAY_VERSION}")
+
+**Why is version matching important?**
+
+Ray version mismatches can cause:
+
+- Incompatible API calls between the SDK and Ray cluster
+- Unexpected behavior in job submission and cluster management
+- Potential data corruption or job failures
+- Difficult-to-debug runtime errors
+
+
 Ray Usage Statistics
 -------------------
 

src/codeflare_sdk/common/utils/test_constants.py

@@ -0,0 +1,32 @@
+# Copyright 2024 IBM, Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from codeflare_sdk.common.utils.constants import RAY_VERSION, CUDA_RUNTIME_IMAGE
+
+
+class TestConstants:
+    """Test constants module for expected values."""
+
+    def test_ray_version_is_defined(self):
+        """Test that RAY_VERSION constant is properly defined."""
+        assert RAY_VERSION is not None
+        assert isinstance(RAY_VERSION, str)
+        assert RAY_VERSION == "2.47.1"
+
+    def test_cuda_runtime_image_is_defined(self):
+        """Test that CUDA_RUNTIME_IMAGE constant is properly defined."""
+        assert CUDA_RUNTIME_IMAGE is not None
+        assert isinstance(CUDA_RUNTIME_IMAGE, str)
+        assert "quay.io/modh/ray" in CUDA_RUNTIME_IMAGE

src/codeflare_sdk/common/utils/test_validation.py

@@ -0,0 +1,155 @@
+# Copyright 2024 IBM, Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from codeflare_sdk.common.utils.validation import (
+    extract_ray_version_from_image,
+    validate_ray_version_compatibility,
+)
+from codeflare_sdk.common.utils.constants import RAY_VERSION
+
+
+class TestRayVersionDetection:
+    """Test Ray version detection from container image names."""
+
+    def test_extract_ray_version_standard_format(self):
+        """Test extraction from standard Ray image formats."""
+        # Standard format
+        assert extract_ray_version_from_image("ray:2.47.1") == "2.47.1"
+        assert extract_ray_version_from_image("ray:2.46.0") == "2.46.0"
+        assert extract_ray_version_from_image("ray:1.13.0") == "1.13.0"
+
+    def test_extract_ray_version_with_registry(self):
+        """Test extraction from images with registry prefixes."""
+        assert extract_ray_version_from_image("quay.io/ray:2.47.1") == "2.47.1"
+        assert extract_ray_version_from_image("docker.io/rayproject/ray:2.47.1") == "2.47.1"
+        assert extract_ray_version_from_image("gcr.io/my-project/ray:2.47.1") == "2.47.1"
+
+    def test_extract_ray_version_with_suffixes(self):
+        """Test extraction from images with version suffixes."""
+        assert extract_ray_version_from_image("quay.io/modh/ray:2.47.1-py311-cu121") == "2.47.1"
+        assert extract_ray_version_from_image("ray:2.47.1-py311") == "2.47.1"
+        assert extract_ray_version_from_image("ray:2.47.1-gpu") == "2.47.1"
+        assert extract_ray_version_from_image("ray:2.47.1-rocm62") == "2.47.1"
+
+    def test_extract_ray_version_complex_registry_paths(self):
+        """Test extraction from complex registry paths."""
+        assert extract_ray_version_from_image("quay.io/modh/ray:2.47.1-py311-cu121") == "2.47.1"
+        assert extract_ray_version_from_image("registry.company.com/team/ray:2.47.1") == "2.47.1"
+
+    def test_extract_ray_version_no_version_found(self):
+        """Test cases where no version can be extracted."""
+        # SHA-based tags
+        assert extract_ray_version_from_image("quay.io/modh/ray@sha256:6d076aeb38ab3c34a6a2ef0f58dc667089aa15826fa08a73273c629333e12f1e") is None
+        
+        # Non-semantic versions
+        assert extract_ray_version_from_image("ray:latest") is None
+        assert extract_ray_version_from_image("ray:nightly") is None
+        assert extract_ray_version_from_image("ray:v2.47") is None  # Missing patch version
+        
+        # Non-Ray images
+        assert extract_ray_version_from_image("python:3.11") is None
+        assert extract_ray_version_from_image("ubuntu:20.04") is None
+        
+        # Empty or None
+        assert extract_ray_version_from_image("") is None
+        assert extract_ray_version_from_image(None) is None
+
+    def test_extract_ray_version_edge_cases(self):
+        """Test edge cases for version extraction."""
+        # Version with 'v' prefix should not match our pattern
+        assert extract_ray_version_from_image("ray:v2.47.1") is None
+        
+        # Multiple version-like patterns - should match the first valid one
+        assert extract_ray_version_from_image("registry/ray:2.47.1-based-on-1.0.0") == "2.47.1"
+
+
+class TestRayVersionValidation:
+    """Test Ray version compatibility validation."""
+
+    def test_validate_compatible_versions(self):
+        """Test validation with compatible Ray versions."""
+        # Exact match
+        is_compatible, message = validate_ray_version_compatibility(f"ray:{RAY_VERSION}")
+        assert is_compatible is True
+        assert "Ray versions match" in message
+
+        # With registry and suffixes
+        is_compatible, message = validate_ray_version_compatibility(f"quay.io/modh/ray:{RAY_VERSION}-py311-cu121")
+        assert is_compatible is True
+        assert "Ray versions match" in message
+
+    def test_validate_incompatible_versions(self):
+        """Test validation with incompatible Ray versions."""
+        # Different version
+        is_compatible, message = validate_ray_version_compatibility("ray:2.46.0")
+        assert is_compatible is False
+        assert "Ray version mismatch detected" in message
+        assert "CodeFlare SDK uses Ray" in message
+        assert "runtime image uses Ray" in message
+
+        # Older version
+        is_compatible, message = validate_ray_version_compatibility("ray:1.13.0")
+        assert is_compatible is False
+        assert "Ray version mismatch detected" in message
+
+    def test_validate_empty_image(self):
+        """Test validation with no custom image (should use default)."""
+        # Empty string
+        is_compatible, message = validate_ray_version_compatibility("")
+        assert is_compatible is True
+        assert "Using default Ray image compatible with SDK" in message
+
+        # None
+        is_compatible, message = validate_ray_version_compatibility(None)
+        assert is_compatible is True
+        assert "Using default Ray image compatible with SDK" in message
+
+    def test_validate_unknown_version(self):
+        """Test validation when version cannot be determined."""
+        # SHA-based image
+        is_compatible, message = validate_ray_version_compatibility(
+            "quay.io/modh/ray@sha256:6d076aeb38ab3c34a6a2ef0f58dc667089aa15826fa08a73273c629333e12f1e"
+        )
+        assert is_compatible is True
+        assert "Warning: Cannot determine Ray version" in message
+
+        # Custom image without version
+        is_compatible, message = validate_ray_version_compatibility("my-custom-ray:latest")
+        assert is_compatible is True
+        assert "Warning: Cannot determine Ray version" in message
+
+    def test_validate_custom_sdk_version(self):
+        """Test validation with custom SDK version."""
+        # Compatible with custom SDK version
+        is_compatible, message = validate_ray_version_compatibility("ray:2.46.0", "2.46.0")
+        assert is_compatible is True
+        assert "Ray versions match" in message
+
+        # Incompatible with custom SDK version
+        is_compatible, message = validate_ray_version_compatibility("ray:2.47.1", "2.46.0")
+        assert is_compatible is False
+        assert "CodeFlare SDK uses Ray 2.46.0" in message
+        assert "runtime image uses Ray 2.47.1" in message
+
+    def test_validate_message_content(self):
+        """Test that validation messages contain expected guidance."""
+        # Mismatch message should contain helpful guidance
+        is_compatible, message = validate_ray_version_compatibility("ray:2.46.0")
+        assert is_compatible is False
+        assert "compatibility issues" in message.lower()
+        assert "unexpected behavior" in message.lower()
+        assert "please use a runtime image" in message.lower()
+        assert "update your sdk version" in message.lower()
+

src/codeflare_sdk/common/utils/validation.py

@@ -0,0 +1,98 @@
+# Copyright 2024 IBM, Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Validation utilities for the CodeFlare SDK.
+
+This module contains validation functions used across the SDK for ensuring
+configuration compatibility and correctness.
+"""
+
+import re
+from typing import Optional, Tuple
+from .constants import RAY_VERSION
+
+
+def extract_ray_version_from_image(image_name: str) -> Optional[str]:
+    """
+    Extract Ray version from a container image name.
+    
+    Supports various image naming patterns:
+    - quay.io/modh/ray:2.47.1-py311-cu121
+    - ray:2.47.1
+    - some-registry/ray:2.47.1-py311
+    - quay.io/modh/ray@sha256:... (falls back to None)
+    
+    Args:
+        image_name: The container image name/tag
+        
+    Returns:
+        The extracted Ray version, or None if not found
+    """
+    if not image_name:
+        return None
+    
+    # Pattern to match semantic version after ray: or ray/
+    # Looks for patterns like ray:2.47.1, ray:2.47.1-py311, etc.
+    patterns = [
+        r'ray:(\d+\.\d+\.\d+)',  # ray:2.47.1
+        r'ray/[^:]*:(\d+\.\d+\.\d+)',  # registry/ray:2.47.1
+        r'/ray:(\d+\.\d+\.\d+)',  # any-registry/ray:2.47.1
+    ]
+    
+    for pattern in patterns:
+        match = re.search(pattern, image_name)
+        if match:
+            return match.group(1)
+    
+    # If we can't extract version, return None to indicate unknown
+    return None
+
+
+def validate_ray_version_compatibility(image_name: str, sdk_ray_version: str = RAY_VERSION) -> Tuple[bool, str]:
+    """
+    Validate that the Ray version in the runtime image matches the SDK's Ray version.
+    
+    Args:
+        image_name: The container image name/tag
+        sdk_ray_version: The Ray version used by the CodeFlare SDK
+        
+    Returns:
+        tuple: (is_compatible, message)
+            - is_compatible: True if versions match or cannot be determined, False if mismatch
+            - message: Descriptive message about the validation result
+    """
+    if not image_name:
+        # No custom image specified, will use default - this is compatible
+        return True, "Using default Ray image compatible with SDK"
+    
+    image_ray_version = extract_ray_version_from_image(image_name)
+    
+    if image_ray_version is None:
+        # Cannot determine version from image name, issue a warning but allow
+        return True, f"Warning: Cannot determine Ray version from image '{image_name}'. Please ensure it's compatible with Ray {sdk_ray_version}"
+    
+    if image_ray_version != sdk_ray_version:
+        # Version mismatch detected
+        message = (
+            f"Ray version mismatch detected!\n"
+            f"CodeFlare SDK uses Ray {sdk_ray_version}, but runtime image uses Ray {image_ray_version}.\n"
+            f"This mismatch can cause compatibility issues and unexpected behavior.\n"
+            f"Please use a runtime image with Ray {sdk_ray_version} or update your SDK version."
+        )
+        return False, message
+    
+    # Versions match
+    return True, f"Ray versions match: SDK and runtime image both use Ray {sdk_ray_version}"
+

src/codeflare_sdk/ray/cluster/config.py

@@ -23,6 +23,7 @@
 from dataclasses import dataclass, field, fields
 from typing import Dict, List, Optional, Union, get_args, get_origin
 from kubernetes.client import V1Toleration, V1Volume, V1VolumeMount
+from ...common.utils.validation import validate_ray_version_compatibility
 
 dir = pathlib.Path(__file__).parent.parent.resolve()
 
@@ -182,6 +183,7 @@ def __post_init__(self):
         self._validate_extended_resource_requests(
             self.worker_extended_resource_requests
         )
+        self._validate_ray_version_compatibility()
 
     def _combine_extended_resource_mapping(self):
         if overwritten := set(self.extended_resource_mapping.keys()).intersection(
@@ -289,3 +291,15 @@ def check_type(value, expected_type):
             return isinstance(value, expected_type)
 
         return check_type(value, expected_type)
+
+    def _validate_ray_version_compatibility(self):
+        """
+        Validate that the Ray version in the runtime image matches the SDK's Ray version.
+        Raises ValueError if there is a version mismatch.
+        """
+        is_compatible, message = validate_ray_version_compatibility(self.image)
+
+        if not is_compatible:
+            raise ValueError(message)
+        elif "Warning:" in message:
+            warnings.warn(message)