changes for tei byoc deployment

Author: VipulMascarenhas

ads/aqua/common/utils.py

@@ -1111,3 +1111,47 @@ def generate_tei_cmd_var(os_path: str) -> List[str]:
     ]
 
     return cmd_var
+
+
+def parse_cmd_var(cmd_list: List[str]) -> dict:
+    """Helper functions that parses a list into a key-value dictionary. The list contains keys separated by the prefix
+    '--' and the value of the key is the subsequent element.
+    """
+    it = iter(cmd_list)
+    return {
+        key: next(it, None) if not key.startswith("--") else next(it, None)
+        for key in it
+        if key.startswith("--")
+    }
+
+
+def validate_cmd_var(cmd_var: List[str], overrides: List[str]) -> List[str]:
+    """This function accepts two lists of parameters and combines them. If the second list shares the common parameter
+    names/keys, then it raises an error.
+    Parameters
+    ----------
+    cmd_var: List[str]
+        Default list of parameters
+    overrides: List[str]
+        List of parameters to override
+    Returns
+    -------
+        List[str] of combined parameters
+    """
+    cmd_var = [str(x) for x in cmd_var]
+    if not overrides:
+        return cmd_var
+    overrides = [str(x) for x in overrides]
+
+    cmd_dict = parse_cmd_var(cmd_var)
+    overrides_dict = parse_cmd_var(overrides)
+
+    # check for conflicts
+    common_keys = set(cmd_dict.keys()) & set(overrides_dict.keys())
+    if common_keys:
+        raise AquaValueError(
+            f"The following keys cannot be overridden: {', '.join(common_keys)}"
+        )
+
+    combined_cmd_var = cmd_var + overrides
+    return combined_cmd_var

ads/aqua/model/model.py

@@ -720,6 +720,7 @@ def _create_model_catalog_entry(
                     description=f"Inference container cmd vars for {model_name}",
                     category="Other",
                 )
+
             if finetuning_container:
                 tags[Tags.READY_TO_FINE_TUNE] = "true"
                 metadata.add(
@@ -1232,9 +1233,12 @@ def register(
         inference_container = ds_model.custom_metadata_list.get(
             ModelCustomMetadataFields.DEPLOYMENT_CONTAINER
         ).value
-        evaluation_container = ds_model.custom_metadata_list.get(
-            ModelCustomMetadataFields.EVALUATION_CONTAINER,
-        ).value
+        try:
+            evaluation_container = ds_model.custom_metadata_list.get(
+                ModelCustomMetadataFields.EVALUATION_CONTAINER,
+            ).value
+        except Exception:
+            evaluation_container = None
         try:
             finetuning_container = ds_model.custom_metadata_list.get(
                 ModelCustomMetadataFields.FINETUNE_CONTAINER,

ads/aqua/modeldeployment/deployment.py

@@ -24,6 +24,7 @@
     get_resource_name,
     get_restricted_params_by_container,
     load_config,
+    validate_cmd_var,
 )
 from ads.aqua.constants import (
     AQUA_MODEL_ARTIFACT_FILE,
@@ -263,12 +264,15 @@ def create(
                 cmd_var_string = aqua_model.custom_metadata_list.get(
                     AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME
                 ).value
-                cmd_var.extend(cmd_var_string.split(","))
             except ValueError as err:
                 raise AquaValueError(
                     f"{AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME} key is not available in the custom metadata "
                     f"field. Please check if the model was registered with {container_type_key} inference container."
                 ) from err
+            default_cmd_var = cmd_var_string.split(",")
+            if default_cmd_var:
+                cmd_var = validate_cmd_var(default_cmd_var, cmd_var)
+            logging.info(f"CMD used for deploying {aqua_model.id} :{cmd_var}")
         else:
             # fetch image name from config
             container_image_uri = get_container_image(container_type=container_type_key)

ads/aqua/modeldeployment/entities.py

@@ -3,7 +3,7 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from dataclasses import dataclass, field
-from typing import Union
+from typing import List, Union
 
 from oci.data_science.models import (
     ModelDeployment,
@@ -52,6 +52,7 @@ class AquaDeployment(DataClassSerializable):
     shape_info: field(default_factory=ShapeInfo) = None
     tags: dict = None
     environment_variables: dict = None
+    cmd: List[str] = None
 
     @classmethod
     def from_oci_model_deployment(
@@ -80,6 +81,7 @@ def from_oci_model_deployment(
         )
         instance_count = oci_model_deployment.model_deployment_configuration_details.model_configuration_details.scaling_policy.instance_count
         environment_variables = oci_model_deployment.model_deployment_configuration_details.environment_configuration_details.environment_variables
+        cmd = oci_model_deployment.model_deployment_configuration_details.environment_configuration_details.cmd
         shape_info = ShapeInfo(
             instance_shape=instance_configuration.instance_shape_name,
             instance_count=instance_count,
@@ -120,6 +122,7 @@ def from_oci_model_deployment(
             ),
             tags=freeform_tags,
             environment_variables=environment_variables,
+            cmd=cmd,
         )
 
 

ads/config.py

@@ -65,7 +65,7 @@
 AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME = "deployment-container"
 AQUA_FINETUNING_CONTAINER_METADATA_NAME = "finetune-container"
 AQUA_EVALUATION_CONTAINER_METADATA_NAME = "evaluation-container"
-AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME = "container_cmd_var"
+AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME = "container-cmd-var"
 AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME = "deployment-container-uri"
 AQUA_DEPLOYMENT_CONTAINER_OVERRIDE_FLAG_METADATA_NAME = "deployment-container-custom"
 AQUA_FINETUNING_CONTAINER_OVERRIDE_FLAG_METADATA_NAME = "finetune-container-custom"

tests/unitary/with_extras/aqua/test_data/deployment/aqua_create_embedding_deployment.yaml

@@ -0,0 +1,34 @@
+kind: deployment
+spec:
+  createdBy: ocid1.user.oc1..<OCID>
+  displayName: model-deployment-name
+  freeformTags:
+    OCI_AQUA: active
+    aqua_model_name: model-name
+  id: "ocid1.datasciencemodeldeployment.oc1.<region>.<MD_OCID>"
+  infrastructure:
+    kind: infrastructure
+    spec:
+      bandwidthMbps: 10
+      compartmentId: ocid1.compartment.oc1..<OCID>
+      deploymentType: SINGLE_MODEL
+      policyType: FIXED_SIZE
+      projectId: ocid1.datascienceproject.oc1.iad.<OCID>
+      replica: 1
+      shapeName: "VM.GPU.A10.1"
+    type: datascienceModelDeployment
+  lifecycleState: CREATING
+  modelDeploymentUrl: "https://modeldeployment.customer-oci.com/ocid1.datasciencemodeldeployment.oc1.<region>.<MD_OCID>"
+  runtime:
+    kind: runtime
+    spec:
+      env:
+        BASE_MODEL: service_models/model-name/artifact
+        MODEL_DEPLOY_PREDICT_ENDPOINT: /v1/embeddings
+      healthCheckPort: 8080
+      image: "dsmc://image-name:1.0.0.0"
+      modelUri: "ocid1.datasciencemodeldeployment.oc1.<region>.<MODEL_OCID>"
+      serverPort: 8080
+    type: container
+  timeCreated: 2024-01-01T00:00:00.000000+00:00
+type: modelDeployment

tests/unitary/with_extras/aqua/test_data/deployment/aqua_tei_byoc_embedding_model.yaml

@@ -0,0 +1,88 @@
+kind: datascienceModel
+spec:
+  artifact: oci://service-managed-models@namespace/service_models/model-name/artifact
+  compartmentId: ocid1.compartment.oc1..<OCID>
+  customMetadataList:
+    data:
+    - category: Other
+      description: artifact location
+      key: artifact_location
+      value: service_models/model-name/artifact
+    - category: Other
+      description: model by reference flag
+      key: modelDescription
+      value: true
+    - category: Other
+      description: Deployment container mapping for model model-name
+      key: deployment-container
+      value: odsc-tei-serving
+    - category: Other
+      description: Inference container URI for model model-name
+      key: deployment-container-uri
+      value: region.ocir.io/tenancy/image_name:tag
+    - category: Other
+      description: Inference container cmd vars for model-name
+      key: container-cmd-var
+      value: --model-id,/opt/ds/model/deployed_model/service_models/model-name/artifact/,--port,8080
+  definedTags: {}
+  description: Mock model description
+  displayName: model-name
+  freeformTags:
+    OCI_AQUA: active
+    license: License
+    organization: Organization
+    ready_to_fine_tune: false
+    task: text_embedding
+  id: ocid1.datasciencemodel.oc1.iad.<OCID>
+  lifecycleState: ACTIVE
+  modelDescription:
+    models:
+    - bucketName: service-managed-models
+      namespace: namespace
+      objects:
+      - name: service_models/model-name/artifact/README.md
+        sizeInBytes: 10317
+        version: 450a8124-f5ca-4ee6-b4cf-c1dc05b13d46
+      - name: service_models/model-name/artifact/config.json
+        sizeInBytes: 950
+        version: 3ace781b-4a48-4e89-88b6-61f0db6d51ad
+      - name: service_models/model-name/artifact/configuration_RW.py
+        sizeInBytes: 2607
+        version: ba1df5b6-7546-42e5-964e-63cd013e988c
+      - name: service_models/model-name/artifact/generation_config.json
+        sizeInBytes: 111
+        version: e23a04c8-9725-4f20-8bb1-f455129e2a4e
+      - name: service_models/model-name/artifact/modelling_RW.py
+        sizeInBytes: 47560
+        version: a584c221-afab-441f-901d-fbe8251dccf6
+      - name: service_models/model-name/artifact/pytorch_model-00001-of-00002.bin
+        sizeInBytes: 9951028193
+        version: e919676e-48dd-4bea-af82-14b5f3eb2b9b
+      - name: service_models/model-name/artifact/pytorch_model-00002-of-00002.bin
+        sizeInBytes: 4483421659
+        version: d6255d3e-bd91-4c05-b3ca-fc1be576ee10
+      - name: service_models/model-name/artifact/pytorch_model.bin.index.json
+        sizeInBytes: 16924
+        version: 0419428c-2a7b-45d9-bb78-142fe0630017
+      - name: service_models/model-name/artifact/special_tokens_map.json
+        sizeInBytes: 281
+        version: 5569231a-a526-4881-8945-a94a1bb59b2e
+      - name: service_models/model-name/artifact/tokenizer.json
+        sizeInBytes: 2734130
+        version: d3a8a00a-de79-4d80-aa69-d8f68ee800ec
+      - name: service_models/model-name/artifact/tokenizer_config.json
+        sizeInBytes: 220
+        version: 84eed6ff-c1ed-4641-8c10-e6a49364d7dd
+      prefix: service_models/model-name/artifact
+    type: modelOSSReferenceDescription
+    version: '1.0'
+  projectId: ocid1.datascienceproject.oc1.iad.<OCID>
+  provenanceMetadata:
+    artifact_dir: null
+    git_branch: null
+    git_commit: 123456
+    repository_url: https://model-name-url.com
+    training_id: null
+    training_script_path: null
+  timeCreated: 2024-01-01T00:00:00.000000+00:00
+type: dataScienceModel