Merge branch 'main' into ODSC-65658/llamaindex_docs

Author: mrDzurb

.github/workflows/run-forecast-unit-tests.yml

@@ -56,6 +56,6 @@ jobs:
           $CONDA/bin/conda init
           source /home/runner/.bashrc
           pip install -r test-requirements-operators.txt
-          pip install "oracle-automlx[forecasting]>=24.4.0"
+          pip install "oracle-automlx[forecasting]>=24.4.1"
           pip install pandas>=2.2.0
           python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast

.github/workflows/run-unittests-default_setup.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8-3.11] - Default Tests"
+name: "[Py3.9-3.11] - Default Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/run-unittests-py39-py310.yml

@@ -74,16 +74,16 @@ jobs:
         name: "Test env setup"
         timeout-minutes: 30
 
-      - name: "Run hpo tests"
-        timeout-minutes: 10
-        shell: bash
-        if: ${{ matrix.name }} == "unitary"
-        run: |
-          set -x # print commands that are executed
+      # - name: "Run hpo tests"
+      #   timeout-minutes: 10
+      #   shell: bash
+      #   if: ${{ matrix.name }} == "unitary"
+      #   run: |
+      #     set -x # print commands that are executed
 
-          # Run hpo tests, which hangs if run together with all unitary tests
-          python -m pytest -v -p no:warnings -n auto --dist loadfile \
-          tests/unitary/with_extras/hpo
+      #     # Run hpo tests, which hangs if run together with all unitary tests
+      #     python -m pytest -v -p no:warnings -n auto --dist loadfile \
+      #     tests/unitary/with_extras/hpo
 
       - name: "Run unitary tests folder with maximum ADS dependencies"
         timeout-minutes: 60

THIRD_PARTY_LICENSES.txt

@@ -96,6 +96,12 @@ fsspec
 * Source code: https://github.com/intake/filesystem_spec
 * Project home: https://github.com/intake/filesystem_spec
 
+httpx
+* Copyright (c) 2021 ProjectDiscovery, Inc.
+* License: MIT License
+* Source code: https://github.com/projectdiscovery/httpx
+* Project home: https://github.com/projectdiscovery/httpx
+
 geopandas
 * Copyright  (c) 2013-2016, GeoPandas developers
 * License: BSD 3-Clause "New" or "Revised" License
@@ -391,6 +397,12 @@ tabulate
 * Source code: https://github.com/astanin/python-tabulate
 * Project home: https://github.com/astanin/python-tabulate
 
+tenacity
+* No listed copyright holder
+* License: Apache-2.0 license
+* Source code: https://github.com/jd/tenacity
+* Project home: https://github.com/jd/tenacity
+
 tensorflow
 * Copyright (c) 2023 Google, Inc.
 * License: Apache-2.0 license

ads/aqua/__init__.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
 import os
+from logging import getLogger
 
 from ads import logger, set_auth
+from ads.aqua.client.client import AsyncClient, Client
 from ads.aqua.common.utils import fetch_service_compartment
 from ads.config import OCI_RESOURCE_PRINCIPAL_VERSION
 
@@ -19,6 +20,7 @@ def get_logger_level():
     return level
 
 
+logger = getLogger(__name__)
 logger.setLevel(get_logger_level())
 
 
@@ -27,7 +29,6 @@ def set_log_level(log_level: str):
 
     log_level = log_level.upper()
     logger.setLevel(log_level.upper())
-    logger.handlers[0].setLevel(log_level)
 
 
 if OCI_RESOURCE_PRINCIPAL_VERSION:

ads/aqua/app.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+import json
 import os
+import traceback
 from dataclasses import fields
 from typing import Dict, Union
 
@@ -22,7 +24,7 @@
 from ads.aqua.constants import UNKNOWN
 from ads.common import oci_client as oc
 from ads.common.auth import default_signer
-from ads.common.utils import extract_region
+from ads.common.utils import extract_region, is_path_exists
 from ads.config import (
     AQUA_TELEMETRY_BUCKET,
     AQUA_TELEMETRY_BUCKET_NS,
@@ -135,6 +137,8 @@ def create_model_version_set(
         description: str = None,
         compartment_id: str = None,
         project_id: str = None,
+        freeform_tags: dict = None,
+        defined_tags: dict = None,
         **kwargs,
     ) -> tuple:
         """Creates ModelVersionSet from given ID or Name.
@@ -153,7 +157,10 @@ def create_model_version_set(
             Project OCID.
         tag: (str, optional)
             calling tag, can be Tags.AQUA_FINE_TUNING or Tags.AQUA_EVALUATION
-
+        freeform_tags: (dict, optional)
+            Freeform tags for the model version set
+        defined_tags: (dict, optional)
+            Defined tags for the model version set
         Returns
         -------
         tuple: (model_version_set_id, model_version_set_name)
@@ -182,13 +189,15 @@ def create_model_version_set(
                 mvs_freeform_tags = {
                     tag: tag,
                 }
+                mvs_freeform_tags = {**mvs_freeform_tags, **(freeform_tags or {})}
                 model_version_set = (
                     ModelVersionSet()
                     .with_compartment_id(compartment_id)
                     .with_project_id(project_id)
                     .with_name(model_version_set_name)
                     .with_description(description)
                     .with_freeform_tags(**mvs_freeform_tags)
+                    .with_defined_tags(**(defined_tags or {}))
                     # TODO: decide what parameters will be needed
                     # when refactor eval to use this method, we need to pass tag here.
                     .create(**kwargs)
@@ -288,33 +297,46 @@ def get_config(self, model_id: str, config_file_name: str) -> Dict:
             raise AquaRuntimeError(f"Target model {oci_model.id} is not Aqua model.")
 
         config = {}
-        artifact_path = get_artifact_path(oci_model.custom_metadata_list)
+        # if the current model has a service model tag, then
+        if Tags.AQUA_SERVICE_MODEL_TAG in oci_model.freeform_tags:
+            base_model_ocid = oci_model.freeform_tags[Tags.AQUA_SERVICE_MODEL_TAG]
+            logger.info(
+                f"Base model found for the model: {oci_model.id}. "
+                f"Loading {config_file_name} for base model {base_model_ocid}."
+            )
+            base_model = self.ds_client.get_model(base_model_ocid).data
+            artifact_path = get_artifact_path(base_model.custom_metadata_list)
+        else:
+            logger.info(f"Loading {config_file_name} for model {oci_model.id}...")
+            artifact_path = get_artifact_path(oci_model.custom_metadata_list)
+
         if not artifact_path:
-            logger.error(
+            logger.debug(
                 f"Failed to get artifact path from custom metadata for the model: {model_id}"
             )
             return config
 
-        try:
-            config_path = f"{os.path.dirname(artifact_path)}/config/"
-            config = load_config(
-                config_path,
-                config_file_name=config_file_name,
-            )
-        except Exception:
-            # todo: temp fix for issue related to config load for byom models, update logic to choose the right path
+        config_path = f"{os.path.dirname(artifact_path)}/config/"
+        if not is_path_exists(config_path):
+            config_path = f"{artifact_path.rstrip('/')}/config/"
+
+        config_file_path = f"{config_path}{config_file_name}"
+        if is_path_exists(config_file_path):
             try:
-                config_path = f"{artifact_path.rstrip('/')}/config/"
                 config = load_config(
                     config_path,
                     config_file_name=config_file_name,
                 )
             except Exception:
-                pass
+                logger.debug(
+                    f"Error loading the {config_file_name} at path {config_path}.\n"
+                    f"{traceback.format_exc()}"
+                )
 
         if not config:
-            logger.error(
-                f"{config_file_name} is not available for the model: {model_id}. Check if the custom metadata has the artifact path set."
+            logger.debug(
+                f"{config_file_name} is not available for the model: {model_id}. "
+                f"Check if the custom metadata has the artifact path set."
             )
             return config
 
@@ -340,7 +362,9 @@ def build_cli(self) -> str:
         """
         cmd = f"ads aqua {self._command}"
         params = [
-            f"--{field.name} {getattr(self,field.name)}"
+            f"--{field.name} {json.dumps(getattr(self, field.name))}"
+            if isinstance(getattr(self, field.name), dict)
+            else f"--{field.name} {getattr(self, field.name)}"
             for field in fields(self.__class__)
             if getattr(self, field.name) is not None
         ]

ads/aqua/client/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# Copyright (c) 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/