From f0810abcd473e58546c2cb8fe626ee8f82f59aad Mon Sep 17 00:00:00 2001 From: John Walz Date: Wed, 29 Jan 2025 11:10:01 -0500 Subject: [PATCH 1/3] fix: adding anywidget as a required dependency --- poetry.lock | 70 +++++++++++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 3 ++- 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index b352501d2..30d2ff0b8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -212,6 +212,26 @@ doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"] trio = ["trio (>=0.26.1)"] +[[package]] +name = "anywidget" +version = "0.9.13" +description = "custom jupyter widgets made easy" +optional = false +python-versions = ">=3.7" +files = [ + {file = "anywidget-0.9.13-py3-none-any.whl", hash = "sha256:43d1658f1043b8c95cd350b2f5deccb123fd37810a36f656d6163aefe8163705"}, + {file = "anywidget-0.9.13.tar.gz", hash = "sha256:c655455bf51f82182eb23c5947d37cc41f0b1ffacaf7e2b763147a2332cb3f07"}, +] + +[package.dependencies] +ipywidgets = ">=7.6.0" +psygnal = ">=0.8.1" +typing-extensions = ">=4.2.0" + +[package.extras] +dev = ["comm (>=0.1.0)", "watchfiles (>=0.18.0)"] +test = ["ipython (<8.13)", "msgspec", "mypy (==1.10.0)", "pydantic", "pytest", "pytest-cov", "ruff"] + [[package]] name = "appdirs" version = "1.4.4" @@ -1351,8 +1371,11 @@ name = "docutils" version = "0.18.1" description = "Docutils -- Python Documentation Utilities" optional = false -python-versions = "*" -files = [] +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "docutils-0.18.1-py2.py3-none-any.whl", hash = "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c"}, + {file = "docutils-0.18.1.tar.gz", hash = "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"}, +] [[package]] name = "entrypoints" @@ -4281,8 +4304,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -4901,6 +4924,45 @@ files = [ dev = ["abi3audit", "black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] test = ["pytest", "pytest-xdist", "setuptools"] +[[package]] +name = "psygnal" +version = "0.11.1" +description = "Fast python callback/event system modeled after Qt Signals" +optional = false +python-versions = ">=3.8" +files = [ + {file = "psygnal-0.11.1-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:8d9187700fc608abefeb287bf2e0980a26c62471921ffd1a3cd223ccc554181b"}, + {file = "psygnal-0.11.1-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:cec87aee468a1fe564094a64bc3c30edc86ce34d7bb37ab69332c7825b873396"}, + {file = "psygnal-0.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7676e89225abc2f37ca7022c300ffd26fefaf21bdc894bc7c41dffbad5e969df"}, + {file = "psygnal-0.11.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c392f638aac2cdc4f13fffb904455224ae9b4dbb2f26d7f3264e4208fee5334d"}, + {file = "psygnal-0.11.1-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:3c04baec10f882cdf784a7312e23892416188417ad85607e6d1de2e8a9e70709"}, + {file = "psygnal-0.11.1-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:8f77317cbd11fbed5bfdd40ea41b4e551ee0cf37881cdbc325b67322af577485"}, + {file = "psygnal-0.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24e69ea57ee39e3677298f38a18828af87cdc0bf0aa64685d44259e608bae3ec"}, + {file = "psygnal-0.11.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d77f1a71fe9859c0335c87d92afe1b17c520a4137326810e94351839342d8fc7"}, + {file = "psygnal-0.11.1-cp312-cp312-macosx_10_16_arm64.whl", hash = "sha256:0b55cb42e468f3a7de75392520778604fef2bc518b7df36c639b35ce4ed92016"}, + {file = "psygnal-0.11.1-cp312-cp312-macosx_10_16_x86_64.whl", hash = "sha256:c7dd3cf809c9c1127d90c6b11fbbd1eb2d66d512ccd4d5cab048786f13d11220"}, + {file = "psygnal-0.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:885922a6e65ece9ff8ccf2b6810f435ca8067f410889f7a8fffb6b0d61421a0d"}, + {file = "psygnal-0.11.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1c2388360a9ffcd1381e9b36d0f794287a270d58e69bf17658a194bbf86685c1"}, + {file = "psygnal-0.11.1-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:2deec4bf7adbb9e3ef0513ae8b9e98bb815eb62b76a7bf1986f1d6ed626c8784"}, + {file = "psygnal-0.11.1-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:36cd667dd1d3e70e3fd970463a8571436e5ae58f02cc05a4a1669e6d8550d263"}, + {file = "psygnal-0.11.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc260f19349485bd58e276e731cf8be40d8891cc6ff1c165762bd2c1b84f1ff7"}, + {file = "psygnal-0.11.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fe70023fe4cf8bb6a0f27e89fd8f1cf715893dfb004b790937a0bc59d9071aab"}, + {file = "psygnal-0.11.1-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:c9dde42a2cdf34f9c5fe0cd7515e2ab1524e3207afb37d096733c7a3dcdf388a"}, + {file = "psygnal-0.11.1-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c05f474b297e2577506b354132c3fed054f0444ccce6d431f299d3750c2ede4b"}, + {file = "psygnal-0.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:713dfb96a1315378ce9120376d975671ede3133de4985884a43d4b6b332faeee"}, + {file = "psygnal-0.11.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09c75d21eb090e2ffafb32893bc5d104b98ed237ed64bebccb45cca759c7dcf4"}, + {file = "psygnal-0.11.1-py3-none-any.whl", hash = "sha256:04255fe28828060a80320f8fda937c47bc0c21ca14f55a13eb7c494b165ea395"}, + {file = "psygnal-0.11.1.tar.gz", hash = "sha256:f9b02ca246ab0adb108c4010b4a486e464f940543201074591e50370cd7b0cc0"}, +] + +[package.extras] +dev = ["ipython", "mypy", "mypy-extensions", "pre-commit", "pyqt5", "pytest-mypy-plugins", "rich", "ruff", "typing-extensions"] +docs = ["griffe (==0.25.5)", "mkdocs (==1.4.2)", "mkdocs-material (==8.5.10)", "mkdocs-minify-plugin", "mkdocs-spellcheck[all]", "mkdocstrings (==0.20.0)", "mkdocstrings-python (==0.8.3)"] +proxy = ["wrapt"] +pydantic = ["pydantic"] +test = ["attrs", "dask", "msgspec", "numpy", "pydantic", "pyinstaller (>=4.0)", "pytest (>=6.0)", "pytest-cov", "toolz", "wrapt"] +testqt = ["pytest-qt", "qtpy"] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -8069,4 +8131,4 @@ pytorch = ["torch"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.12" -content-hash = "6c3c40ab7dbfe990a6c1b02c4c23a558e49230daaba554fd5571b337c8e0c033" +content-hash = "c998f65816b0b08d89c839c0f2bb7c1474b12e753bcd4a9b1d6ec322fa5c18a4" diff --git a/pyproject.toml b/pyproject.toml index f9d018084..1c971e23b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,9 @@ readme = "README.pypi.md" version = "2.8.3" [tool.poetry.dependencies] +python = ">=3.8.1,<3.12" aiohttp = {extras = ["speedups"], version = "*"} +anywidget = "^0.9.13" arch = "*" bert-score = ">=0.3.13" catboost = "*" @@ -36,7 +38,6 @@ plotly = "*" plotly-express = "*" polars = "*" pycocoevalcap = {version = "^1.2", optional = true} -python = ">=3.8.1,<3.12" python-dotenv = "*" ragas = {version = ">=0.2.3", optional = true} rouge = ">=1" From 5a95917871bcc76a4a1e3d39245c25a21f43485a Mon Sep 17 00:00:00 2001 From: John Walz Date: Wed, 29 Jan 2025 11:33:21 -0500 Subject: [PATCH 2/3] 2.8.4 --- pyproject.toml | 2 +- validmind/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1c971e23b..346914f33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ description = "ValidMind Library" license = "Commercial License" name = "validmind" readme = "README.pypi.md" -version = "2.8.3" +version = "2.8.4" [tool.poetry.dependencies] python = ">=3.8.1,<3.12" diff --git a/validmind/__version__.py b/validmind/__version__.py index 764271bc1..e897069b0 100644 --- a/validmind/__version__.py +++ b/validmind/__version__.py @@ -1 +1 @@ -__version__ = "2.8.3" +__version__ = "2.8.4" From 0090af6af94b10cad20468750e37e103e94dbbb3 Mon Sep 17 00:00:00 2001 From: John Walz Date: Wed, 29 Jan 2025 11:33:46 -0500 Subject: [PATCH 3/3] feat: adding global test description generation toggle --- validmind/ai/test_descriptions.py | 3 ++- validmind/api_client.py | 6 +++++- validmind/tests/model_validation/ragas/utils.py | 15 ++++++++------- .../tests/prompt_validation/ai_powered_test.py | 9 +++++---- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/validmind/ai/test_descriptions.py b/validmind/ai/test_descriptions.py index 1e954f26c..b52bc3820 100644 --- a/validmind/ai/test_descriptions.py +++ b/validmind/ai/test_descriptions.py @@ -150,7 +150,8 @@ def get_result_description( # Check the feature flag first, then the environment variable llm_descriptions_enabled = ( client_config.can_generate_llm_test_descriptions() - and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") not in ["0", "false"] + and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1").lower() + not in ["0", "false"] ) # TODO: fix circular import diff --git a/validmind/api_client.py b/validmind/api_client.py index a207174bc..cb6228616 100644 --- a/validmind/api_client.py +++ b/validmind/api_client.py @@ -194,6 +194,7 @@ def init( api_host: Optional[str] = None, model: Optional[str] = None, monitoring: bool = False, + generate_descriptions: Optional[bool] = None, ): """ Initializes the API client instances and calls the /ping endpoint to ensure @@ -209,7 +210,7 @@ def init( api_secret (str, optional): The API secret. Defaults to None. api_host (str, optional): The API host. Defaults to None. monitoring (bool): The ongoing monitoring flag. Defaults to False. - + generate_descriptions (bool): Whether to use GenAI to generate test result descriptions. Defaults to True. Raises: ValueError: If the API key and secret are not provided """ @@ -235,6 +236,9 @@ def init( _monitoring = monitoring + if generate_descriptions is not None: + os.environ["VALIDMIND_LLM_DESCRIPTIONS_ENABLED"] = str(generate_descriptions) + reload() diff --git a/validmind/tests/model_validation/ragas/utils.py b/validmind/tests/model_validation/ragas/utils.py index 92dbf73da..01426a6f8 100644 --- a/validmind/tests/model_validation/ragas/utils.py +++ b/validmind/tests/model_validation/ragas/utils.py @@ -4,24 +4,25 @@ import os -from validmind.ai.utils import get_client_and_model -from validmind.client_config import client_config +from validmind.ai.utils import get_client_and_model, is_configured EMBEDDINGS_MODEL = "text-embedding-3-small" def get_ragas_config(): - if not client_config.can_generate_llm_test_descriptions(): - raise ValueError( - "LLM based descriptions are not enabled in the current configuration." - ) - # import here since its an optional dependency try: from langchain_openai import ChatOpenAI, OpenAIEmbeddings except ImportError: raise ImportError("Please run `pip install validmind[llm]` to use LLM tests") + if not is_configured(): + raise ValueError( + "LLM is not configured. Please set an `OPENAI_API_KEY` environment variable " + "or ensure that you are connected to the ValidMind API and ValidMind AI is " + "enabled for your account." + ) + client, model = get_client_and_model() os.environ["OPENAI_API_BASE"] = str(client.base_url) diff --git a/validmind/tests/prompt_validation/ai_powered_test.py b/validmind/tests/prompt_validation/ai_powered_test.py index d3fd85c1f..49d604f40 100644 --- a/validmind/tests/prompt_validation/ai_powered_test.py +++ b/validmind/tests/prompt_validation/ai_powered_test.py @@ -4,8 +4,7 @@ import re -from validmind.ai.utils import get_client_and_model -from validmind.client_config import client_config +from validmind.ai.utils import get_client_and_model, is_configured missing_prompt_message = """ Cannot run prompt validation tests on a model with no prompt. @@ -25,9 +24,11 @@ def call_model( system_prompt: str, user_prompt: str, temperature: float = 0.0, seed: int = 42 ): """Call LLM with the given prompts and return the response""" - if not client_config.can_generate_llm_test_descriptions(): + if not is_configured(): raise ValueError( - "LLM based descriptions are not enabled for your organization." + "LLM is not configured. Please set an `OPENAI_API_KEY` environment variable " + "or ensure that you are connected to the ValidMind API and ValidMind AI is " + "enabled for your account." ) client, model = get_client_and_model()