From 0a0251e0c791466dbcd0e827467363ad1473a9a4 Mon Sep 17 00:00:00 2001 From: John Walz Date: Tue, 11 Feb 2025 12:38:18 -0500 Subject: [PATCH 1/4] fix: handle context length too large --- validmind/ai/test_descriptions.py | 30 ++++++++++++++++++++++++++++-- validmind/tests/run.py | 2 +- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/validmind/ai/test_descriptions.py b/validmind/ai/test_descriptions.py index e52a37e46..0fa794368 100644 --- a/validmind/ai/test_descriptions.py +++ b/validmind/ai/test_descriptions.py @@ -8,6 +8,7 @@ from concurrent.futures import ThreadPoolExecutor from typing import List, Optional, Union +import tiktoken from jinja2 import Template from ..client_config import client_config @@ -82,6 +83,25 @@ def _get_llm_global_context(): return context if context_enabled and context else None +def _truncate_summary(summary: str, max_tokens: int = 100_000): + if len(summary) < max_tokens: + # since string itself is less than max_tokens, definitely small enough + return summary + + # TODO: better context length handling + encoding = tiktoken.encoding_for_model("gpt-4o") + summary_tokens = encoding.encode(summary) + + if len(summary_tokens) > max_tokens: + summary = ( + encoding.decode(summary_tokens[:max_tokens]) + + "...[truncated]" + + encoding.decode(summary_tokens[-100:]) + ) + + return summary + + def generate_description( test_id: str, test_description: str, @@ -127,7 +147,7 @@ def generate_description( "test_name": test_name, "test_description": test_description, "title": title, - "summary": summary, + "summary": _truncate_summary(summary), "figures": [figure._get_b64_url() for figure in ([] if tables else figures)], "context": context, } @@ -165,7 +185,13 @@ def wrapped(): title=title, ) except Exception as e: - logger.error(f"Failed to generate description: {e}") + if "maximum context length" in str(e): + logger.warning( + f"Test result {test_id} is too large to generate a description" + ) + else: + logger.warning(f"Failed to generate description for {test_id}: {e}") + logger.warning(f"Using default description for {test_id}") return test_description diff --git a/validmind/tests/run.py b/validmind/tests/run.py index 9401b2d1b..a86047c44 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -3,11 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial import platform +import pprint import subprocess import time from datetime import datetime from inspect import getdoc -import pprint from typing import Any, Callable, Dict, List, Optional, Tuple, Union from uuid import uuid4 From f1104813a0ac95025bf1f27b5b71cd9e12b0ab6d Mon Sep 17 00:00:00 2001 From: John Walz Date: Tue, 11 Feb 2025 12:39:39 -0500 Subject: [PATCH 2/4] 2.7.8 --- pyproject.toml | 2 +- validmind/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c9663eb1c..c1884bc04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ description = "ValidMind Library" license = "Commercial License" name = "validmind" readme = "README.pypi.md" -version = "2.7.7" +version = "2.7.8" [tool.poetry.dependencies] aiohttp = {extras = ["speedups"], version = "*"} diff --git a/validmind/__version__.py b/validmind/__version__.py index 3da1ab5e9..dc6ddde7a 100644 --- a/validmind/__version__.py +++ b/validmind/__version__.py @@ -1 +1 @@ -__version__ = "2.7.7" +__version__ = "2.7.8" From ec7c0721b89fe91eb72bb6e3a5e2027bf5c19e13 Mon Sep 17 00:00:00 2001 From: John Walz Date: Tue, 11 Feb 2025 13:01:20 -0500 Subject: [PATCH 3/4] chore: add warning when truncating test result for llm --- validmind/ai/test_descriptions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/validmind/ai/test_descriptions.py b/validmind/ai/test_descriptions.py index 0fa794368..85921d69d 100644 --- a/validmind/ai/test_descriptions.py +++ b/validmind/ai/test_descriptions.py @@ -83,7 +83,7 @@ def _get_llm_global_context(): return context if context_enabled and context else None -def _truncate_summary(summary: str, max_tokens: int = 100_000): +def _truncate_summary(summary: str, test_id: str, max_tokens: int = 100_000): if len(summary) < max_tokens: # since string itself is less than max_tokens, definitely small enough return summary @@ -93,6 +93,10 @@ def _truncate_summary(summary: str, max_tokens: int = 100_000): summary_tokens = encoding.encode(summary) if len(summary_tokens) > max_tokens: + logger.warning( + "Truncating test result due to context length restrictions..." + f"Generated description for {test_id} may be innacurate" + ) summary = ( encoding.decode(summary_tokens[:max_tokens]) + "...[truncated]" @@ -147,7 +151,7 @@ def generate_description( "test_name": test_name, "test_description": test_description, "title": title, - "summary": _truncate_summary(summary), + "summary": _truncate_summary(summary, test_id), "figures": [figure._get_b64_url() for figure in ([] if tables else figures)], "context": context, } From 9f9e9aa1f875c571818e0e8954d66aa82cf7f60f Mon Sep 17 00:00:00 2001 From: John Walz Date: Tue, 11 Feb 2025 13:02:40 -0500 Subject: [PATCH 4/4] chore: update warning a bit --- validmind/ai/test_descriptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validmind/ai/test_descriptions.py b/validmind/ai/test_descriptions.py index 85921d69d..73f9ae28a 100644 --- a/validmind/ai/test_descriptions.py +++ b/validmind/ai/test_descriptions.py @@ -94,8 +94,8 @@ def _truncate_summary(summary: str, test_id: str, max_tokens: int = 100_000): if len(summary_tokens) > max_tokens: logger.warning( - "Truncating test result due to context length restrictions..." - f"Generated description for {test_id} may be innacurate" + f"Truncating {test_id} due to context length restrictions..." + " Generated description may be innacurate" ) summary = ( encoding.decode(summary_tokens[:max_tokens])