From 395dc1ab3c76be879bb16ceafb62791b59c7255b Mon Sep 17 00:00:00 2001 From: John Walz Date: Wed, 29 Jan 2025 11:45:57 -0500 Subject: [PATCH 1/5] fix: adding raw data support to comparison tests --- validmind/tests/comparison.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/validmind/tests/comparison.py b/validmind/tests/comparison.py index 0cacab0ba..863022bb9 100644 --- a/validmind/tests/comparison.py +++ b/validmind/tests/comparison.py @@ -15,7 +15,7 @@ is_png_image, ) from validmind.vm_models.input import VMInput -from validmind.vm_models.result import ResultTable, TestResult +from validmind.vm_models.result import RawData, ResultTable, TestResult logger = get_logger(__name__) @@ -312,6 +312,25 @@ def list_to_dict(grid_list): return test_configs +def _combine_raw_data(results: List[TestResult]) -> RawData: + """Combine RawData objects""" + attribute_names = results[0].raw_data.__dict__.keys() + + # check that all the raw data objects have the same attributes + for result in results: + if not isinstance(result.raw_data, RawData): + raise ValueError("All raw data objects must be of type RawData") + if result.raw_data.__dict__.keys() != attribute_names: + raise ValueError("RawData objects must have the same attributes") + + return RawData( + **{ + key: [getattr(result.raw_data, key) for result in results] + for key in attribute_names + } + ) + + def combine_results( results: List[TestResult], ) -> Tuple[List[Any], Dict[str, List[Any]], Dict[str, List[Any]]]: From 0364634a85c731511399ece208558c593a5a5764 Mon Sep 17 00:00:00 2001 From: John Walz Date: Wed, 29 Jan 2025 11:52:19 -0500 Subject: [PATCH 2/5] fix: getting raw data working for comparison tests --- validmind/tests/comparison.py | 8 +++++++- validmind/tests/run.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/validmind/tests/comparison.py b/validmind/tests/comparison.py index 863022bb9..1c7e78515 100644 --- a/validmind/tests/comparison.py +++ b/validmind/tests/comparison.py @@ -378,4 +378,10 @@ def combine_results( combined_inputs = _combine_dict_values(combined_inputs) combined_params = _combine_dict_values(combined_params) - return combined_outputs, combined_inputs, combined_params + combined_raw_data = _combine_raw_data(results) + + return ( + (*combined_outputs, combined_raw_data), # raw data is part of the output + combined_inputs, + combined_params, + ) diff --git a/validmind/tests/run.py b/validmind/tests/run.py index 610b9acd6..66dd40e7d 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -256,7 +256,7 @@ def _run_comparison_test( combined_outputs, combined_inputs, combined_params = combine_results(results) return build_test_result( - outputs=tuple(combined_outputs), + outputs=combined_outputs, test_id=test_id, test_doc=test_doc, inputs=combined_inputs, From d51c4da3431d6db38537fb7d5ae8a4fa00d47d93 Mon Sep 17 00:00:00 2001 From: John Walz Date: Fri, 31 Jan 2025 11:12:48 -0500 Subject: [PATCH 3/5] fix: fixing issue that arises when raw data is not present --- validmind/tests/comparison.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/validmind/tests/comparison.py b/validmind/tests/comparison.py index 1c7e78515..6f94f8865 100644 --- a/validmind/tests/comparison.py +++ b/validmind/tests/comparison.py @@ -357,6 +357,9 @@ def combine_results( # handle threshold tests (i.e. tests that have pass/fail bool status) if results[0].passed is not None: combined_outputs.append(all(result.passed for result in results)) + # handle raw data (if any) + if results[0].raw_data: + combined_outputs.append(_combine_raw_data(results)) # combine inputs and params combined_inputs = {} @@ -378,10 +381,8 @@ def combine_results( combined_inputs = _combine_dict_values(combined_inputs) combined_params = _combine_dict_values(combined_params) - combined_raw_data = _combine_raw_data(results) - return ( - (*combined_outputs, combined_raw_data), # raw data is part of the output + tuple(combined_outputs), combined_inputs, combined_params, ) From 44855d04284e2563f5584c3d463ce2268b4dbeea Mon Sep 17 00:00:00 2001 From: John Walz Date: Fri, 31 Jan 2025 11:15:48 -0500 Subject: [PATCH 4/5] chore: using high code instead of full suite notebook for time series to test comparison tests better --- scripts/run_e2e_notebooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_e2e_notebooks.py b/scripts/run_e2e_notebooks.py index c2b45c3a6..81ded70e8 100644 --- a/scripts/run_e2e_notebooks.py +++ b/scripts/run_e2e_notebooks.py @@ -40,7 +40,7 @@ NOTEBOOKS_TO_RUN = [ "notebooks/code_samples/quickstart_customer_churn_full_suite.ipynb", - "notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb", + "notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb", "notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb", "notebooks/how_to/run_unit_metrics.ipynb", "notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb", From ffcfcaef72061f8aad74bf0a0721950825cac4af Mon Sep 17 00:00:00 2001 From: John Walz Date: Fri, 31 Jan 2025 11:20:11 -0500 Subject: [PATCH 5/5] 2.8.7 --- pyproject.toml | 2 +- validmind/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6bf242f07..a332f689a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ description = "ValidMind Library" license = "Commercial License" name = "validmind" readme = "README.pypi.md" -version = "2.8.6" +version = "2.8.7" [tool.poetry.dependencies] python = ">=3.8.1,<3.12" diff --git a/validmind/__version__.py b/validmind/__version__.py index 261017211..c2c8ceb46 100644 --- a/validmind/__version__.py +++ b/validmind/__version__.py @@ -1 +1 @@ -__version__ = "2.8.6" +__version__ = "2.8.7"