From 6e2020b20d3c99c162e42148e5192b0d33ca7297 Mon Sep 17 00:00:00 2001 From: Juan Date: Wed, 2 Apr 2025 21:54:38 +0200 Subject: [PATCH 1/2] Fix param_grid issue when passing a list of dictionaries --- validmind/tests/comparison.py | 98 +++++++++++++++++++++++------------ validmind/tests/run.py | 9 +++- 2 files changed, 73 insertions(+), 34 deletions(-) diff --git a/validmind/tests/comparison.py b/validmind/tests/comparison.py index 6f94f8865..d1d167047 100644 --- a/validmind/tests/comparison.py +++ b/validmind/tests/comparison.py @@ -146,7 +146,9 @@ def _combine_tables(results: List[TestResult]) -> List[pd.DataFrame]: return [_combine_single_table(results, i) for i in range(len(results[0].tables))] -def _build_input_param_string(result: TestResult, results: List[TestResult]) -> str: +def _build_input_param_string( + result: TestResult, results: List[TestResult], show_params: bool +) -> str: """Build a string repr of unique inputs + params for a figure title""" parts = [] unique_inputs = _get_unique_inputs(results) @@ -162,19 +164,29 @@ def _build_input_param_string(result: TestResult, results: List[TestResult]) -> input_val = _get_input_key(input_obj) parts.append(f"{input_name}={input_val}") - # TODO: revisit this when we can create a value/title to show for params - # unique_params = _get_unique_params(results) - # # if theres only one unique value for a param, don't show it - # # however, if there is only one unique value for all params then show it as - # # long as there is no existing inputs in the parts list - # if result.params: - # should_show = ( - # all(len(unique_params[param_name]) == 1 for param_name in unique_params) - # and not parts - # ) - # for param_name, param_value in result.params.items(): - # if should_show or len(unique_params[param_name]) > 1: - # parts.append(f"{param_name}={param_value}") + # Handle params if show_params is enabled + if show_params and result.params: + unique_params = _get_unique_params(results) + # If there's only one unique value for a param, don't show it + # unless there is only one unique value for all params and no inputs shown + should_show = ( + all(len(unique_params[param_name]) == 1 for param_name in unique_params) + and not parts + ) + for param_name, param_value in result.params.items(): + if should_show or len(unique_params[param_name]) > 1: + # Convert the param_value to a string representation + if isinstance(param_value, list): + # For lists, join elements with commas + str_value = ",".join(str(v) for v in param_value) + elif hasattr(param_value, "__str__"): + # Use string representation if available + str_value = str(param_value) + else: + # Default fallback + str_value = repr(param_value) + + parts.append(f"{param_name}={str_value}") return ", ".join(parts) @@ -207,7 +219,7 @@ def _update_figure_title(figure: Any, input_param_str: str) -> None: raise ValueError(f"Unsupported figure type: {type(figure)}") -def _combine_figures(results: List[TestResult]) -> List[Any]: +def _combine_figures(results: List[TestResult], show_params: bool) -> List[Any]: """Combine figures from multiple test results (gets raw figure objects, not vm Figures)""" combined_figures = [] @@ -216,7 +228,7 @@ def _combine_figures(results: List[TestResult]) -> List[Any]: # update the figure object in-place with the new title _update_figure_title( figure=figure.figure, - input_param_str=_build_input_param_string(result, results), + input_param_str=_build_input_param_string(result, results, show_params), ) combined_figures.append(figure) @@ -279,35 +291,53 @@ def get_comparison_test_configs( A list of test configurations. """ - # Convert list of dicts to dict of lists if necessary + # Convert list of dicts to dict of lists if necessary for input_grid def list_to_dict(grid_list): return {k: [d[k] for d in grid_list] for k in grid_list[0].keys()} + # Handle input_grid the same way as before if isinstance(input_grid, list): input_grid = list_to_dict(input_grid) - if isinstance(param_grid, list): - param_grid = list_to_dict(param_grid) - test_configs = [] - if input_grid and param_grid: - input_combinations = _cartesian_product(input_grid) - param_combinations = _cartesian_product(param_grid) - test_configs = [ - {"inputs": i, "params": p} - for i, p in product(input_combinations, param_combinations) - ] + # Check if param_grid is a list of dictionaries + is_param_grid_list = isinstance(param_grid, list) + + # Special handling for list-based param_grid + if is_param_grid_list: + if input_grid: + # Generate all combinations of input_grid and each param dictionary + input_combinations = _cartesian_product(input_grid) + test_configs = [ + {"inputs": i, "params": p} + for i in input_combinations + for p in param_grid + ] + else: + # Each dictionary in param_grid is a specific test configuration + test_configs = [{"inputs": inputs or {}, "params": p} for p in param_grid] + + # Dictionary-based param_grid + elif param_grid: + if input_grid: + input_combinations = _cartesian_product(input_grid) + param_combinations = _cartesian_product(param_grid) + test_configs = [ + {"inputs": i, "params": p} + for i, p in product(input_combinations, param_combinations) + ] + else: + param_combinations = _cartesian_product(param_grid) + test_configs = [ + {"inputs": inputs or {}, "params": p} for p in param_combinations + ] + # Just input_grid, no param_grid elif input_grid: input_combinations = _cartesian_product(input_grid) test_configs = [ {"inputs": i, "params": params or {}} for i in input_combinations ] - elif param_grid: - param_combinations = _cartesian_product(param_grid) - test_configs = [ - {"inputs": inputs or {}, "params": p} for p in param_combinations - ] return test_configs @@ -333,12 +363,14 @@ def _combine_raw_data(results: List[TestResult]) -> RawData: def combine_results( results: List[TestResult], + show_params: bool, ) -> Tuple[List[Any], Dict[str, List[Any]], Dict[str, List[Any]]]: """ Combine multiple test results into a single set of outputs. Args: results: A list of TestResult objects to combine. + show_params: Whether to show parameter values in figure titles. Returns: A tuple containing: @@ -353,7 +385,7 @@ def combine_results( # handle tables (if any) combined_outputs.extend(_combine_tables(results)) # handle figures (if any) - combined_outputs.extend(_combine_figures(results)) + combined_outputs.extend(_combine_figures(results, show_params)) # handle threshold tests (i.e. tests that have pass/fail bool status) if results[0].passed is not None: combined_outputs.append(all(result.passed for result in results)) diff --git a/validmind/tests/run.py b/validmind/tests/run.py index 161021150..e61a3fa46 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -222,6 +222,7 @@ def _run_comparison_test( params: Union[Dict[str, Any], None], param_grid: Union[Dict[str, List[Any]], List[Dict[str, Any]], None], title: Optional[str] = None, + show_params: bool = True, ): """Run a comparison test i.e. a test that compares multiple outputs of a test across different input and/or param combinations""" @@ -242,6 +243,7 @@ def _run_comparison_test( show=False, generate_description=False, title=title, + show_params=show_params, ) for config in run_test_configs ] @@ -253,7 +255,9 @@ def _run_comparison_test( else: test_doc = describe_test(test_id, raw=True)["Description"] - combined_outputs, combined_inputs, combined_params = combine_results(results) + combined_outputs, combined_inputs, combined_params = combine_results( + results, show_params + ) return build_test_result( outputs=combined_outputs, @@ -297,6 +301,7 @@ def run_test( # noqa: C901 generate_description: bool = True, title: Optional[str] = None, post_process_fn: Union[Callable[[TestResult], None], None] = None, + show_params: bool = True, **kwargs, ) -> TestResult: """Run a ValidMind or custom test @@ -321,6 +326,7 @@ def run_test( # noqa: C901 generate_description (bool, optional): Whether to generate a description. Defaults to True. title (str, optional): Custom title for the test result post_process_fn (Callable[[TestResult], None], optional): Function to post-process the test result + show_params (bool, optional): Whether to include parameter values in figure titles for comparison tests. Defaults to True. Returns: TestResult: A TestResult object containing the test results @@ -358,6 +364,7 @@ def run_test( # noqa: C901 input_grid=input_grid, params=params, param_grid=param_grid, + show_params=show_params, ) elif unit_metrics: From beb4ee46d1eaa7da8e9801d961600a148ab1b8e2 Mon Sep 17 00:00:00 2001 From: Juan Date: Thu, 3 Apr 2025 10:50:48 +0200 Subject: [PATCH 2/2] 2.8.17 --- pyproject.toml | 2 +- validmind/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a7bbd54df..94e353999 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ description = "ValidMind Library" license = "Commercial License" name = "validmind" readme = "README.pypi.md" -version = "2.8.16" +version = "2.8.17" [tool.poetry.dependencies] aiohttp = {extras = ["speedups"], version = "*"} diff --git a/validmind/__version__.py b/validmind/__version__.py index 070a948f5..42e92e36d 100644 --- a/validmind/__version__.py +++ b/validmind/__version__.py @@ -1 +1 @@ -__version__ = "2.8.16" +__version__ = "2.8.17"