From 0010691a4eee987d061bf59e7d4ebacbad9d4f8b Mon Sep 17 00:00:00 2001 From: John Walz Date: Mon, 16 Dec 2024 11:38:20 -0500 Subject: [PATCH 1/7] feat: merging the new test doc changes --- validmind/tests/run.py | 58 +++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 35 deletions(-) diff --git a/validmind/tests/run.py b/validmind/tests/run.py index 24690e7f4..6d7f06555 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -134,11 +134,9 @@ def _get_test_kwargs( def build_test_result( outputs: Union[Any, Tuple[Any, ...]], test_id: str, + test_doc: str, inputs: Dict[str, Union[VMInput, List[VMInput]]], params: Union[Dict[str, Any], None], - doc: str, - description: str, - generate_description: bool = True, title: Optional[str] = None, ): """Build a TestResult object from a set of raw test function outputs""" @@ -150,7 +148,7 @@ def build_test_result( ref_id=ref_id, inputs=inputs, params=params if params else None, # None if empty dict or None - doc=doc, + doc=test_doc, ) if not isinstance(outputs, tuple): @@ -159,16 +157,6 @@ def build_test_result( for item in outputs: process_output(item, result) - result.description = get_result_description( - test_id=test_id, - test_description=description, - tables=result.tables, - figures=result.figures, - metric=result.metric, - should_generate=generate_description, - title=title, - ) - return result @@ -179,7 +167,6 @@ def _run_composite_test( input_grid: Union[Dict[str, List[Any]], List[Dict[str, Any]], None], params: Union[Dict[str, Any], None], param_grid: Union[Dict[str, List[Any]], List[Dict[str, Any]], None], - generate_description: bool, title: Optional[str] = None, ): """Run a composite test i.e. a test made up of multiple metrics""" @@ -201,9 +188,12 @@ def _run_composite_test( if not all(result.metric is not None for result in results): raise ValueError("All tests must return a metric when used as a composite test") - # Create composite doc from all test results + # Create composite docstring from all test results composite_doc = "\n\n".join( - [f"{test_id_to_name(result.result_id)}:\n{result.doc}" for result in results] + [ + f"{test_id_to_name(result.result_id)}:\n{_test_description(result.doc)}" + for result in results + ] ) return build_test_result( @@ -215,13 +205,9 @@ def _run_composite_test( for result in results ], # pass in a single table with metric values as our 'outputs' test_id=test_id, + test_doc=composite_doc, inputs=results[0].inputs, params=results[0].params, - doc=composite_doc, - description="\n\n".join( - [_test_description(result.description, num_lines=1) for result in results] - ), # join truncated (first line only) test descriptions - generate_description=generate_description, title=title, ) @@ -234,7 +220,6 @@ def _run_comparison_test( input_grid: Union[Dict[str, List[Any]], List[Dict[str, Any]], None], params: Union[Dict[str, Any], None], param_grid: Union[Dict[str, List[Any]], List[Dict[str, Any]], None], - generate_description: bool, title: Optional[str] = None, ): """Run a comparison test i.e. a test that compares multiple outputs of a test across @@ -263,22 +248,18 @@ def _run_comparison_test( # composite tests have a test_id thats built from the name if not test_id: test_id = results[0].result_id - description = results[0].description + test_doc = results[0].test_doc else: - description = describe_test(test_id, raw=True)["Description"] + test_doc = describe_test(test_id, raw=True)["Description"] combined_outputs, combined_inputs, combined_params = combine_results(results) - doc = getdoc(load_test(test_id)) - return build_test_result( outputs=tuple(combined_outputs), test_id=test_id, + test_doc=test_doc, inputs=combined_inputs, params=combined_params, - doc=doc, - description=description, - generate_description=generate_description, title=title, ) @@ -394,16 +375,23 @@ def run_test( raw_result = test_func(**input_kwargs, **param_kwargs) - doc = getdoc(test_func) - result = build_test_result( outputs=raw_result, test_id=test_id, + test_doc=getdoc(test_func), inputs=input_kwargs, params=param_kwargs, - doc=doc, - description=doc, - generate_description=generate_description, + title=title, + ) + + if generate_description: + result.description = get_result_description( + test_id=test_id, + test_description=result.doc, + tables=result.tables, + figures=result.figures, + metric=result.metric, + should_generate=generate_description, title=title, ) From 281bfd26bf942658624a95d502d9eef20a41ab2a Mon Sep 17 00:00:00 2001 From: John Walz Date: Thu, 2 Jan 2025 12:39:06 -0500 Subject: [PATCH 2/7] feat: generating description after post processing --- validmind/tests/run.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/validmind/tests/run.py b/validmind/tests/run.py index 6d7f06555..903072e52 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -384,6 +384,12 @@ def run_test( title=title, ) + end_time = time.perf_counter() + result.metadata = _get_run_metadata(duration_seconds=end_time - start_time) + + if post_process_fn: + result = post_process_fn(result) + if generate_description: result.description = get_result_description( test_id=test_id, @@ -395,12 +401,6 @@ def run_test( title=title, ) - end_time = time.perf_counter() - result.metadata = _get_run_metadata(duration_seconds=end_time - start_time) - - if post_process_fn: - result = post_process_fn(result) - if show: result.show() From 00393cb0d255361a7cd2efb0ae7f5bfc6faa6b31 Mon Sep 17 00:00:00 2001 From: John Walz Date: Thu, 2 Jan 2025 12:43:09 -0500 Subject: [PATCH 3/7] chore: fixing linter complaints --- validmind/tests/run.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/validmind/tests/run.py b/validmind/tests/run.py index 903072e52..5607656c3 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -264,7 +264,22 @@ def _run_comparison_test( ) -def run_test( +def _run_test(test_id: TestID, inputs: Dict[str, Any], params: Dict[str, Any]): + """Run a standard test and return a TestResult object""" + test_func = load_test(test_id) + input_kwargs, param_kwargs = _get_test_kwargs(test_func, inputs, params) + raw_result = test_func(**input_kwargs, **param_kwargs) + + return build_test_result( + outputs=raw_result, + test_id=test_id, + test_doc=getdoc(test_func), + inputs=input_kwargs, + params=param_kwargs, + ) + + +def run_test( # noqa: C901 test_id: Union[TestID, None] = None, name: Union[str, None] = None, unit_metrics: Union[List[TestID], None] = None, @@ -367,22 +382,7 @@ def run_test( ) else: - test_func = load_test(test_id) - - input_kwargs, param_kwargs = _get_test_kwargs( - test_func, inputs or {}, params or {} - ) - - raw_result = test_func(**input_kwargs, **param_kwargs) - - result = build_test_result( - outputs=raw_result, - test_id=test_id, - test_doc=getdoc(test_func), - inputs=input_kwargs, - params=param_kwargs, - title=title, - ) + result = _run_test(test_id, inputs, params) end_time = time.perf_counter() result.metadata = _get_run_metadata(duration_seconds=end_time - start_time) From da82823b77d92b7ccc3eb31e69727b64da2cc618 Mon Sep 17 00:00:00 2001 From: John Walz Date: Thu, 2 Jan 2025 12:45:32 -0500 Subject: [PATCH 4/7] fix: fix issue with doc attr --- validmind/tests/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validmind/tests/run.py b/validmind/tests/run.py index 5607656c3..c3be2c457 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -248,7 +248,7 @@ def _run_comparison_test( # composite tests have a test_id thats built from the name if not test_id: test_id = results[0].result_id - test_doc = results[0].test_doc + test_doc = results[0].doc else: test_doc = describe_test(test_id, raw=True)["Description"] From a025dfe68f08c0e9e970a8e650471b825e781473 Mon Sep 17 00:00:00 2001 From: John Walz Date: Thu, 2 Jan 2025 12:52:58 -0500 Subject: [PATCH 5/7] fix: fixing issue with tests that don't have params --- validmind/tests/run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validmind/tests/run.py b/validmind/tests/run.py index c3be2c457..c22336902 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -124,6 +124,9 @@ def _get_test_kwargs( input_kwargs[key] = _input + if not test_func.params: + return input_kwargs, None + param_kwargs = { key: value for key, value in params.items() if key in test_func.params } From 48ea81c6209f377d741b7ec92387a1d0b11e7bde Mon Sep 17 00:00:00 2001 From: John Walz Date: Thu, 2 Jan 2025 13:03:32 -0500 Subject: [PATCH 6/7] fix: quick fix to use empty dict instead of none --- validmind/tests/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validmind/tests/run.py b/validmind/tests/run.py index c22336902..7c0c7c7ea 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -125,7 +125,7 @@ def _get_test_kwargs( input_kwargs[key] = _input if not test_func.params: - return input_kwargs, None + return input_kwargs, {} param_kwargs = { key: value for key, value in params.items() if key in test_func.params From 2d4e7dcd180c2e3ba303d84b51cdadc874015642 Mon Sep 17 00:00:00 2001 From: John Walz Date: Thu, 2 Jan 2025 13:16:18 -0500 Subject: [PATCH 7/7] fix: wrong fix was implemented --- validmind/tests/run.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/validmind/tests/run.py b/validmind/tests/run.py index 7c0c7c7ea..d12511510 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -124,9 +124,6 @@ def _get_test_kwargs( input_kwargs[key] = _input - if not test_func.params: - return input_kwargs, {} - param_kwargs = { key: value for key, value in params.items() if key in test_func.params } @@ -270,7 +267,12 @@ def _run_comparison_test( def _run_test(test_id: TestID, inputs: Dict[str, Any], params: Dict[str, Any]): """Run a standard test and return a TestResult object""" test_func = load_test(test_id) - input_kwargs, param_kwargs = _get_test_kwargs(test_func, inputs, params) + input_kwargs, param_kwargs = _get_test_kwargs( + test_func=test_func, + inputs=inputs or {}, + params=params or {}, + ) + raw_result = test_func(**input_kwargs, **param_kwargs) return build_test_result(