From f21201243e2cb7330a1aa015ae864c870297ce6d Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 10:45:05 -0800 Subject: [PATCH 01/31] Updating copyright template --- notebooks/templates/copyright.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/templates/copyright.ipynb b/notebooks/templates/copyright.ipynb index a2e5fcd40..43efe0495 100644 --- a/notebooks/templates/copyright.ipynb +++ b/notebooks/templates/copyright.ipynb @@ -10,7 +10,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } From b9e39737b8b38cd88173d3ad1dd703522d3a315e Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 10:46:18 -0800 Subject: [PATCH 02/31] Updating notebooks with new wording --- .../agents/langgraph_agent_simple_banking_demo.ipynb | 2 +- .../capital_markets/quickstart_option_pricing_models.ipynb | 2 +- .../quickstart_option_pricing_models_quantlib.ipynb | 2 +- .../code_explainer/quickstart_code_explainer_demo.ipynb | 2 +- .../credit_risk/application_scorecard_executive.ipynb | 2 +- .../credit_risk/application_scorecard_full_suite.ipynb | 2 +- .../credit_risk/application_scorecard_with_bias.ipynb | 2 +- .../credit_risk/application_scorecard_with_ml.ipynb | 2 +- .../credit_risk/document_excel_application_scorecard.ipynb | 2 +- .../code_samples/custom_tests/implement_custom_tests.ipynb | 2 +- .../custom_tests/integrate_external_test_providers.ipynb | 2 +- .../model_validation/validate_application_scorecard.ipynb | 2 +- .../nlp_and_llm/foundation_models_integration_demo.ipynb | 2 +- .../nlp_and_llm/foundation_models_summarization_demo.ipynb | 2 +- .../nlp_and_llm/hugging_face_integration_demo.ipynb | 2 +- .../nlp_and_llm/hugging_face_summarization_demo.ipynb | 2 +- notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb | 2 +- notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb | 2 +- notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb | 2 +- notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb | 2 +- .../application_scorecard_ongoing_monitoring.ipynb | 2 +- .../quickstart_customer_churn_ongoing_monitoring.ipynb | 2 +- .../regression/quickstart_regression_full_suite.ipynb | 2 +- .../time_series/quickstart_time_series_full_suite.ipynb | 2 +- .../time_series/quickstart_time_series_high_code.ipynb | 2 +- notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb | 2 +- .../credit_risk/assign_prediction_probabilities.ipynb | 2 +- notebooks/code_sharing/deepeval_integration_demo.ipynb | 2 +- .../code_sharing/embeddings/quickstart_embeddings_demo.ipynb | 2 +- notebooks/code_sharing/geval_deepeval_integration_demo.ipynb | 2 +- .../insurance_mortality/insurance_validation_demo.ipynb | 2 +- .../insurance_mortality/validmind_insurance_POC.ipynb | 2 +- notebooks/code_sharing/llm/rag_llamaindex.ipynb | 2 +- notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb | 2 +- .../code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb | 2 +- notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb | 2 +- .../code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb | 2 +- .../genai_vendor_contract_agent_usecase_poc.ipynb | 2 +- notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb | 2 +- .../operational_deposit/operational_deposit_poc.ipynb | 2 +- .../operational_deposit/synthetic_data_generation.ipynb | 2 +- .../customizing_tests_with_output_templates.ipynb | 2 +- notebooks/code_sharing/plots_and_stats_demo.ipynb | 2 +- notebooks/code_sharing/post_processing_functions.ipynb | 2 +- notebooks/code_sharing/regression/regression_unit_metrics.ipynb | 2 +- notebooks/code_sharing/test_configuration_updates_demo.ipynb | 2 +- notebooks/how_to/assign_scores_complete_tutorial.ipynb | 2 +- notebooks/how_to/configure_dataset_features.ipynb | 2 +- notebooks/how_to/customize_test_result_descriptions.ipynb | 2 +- .../how_to/document_multiple_results_for_the_same_test.ipynb | 2 +- notebooks/how_to/enable_pii_detection.ipynb | 2 +- notebooks/how_to/explore_test_suites.ipynb | 2 +- notebooks/how_to/explore_tests.ipynb | 2 +- notebooks/how_to/filter_input_columns.ipynb | 2 +- notebooks/how_to/load_datasets_predictions.ipynb | 2 +- notebooks/how_to/log_metrics_over_time.ipynb | 2 +- notebooks/how_to/run_documentation_sections.ipynb | 2 +- notebooks/how_to/run_documentation_tests_with_config.ipynb | 2 +- notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb | 2 +- notebooks/how_to/run_tests/2_run_comparison_tests.ipynb | 2 +- notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb | 2 +- notebooks/how_to/run_unit_metrics.ipynb | 2 +- notebooks/how_to/understand_utilize_rawdata.ipynb | 2 +- notebooks/how_to/use_dataset_model_objects.ipynb | 2 +- notebooks/quickstart/quickstart_model_documentation.ipynb | 2 +- notebooks/quickstart/quickstart_model_validation.ipynb | 2 +- notebooks/templates/e2e-notebook.ipynb | 2 +- notebooks/tutorials/model_development/1-set_up_validmind.ipynb | 2 +- .../model_development/2-start_development_process.ipynb | 2 +- .../tutorials/model_development/3-integrate_custom_tests.ipynb | 2 +- .../model_development/4-finalize_testing_documentation.ipynb | 2 +- .../model_validation/1-set_up_validmind_for_validation.ipynb | 2 +- .../tutorials/model_validation/2-start_validation_process.ipynb | 2 +- .../model_validation/3-developing_challenger_model.ipynb | 2 +- .../model_validation/4-finalize_validation_reporting.ipynb | 2 +- 75 files changed, 75 insertions(+), 75 deletions(-) diff --git a/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb b/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb index 6f741793a..06e975796 100644 --- a/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb +++ b/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb @@ -1469,7 +1469,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb index 47e81ac1c..0b5da9a12 100644 --- a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb +++ b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb @@ -2069,7 +2069,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb index cf6881d21..f70539d46 100644 --- a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb +++ b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb @@ -1299,7 +1299,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb b/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb index c770872b6..293f53eef 100644 --- a/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb +++ b/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb @@ -830,7 +830,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb index ed848457e..74f4825af 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb @@ -350,7 +350,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb index 5e3f04529..24d104ae1 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb @@ -870,7 +870,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb index 18ad43454..4a7b0919d 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb @@ -1515,7 +1515,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb index 6fdf3d523..7884fc9fd 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb @@ -1965,7 +1965,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb b/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb index 5602777c2..a82fc9ad4 100644 --- a/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb +++ b/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb @@ -974,7 +974,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb b/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb index 8988dcd0a..54fa6e64f 100644 --- a/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb +++ b/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb @@ -1063,7 +1063,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb b/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb index b8733b981..24bbfd048 100644 --- a/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb +++ b/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb @@ -963,7 +963,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb b/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb index c10a0bd97..88186539c 100644 --- a/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb +++ b/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb @@ -1844,7 +1844,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb b/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb index 3da6b8e3f..d7b72d369 100644 --- a/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb @@ -466,7 +466,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb index 82e910fe5..795b8e199 100644 --- a/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb @@ -513,7 +513,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb b/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb index 825a225d5..6a016f168 100644 --- a/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb @@ -450,7 +450,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb index 8ce761d0e..70546c7cf 100644 --- a/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb @@ -471,7 +471,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb index 49f979ceb..6e49cc111 100644 --- a/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb @@ -988,7 +988,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb b/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb index 6e355c3ca..d2d87974e 100644 --- a/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb @@ -521,7 +521,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb b/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb index 14ee7cd4a..f36b0689e 100644 --- a/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb @@ -1825,7 +1825,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb b/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb index 4222911e8..a90116e9b 100644 --- a/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb @@ -1649,7 +1649,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb b/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb index 9763001c0..64d494c61 100644 --- a/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb +++ b/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb @@ -1327,7 +1327,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb b/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb index 520058e85..020e5deea 100644 --- a/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb +++ b/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb @@ -854,7 +854,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb b/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb index 8f70b4811..59af877cb 100644 --- a/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb +++ b/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb @@ -555,7 +555,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb b/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb index fa8cff0eb..6b6f644a7 100644 --- a/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb +++ b/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb @@ -717,7 +717,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb b/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb index bc3a55d0c..f45144cb8 100644 --- a/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb +++ b/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb @@ -975,7 +975,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb b/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb index e3ca995f2..fdc67c0e3 100644 --- a/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb +++ b/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb @@ -456,7 +456,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb b/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb index da201e8f9..7a7242330 100644 --- a/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb +++ b/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb @@ -744,7 +744,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/deepeval_integration_demo.ipynb b/notebooks/code_sharing/deepeval_integration_demo.ipynb index 478818d71..c97c00bec 100644 --- a/notebooks/code_sharing/deepeval_integration_demo.ipynb +++ b/notebooks/code_sharing/deepeval_integration_demo.ipynb @@ -1070,7 +1070,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb b/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb index ea30e964d..9608d3c09 100644 --- a/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb +++ b/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb @@ -319,7 +319,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb b/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb index 6ba96d172..fa302b110 100644 --- a/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb +++ b/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb @@ -492,7 +492,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb b/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb index f5650ec2b..3777d6216 100644 --- a/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb +++ b/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb @@ -509,7 +509,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb b/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb index a318bf79e..061f13b81 100644 --- a/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb +++ b/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb @@ -1439,7 +1439,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/rag_llamaindex.ipynb b/notebooks/code_sharing/llm/rag_llamaindex.ipynb index ef2399a36..131f6ce82 100644 --- a/notebooks/code_sharing/llm/rag_llamaindex.ipynb +++ b/notebooks/code_sharing/llm/rag_llamaindex.ipynb @@ -559,7 +559,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb b/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb index ca04ef108..818c0fb11 100644 --- a/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb @@ -305,7 +305,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb b/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb index 715c12f08..8fa203141 100644 --- a/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb @@ -812,7 +812,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb b/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb index 302fcf20e..b8a6907c6 100644 --- a/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb @@ -1104,7 +1104,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb b/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb index cf4b6e043..e98880ebc 100644 --- a/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb +++ b/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb @@ -1418,7 +1418,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb b/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb index 37b809bfd..c421d3e45 100644 --- a/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb +++ b/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb @@ -516,7 +516,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb b/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb index dcab4eb9d..5529f2712 100644 --- a/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb +++ b/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb @@ -351,7 +351,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb b/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb index 17b9e05da..82e82aabc 100644 --- a/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb +++ b/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb @@ -1167,7 +1167,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb b/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb index 6fd6d7ff7..4b76dbb01 100644 --- a/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb +++ b/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb @@ -171,7 +171,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb b/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb index 9864e4ff6..2139c25ee 100644 --- a/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb +++ b/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb @@ -825,7 +825,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/plots_and_stats_demo.ipynb b/notebooks/code_sharing/plots_and_stats_demo.ipynb index 33eeaf2d5..7e85a2d4e 100644 --- a/notebooks/code_sharing/plots_and_stats_demo.ipynb +++ b/notebooks/code_sharing/plots_and_stats_demo.ipynb @@ -692,7 +692,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/post_processing_functions.ipynb b/notebooks/code_sharing/post_processing_functions.ipynb index 767ad2a09..74429b4fe 100644 --- a/notebooks/code_sharing/post_processing_functions.ipynb +++ b/notebooks/code_sharing/post_processing_functions.ipynb @@ -617,7 +617,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/regression/regression_unit_metrics.ipynb b/notebooks/code_sharing/regression/regression_unit_metrics.ipynb index ba8655a2d..b405c71fa 100644 --- a/notebooks/code_sharing/regression/regression_unit_metrics.ipynb +++ b/notebooks/code_sharing/regression/regression_unit_metrics.ipynb @@ -768,7 +768,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/test_configuration_updates_demo.ipynb b/notebooks/code_sharing/test_configuration_updates_demo.ipynb index 875cffe41..4310a68c9 100644 --- a/notebooks/code_sharing/test_configuration_updates_demo.ipynb +++ b/notebooks/code_sharing/test_configuration_updates_demo.ipynb @@ -63,7 +63,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/assign_scores_complete_tutorial.ipynb b/notebooks/how_to/assign_scores_complete_tutorial.ipynb index f8af3393b..7339a9a43 100644 --- a/notebooks/how_to/assign_scores_complete_tutorial.ipynb +++ b/notebooks/how_to/assign_scores_complete_tutorial.ipynb @@ -781,7 +781,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/configure_dataset_features.ipynb b/notebooks/how_to/configure_dataset_features.ipynb index 13c9e0896..7685c647b 100644 --- a/notebooks/how_to/configure_dataset_features.ipynb +++ b/notebooks/how_to/configure_dataset_features.ipynb @@ -439,7 +439,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/customize_test_result_descriptions.ipynb b/notebooks/how_to/customize_test_result_descriptions.ipynb index b81f97488..fd83794fc 100644 --- a/notebooks/how_to/customize_test_result_descriptions.ipynb +++ b/notebooks/how_to/customize_test_result_descriptions.ipynb @@ -1009,7 +1009,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb b/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb index 42bc0a805..46b3241f2 100644 --- a/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb +++ b/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb @@ -590,7 +590,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/enable_pii_detection.ipynb b/notebooks/how_to/enable_pii_detection.ipynb index a1dc943c0..06a584d2c 100644 --- a/notebooks/how_to/enable_pii_detection.ipynb +++ b/notebooks/how_to/enable_pii_detection.ipynb @@ -608,7 +608,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/explore_test_suites.ipynb b/notebooks/how_to/explore_test_suites.ipynb index 3e75eff10..4a8f2d6e6 100644 --- a/notebooks/how_to/explore_test_suites.ipynb +++ b/notebooks/how_to/explore_test_suites.ipynb @@ -695,7 +695,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/explore_tests.ipynb b/notebooks/how_to/explore_tests.ipynb index 27119aacd..2076662b0 100644 --- a/notebooks/how_to/explore_tests.ipynb +++ b/notebooks/how_to/explore_tests.ipynb @@ -4394,7 +4394,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/filter_input_columns.ipynb b/notebooks/how_to/filter_input_columns.ipynb index 5c8b2ce09..2ed43196c 100644 --- a/notebooks/how_to/filter_input_columns.ipynb +++ b/notebooks/how_to/filter_input_columns.ipynb @@ -71,7 +71,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/load_datasets_predictions.ipynb b/notebooks/how_to/load_datasets_predictions.ipynb index c10eb88dc..092170f25 100644 --- a/notebooks/how_to/load_datasets_predictions.ipynb +++ b/notebooks/how_to/load_datasets_predictions.ipynb @@ -1024,7 +1024,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/log_metrics_over_time.ipynb b/notebooks/how_to/log_metrics_over_time.ipynb index 53b3ee6ee..e0518a15f 100644 --- a/notebooks/how_to/log_metrics_over_time.ipynb +++ b/notebooks/how_to/log_metrics_over_time.ipynb @@ -926,7 +926,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_documentation_sections.ipynb b/notebooks/how_to/run_documentation_sections.ipynb index 0cfda9183..2fc438286 100644 --- a/notebooks/how_to/run_documentation_sections.ipynb +++ b/notebooks/how_to/run_documentation_sections.ipynb @@ -562,7 +562,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_documentation_tests_with_config.ipynb b/notebooks/how_to/run_documentation_tests_with_config.ipynb index d93dfa534..41b5f980b 100644 --- a/notebooks/how_to/run_documentation_tests_with_config.ipynb +++ b/notebooks/how_to/run_documentation_tests_with_config.ipynb @@ -687,7 +687,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb b/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb index cbab7460b..5c5a70756 100644 --- a/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb +++ b/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb @@ -574,7 +574,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb b/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb index 7b8d8878e..f0f72d781 100644 --- a/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb +++ b/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb @@ -758,7 +758,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb b/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb index 0eeb52e62..8ef2e12c3 100644 --- a/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb +++ b/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb @@ -539,7 +539,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_unit_metrics.ipynb b/notebooks/how_to/run_unit_metrics.ipynb index 29f367d37..6f75f82bb 100644 --- a/notebooks/how_to/run_unit_metrics.ipynb +++ b/notebooks/how_to/run_unit_metrics.ipynb @@ -763,7 +763,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/understand_utilize_rawdata.ipynb b/notebooks/how_to/understand_utilize_rawdata.ipynb index a48e602a6..5901232b6 100644 --- a/notebooks/how_to/understand_utilize_rawdata.ipynb +++ b/notebooks/how_to/understand_utilize_rawdata.ipynb @@ -710,7 +710,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/use_dataset_model_objects.ipynb b/notebooks/how_to/use_dataset_model_objects.ipynb index b491ebb41..50e3c9d23 100644 --- a/notebooks/how_to/use_dataset_model_objects.ipynb +++ b/notebooks/how_to/use_dataset_model_objects.ipynb @@ -973,7 +973,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/quickstart/quickstart_model_documentation.ipynb b/notebooks/quickstart/quickstart_model_documentation.ipynb index 75dd43329..adca7b2c3 100644 --- a/notebooks/quickstart/quickstart_model_documentation.ipynb +++ b/notebooks/quickstart/quickstart_model_documentation.ipynb @@ -860,7 +860,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/quickstart/quickstart_model_validation.ipynb b/notebooks/quickstart/quickstart_model_validation.ipynb index f9470bd70..0744b8d4c 100644 --- a/notebooks/quickstart/quickstart_model_validation.ipynb +++ b/notebooks/quickstart/quickstart_model_validation.ipynb @@ -1181,7 +1181,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/templates/e2e-notebook.ipynb b/notebooks/templates/e2e-notebook.ipynb index 1f7b74be9..d7c369c46 100644 --- a/notebooks/templates/e2e-notebook.ipynb +++ b/notebooks/templates/e2e-notebook.ipynb @@ -271,7 +271,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_development/1-set_up_validmind.ipynb b/notebooks/tutorials/model_development/1-set_up_validmind.ipynb index a42e05c06..95b53729c 100644 --- a/notebooks/tutorials/model_development/1-set_up_validmind.ipynb +++ b/notebooks/tutorials/model_development/1-set_up_validmind.ipynb @@ -448,7 +448,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_development/2-start_development_process.ipynb b/notebooks/tutorials/model_development/2-start_development_process.ipynb index c4b19ea85..6f7ff2093 100644 --- a/notebooks/tutorials/model_development/2-start_development_process.ipynb +++ b/notebooks/tutorials/model_development/2-start_development_process.ipynb @@ -996,7 +996,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb b/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb index 7966f7886..85dd9fca5 100644 --- a/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb +++ b/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb @@ -975,7 +975,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb b/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb index 8e5736665..15c3b55ca 100644 --- a/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb +++ b/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb @@ -974,7 +974,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb b/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb index 807d6caf9..ced975536 100644 --- a/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb +++ b/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb @@ -499,7 +499,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_validation/2-start_validation_process.ipynb b/notebooks/tutorials/model_validation/2-start_validation_process.ipynb index e6fb19454..6a36ea4d8 100644 --- a/notebooks/tutorials/model_validation/2-start_validation_process.ipynb +++ b/notebooks/tutorials/model_validation/2-start_validation_process.ipynb @@ -869,7 +869,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb b/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb index 68398c09d..e99a2a936 100644 --- a/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb +++ b/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb @@ -872,7 +872,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb b/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb index c5c5f94ea..4d7c8cbb9 100644 --- a/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb +++ b/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb @@ -1207,7 +1207,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to the [LICENSE file in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } From 492410c919dcde109f053e84306ee1bb802423b8 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 10:50:17 -0800 Subject: [PATCH 03/31] Cell ID --- notebooks/templates/copyright.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/templates/copyright.ipynb b/notebooks/templates/copyright.ipynb index 43efe0495..cbee123d5 100644 --- a/notebooks/templates/copyright.ipynb +++ b/notebooks/templates/copyright.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "38c5fe72", + "id": "copyright", "metadata": {}, "source": [ "\n", From 435ce140c0e1c237cf1c89a0844fac133f8c8844 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 10:57:40 -0800 Subject: [PATCH 04/31] Copyright cell comment --- notebooks/templates/copyright.ipynb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/notebooks/templates/copyright.ipynb b/notebooks/templates/copyright.ipynb index cbee123d5..1c5cb574e 100644 --- a/notebooks/templates/copyright.ipynb +++ b/notebooks/templates/copyright.ipynb @@ -5,6 +5,8 @@ "id": "copyright", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", From b4279e469052775bbc3dbb30ca72762b92c3ffb1 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:00:18 -0800 Subject: [PATCH 05/31] Renaming mini-templates --- notebooks/templates/README.md | 8 ++--- ...validmind.ipynb => _about-validmind.ipynb} | 0 .../{copyright.ipynb => _copyright.ipynb} | 0 ...nb => _install-initialize-validmind.ipynb} | 0 .../{next-steps.ipynb => _next-steps.ipynb} | 0 ...lidmind.ipynb => _upgrade-validmind.ipynb} | 0 notebooks/templates/e2e_template.py | 32 +++++++++---------- validmind/tests/load.py | 8 ++--- 8 files changed, 24 insertions(+), 24 deletions(-) rename notebooks/templates/{about-validmind.ipynb => _about-validmind.ipynb} (100%) rename notebooks/templates/{copyright.ipynb => _copyright.ipynb} (100%) rename notebooks/templates/{install-initialize-validmind.ipynb => _install-initialize-validmind.ipynb} (100%) rename notebooks/templates/{next-steps.ipynb => _next-steps.ipynb} (100%) rename notebooks/templates/{upgrade-validmind.ipynb => _upgrade-validmind.ipynb} (100%) diff --git a/notebooks/templates/README.md b/notebooks/templates/README.md index 8ad589ab7..6864fefc4 100644 --- a/notebooks/templates/README.md +++ b/notebooks/templates/README.md @@ -12,10 +12,10 @@ make notebook The template generation script/notebook draws from the following mini-templates, should you need to revise them or grab the information from them manually: -- [`about-validmind.ipynb`](about-validmind.ipynb): Conceptual overview of ValidMind & prerequisites. -- [`install-initialize-validmind.ipynb`](install-initialize-validmind.ipynb): ValidMind Library installation & initialization instructions. -- [`next-steps.ipynb`](next-steps.ipynb): Directions to review the generated documentation within the ValidMind Platform & additional learning resources. -- [`upgrade-validmind.ipynb`](upgrade-validmind.ipynb): Instructions for comparing & upgrading versions of the ValidMind Library. +- [`_about-validmind.ipynb`](_about-validmind.ipynb): Conceptual overview of ValidMind & prerequisites. +- [`_install-initialize-validmind.ipynb`](_install-initialize-validmind.ipynb): ValidMind Library installation & initialization instructions. +- [`_next-steps.ipynb`](_next-steps.ipynb): Directions to review the generated documentation within the ValidMind Platform & additional learning resources. +- [`_upgrade-validmind.ipynb`](_upgrade-validmind.ipynb): Instructions for comparing & upgrading versions of the ValidMind Library. ## Add table of contents diff --git a/notebooks/templates/about-validmind.ipynb b/notebooks/templates/_about-validmind.ipynb similarity index 100% rename from notebooks/templates/about-validmind.ipynb rename to notebooks/templates/_about-validmind.ipynb diff --git a/notebooks/templates/copyright.ipynb b/notebooks/templates/_copyright.ipynb similarity index 100% rename from notebooks/templates/copyright.ipynb rename to notebooks/templates/_copyright.ipynb diff --git a/notebooks/templates/install-initialize-validmind.ipynb b/notebooks/templates/_install-initialize-validmind.ipynb similarity index 100% rename from notebooks/templates/install-initialize-validmind.ipynb rename to notebooks/templates/_install-initialize-validmind.ipynb diff --git a/notebooks/templates/next-steps.ipynb b/notebooks/templates/_next-steps.ipynb similarity index 100% rename from notebooks/templates/next-steps.ipynb rename to notebooks/templates/_next-steps.ipynb diff --git a/notebooks/templates/upgrade-validmind.ipynb b/notebooks/templates/_upgrade-validmind.ipynb similarity index 100% rename from notebooks/templates/upgrade-validmind.ipynb rename to notebooks/templates/_upgrade-validmind.ipynb diff --git a/notebooks/templates/e2e_template.py b/notebooks/templates/e2e_template.py index a649767af..7899d27be 100644 --- a/notebooks/templates/e2e_template.py +++ b/notebooks/templates/e2e_template.py @@ -88,8 +88,8 @@ def set_title(filepath): print(f"Error updating notebook: {e}") def add_about(filepath): - """Appends the contents of 'about-validmind.ipynb' to the specified notebook if the user agrees.""" - source_notebook_path = os.path.join(os.path.dirname(__file__), "about-validmind.ipynb") + """Appends the contents of '_about-validmind.ipynb' to the specified notebook if the user agrees.""" + source_notebook_path = os.path.join(os.path.dirname(__file__), "_about-validmind.ipynb") if not os.path.exists(source_notebook_path): print(f"Source notebook '{source_notebook_path}' does not exist") @@ -97,7 +97,7 @@ def add_about(filepath): user_input = input("Do you want to include information about ValidMind? (yes/no): ").strip().lower() if user_input not in ("yes", "y"): - print("Skipping appending 'about-validmind.ipynb'") + print("Skipping appending '_about-validmind.ipynb'") return try: @@ -121,7 +121,7 @@ def add_about(filepath): try: with open(filepath, "w") as target_file: nbformat.write(target_notebook, target_file) - print(f"'about-validmind.ipynb' appended to '{filepath}'") + print(f"'_about-validmind.ipynb' appended to '{filepath}'") except Exception as e: print(f"Error appending notebooks: {e}") @@ -337,8 +337,8 @@ def replace_variables( print_func(f"Error replacing variables in file: {e}") def add_install(filepath): - """Appends the contents of 'install-initialize-validmind.ipynb' to the specified notebook if the user agrees.""" - source_notebook_path = os.path.join(os.path.dirname(__file__), "install-initialize-validmind.ipynb") + """Appends the contents of '_install-initialize-validmind.ipynb' to the specified notebook if the user agrees.""" + source_notebook_path = os.path.join(os.path.dirname(__file__), "_install-initialize-validmind.ipynb") if not os.path.exists(source_notebook_path): print(f"Source notebook '{source_notebook_path}' does not exist") @@ -346,7 +346,7 @@ def add_install(filepath): user_input = input("Do you want to include installation and initialization instructions? (yes/no): ").strip().lower() if user_input not in ("yes", "y"): - print("Skipping appending 'install-initialize-validmind.ipynb'") + print("Skipping appending '_install-initialize-validmind.ipynb'") return try: @@ -370,15 +370,15 @@ def add_install(filepath): try: with open(filepath, "w") as target_file: nbformat.write(target_notebook, target_file) - print(f"'install-initialize-validmind.ipynb' appended to '{filepath}'") + print(f"'_install-initialize-validmind.ipynb' appended to '{filepath}'") except Exception as e: print(f"Error appending notebooks: {e}") replace_variables(filepath) def next_steps(filepath): - """Appends the contents of 'next-steps.ipynb' to the specified notebook if the user agrees.""" - source_notebook_path = os.path.join(os.path.dirname(__file__), "next-steps.ipynb") + """Appends the contents of '_next-steps.ipynb' to the specified notebook if the user agrees.""" + source_notebook_path = os.path.join(os.path.dirname(__file__), "_next-steps.ipynb") if not os.path.exists(source_notebook_path): print(f"Source notebook '{source_notebook_path}' does not exist") @@ -386,7 +386,7 @@ def next_steps(filepath): user_input = input("Do you want to include next steps? (yes/no): ").strip().lower() if user_input not in ("yes", "y"): - print("Skipping appending 'next-steps.ipynb'") + print("Skipping appending '_next-steps.ipynb'") return try: @@ -410,13 +410,13 @@ def next_steps(filepath): try: with open(filepath, "w") as target_file: nbformat.write(target_notebook, target_file) - print(f"'next-steps.ipynb' appended to '{filepath}'") + print(f"'_next-steps.ipynb' appended to '{filepath}'") except Exception as e: print(f"Error appending notebooks: {e}") def add_upgrade(filepath): - """Appends the contents of 'upgrade-validmind.ipynb' to the specified notebook if the user agrees.""" - source_notebook_path = os.path.join(os.path.dirname(__file__), "upgrade-validmind.ipynb") + """Appends the contents of '_upgrade-validmind.ipynb' to the specified notebook if the user agrees.""" + source_notebook_path = os.path.join(os.path.dirname(__file__), "_upgrade-validmind.ipynb") if not os.path.exists(source_notebook_path): print(f"Source notebook '{source_notebook_path}' does not exist") @@ -424,7 +424,7 @@ def add_upgrade(filepath): user_input = input("Do you want to include information about upgrading ValidMind? (yes/no): ").strip().lower() if user_input not in ("yes", "y"): - print("Skipping appending 'upgrade-validmind.ipynb'") + print("Skipping appending '_upgrade-validmind.ipynb'") return try: @@ -448,7 +448,7 @@ def add_upgrade(filepath): try: with open(filepath, "w") as target_file: nbformat.write(target_notebook, target_file) - print(f"'upgrade-validmind.ipynb' appended to '{filepath}'") + print(f"'_upgrade-validmind.ipynb' appended to '{filepath}'") except Exception as e: print(f"Error appending notebooks: {e}") diff --git a/validmind/tests/load.py b/validmind/tests/load.py index 4dc97d11a..9a9f13c53 100644 --- a/validmind/tests/load.py +++ b/validmind/tests/load.py @@ -127,7 +127,9 @@ def _inspect_signature( return inputs, params -def _get_test_function_from_provider(test_id: str, namespace: str) -> Callable[..., Any]: +def _get_test_function_from_provider( + test_id: str, namespace: str +) -> Callable[..., Any]: """Load a test function from the appropriate provider or scorer store. Args: @@ -146,9 +148,7 @@ def _get_test_function_from_provider(test_id: str, namespace: str) -> Callable[. return custom_scorer if not test_provider_store.has_test_provider(namespace): - raise LoadTestError( - f"No test provider found for namespace: {namespace}" - ) + raise LoadTestError(f"No test provider found for namespace: {namespace}") provider = test_provider_store.get_test_provider(namespace) From 8a415bcb5568937c799f9e28710680ec2ee8d1a9 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:02:45 -0800 Subject: [PATCH 06/31] Save point --- notebooks/templates/e2e-notebook.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/notebooks/templates/e2e-notebook.ipynb b/notebooks/templates/e2e-notebook.ipynb index d7c369c46..7ea585be6 100644 --- a/notebooks/templates/e2e-notebook.ipynb +++ b/notebooks/templates/e2e-notebook.ipynb @@ -128,7 +128,7 @@ "\n", "### Insert information about ValidMind\n", "\n", - "You'll be asked if you want to include the contents of [`about-validmind.ipynb`](about-validmind.ipynb). " + "You'll be asked if you want to include the contents of [`_about-validmind.ipynb`](_about-validmind.ipynb). " ] }, { @@ -149,7 +149,7 @@ "\n", "### Insert setup instructions\n", "\n", - "Next, you'll be asked if you want to include the contents of [`install-initialize-validmind.ipynb`](install-initialize-validmind.ipynb). This file includes information for:\n", + "Next, you'll be asked if you want to include the contents of [`_install-initialize-validmind.ipynb`](_install-initialize-validmind.ipynb). This file includes information for:\n", "\n", "- Installing the ValidMind Library\n", "- Initializing the ValidMind Library (registering a sample model, (optional) applying a documentation template, retrieving the code snippet, previewing the applied template)\n", @@ -175,7 +175,7 @@ "\n", "### Insert next steps\n", "\n", - "You'll then asked if you want to include the contents of [`next-steps.ipynb`](next-steps.ipynb). " + "You'll then asked if you want to include the contents of [`_next-steps.ipynb`](_next-steps.ipynb). " ] }, { @@ -196,7 +196,7 @@ "\n", "### Insert upgrade information\n", "\n", - "Finally, you'll be asked if you want to include the contents of [`upgrade-validmind.ipynb`](upgrade-validmind.ipynb). " + "Finally, you'll be asked if you want to include the contents of [`_upgrade-validmind.ipynb`](_upgrade-validmind.ipynb). " ] }, { From db2403936b8de7654bc97b4fffc3ac4760d8aecd Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:04:40 -0800 Subject: [PATCH 07/31] Updating cells with comments --- .../agents/langgraph_agent_simple_banking_demo.ipynb | 2 ++ .../capital_markets/quickstart_option_pricing_models.ipynb | 2 ++ .../quickstart_option_pricing_models_quantlib.ipynb | 2 ++ .../code_explainer/quickstart_code_explainer_demo.ipynb | 2 ++ .../credit_risk/application_scorecard_executive.ipynb | 2 ++ .../credit_risk/application_scorecard_full_suite.ipynb | 2 ++ .../credit_risk/application_scorecard_with_bias.ipynb | 2 ++ .../credit_risk/application_scorecard_with_ml.ipynb | 2 ++ .../credit_risk/document_excel_application_scorecard.ipynb | 2 ++ .../code_samples/custom_tests/implement_custom_tests.ipynb | 2 ++ .../custom_tests/integrate_external_test_providers.ipynb | 2 ++ .../model_validation/validate_application_scorecard.ipynb | 2 ++ .../nlp_and_llm/foundation_models_integration_demo.ipynb | 2 ++ .../nlp_and_llm/foundation_models_summarization_demo.ipynb | 2 ++ .../nlp_and_llm/hugging_face_integration_demo.ipynb | 2 ++ .../nlp_and_llm/hugging_face_summarization_demo.ipynb | 2 ++ notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb | 2 ++ notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb | 2 ++ notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb | 2 ++ notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb | 2 ++ .../application_scorecard_ongoing_monitoring.ipynb | 2 ++ .../quickstart_customer_churn_ongoing_monitoring.ipynb | 2 ++ .../regression/quickstart_regression_full_suite.ipynb | 2 ++ .../time_series/quickstart_time_series_full_suite.ipynb | 2 ++ .../time_series/quickstart_time_series_high_code.ipynb | 2 ++ notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb | 2 ++ .../credit_risk/assign_prediction_probabilities.ipynb | 2 ++ .../code_sharing/embeddings/quickstart_embeddings_demo.ipynb | 2 ++ .../insurance_mortality/insurance_validation_demo.ipynb | 2 ++ .../insurance_mortality/validmind_insurance_POC.ipynb | 2 ++ notebooks/code_sharing/llm/rag_llamaindex.ipynb | 2 ++ notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb | 2 ++ .../code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb | 2 ++ notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb | 2 ++ .../code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb | 2 ++ .../genai_vendor_contract_agent_usecase_poc.ipynb | 2 ++ notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb | 2 ++ .../operational_deposit/operational_deposit_poc.ipynb | 2 ++ .../operational_deposit/synthetic_data_generation.ipynb | 2 ++ .../customizing_tests_with_output_templates.ipynb | 2 ++ notebooks/code_sharing/post_processing_functions.ipynb | 2 ++ notebooks/code_sharing/regression/regression_unit_metrics.ipynb | 2 ++ notebooks/code_sharing/test_configuration_updates_demo.ipynb | 2 ++ notebooks/how_to/configure_dataset_features.ipynb | 2 ++ notebooks/how_to/customize_test_result_descriptions.ipynb | 2 ++ .../how_to/document_multiple_results_for_the_same_test.ipynb | 2 ++ notebooks/how_to/enable_pii_detection.ipynb | 2 ++ notebooks/how_to/explore_test_suites.ipynb | 2 ++ notebooks/how_to/filter_input_columns.ipynb | 2 ++ notebooks/how_to/load_datasets_predictions.ipynb | 2 ++ notebooks/how_to/log_metrics_over_time.ipynb | 2 ++ notebooks/how_to/run_documentation_sections.ipynb | 2 ++ notebooks/how_to/run_documentation_tests_with_config.ipynb | 2 ++ notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb | 2 ++ notebooks/how_to/run_tests/2_run_comparison_tests.ipynb | 2 ++ notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb | 2 ++ notebooks/how_to/run_unit_metrics.ipynb | 2 ++ notebooks/how_to/understand_utilize_rawdata.ipynb | 2 ++ notebooks/how_to/use_dataset_model_objects.ipynb | 2 ++ notebooks/quickstart/quickstart_model_documentation.ipynb | 2 ++ notebooks/quickstart/quickstart_model_validation.ipynb | 2 ++ notebooks/templates/_copyright.ipynb | 2 ++ notebooks/templates/e2e-notebook.ipynb | 2 ++ notebooks/tutorials/model_development/1-set_up_validmind.ipynb | 2 ++ .../model_development/2-start_development_process.ipynb | 2 ++ .../tutorials/model_development/3-integrate_custom_tests.ipynb | 2 ++ .../model_development/4-finalize_testing_documentation.ipynb | 2 ++ .../model_validation/1-set_up_validmind_for_validation.ipynb | 2 ++ .../tutorials/model_validation/2-start_validation_process.ipynb | 2 ++ .../model_validation/3-developing_challenger_model.ipynb | 2 ++ .../model_validation/4-finalize_validation_reporting.ipynb | 2 ++ 71 files changed, 142 insertions(+) diff --git a/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb b/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb index 06e975796..27f0cfb78 100644 --- a/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb +++ b/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb @@ -1464,6 +1464,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb index 0b5da9a12..a5aa91c81 100644 --- a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb +++ b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb @@ -2064,6 +2064,8 @@ "id": "adb15089", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb index f70539d46..8311bdf9c 100644 --- a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb +++ b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb @@ -1294,6 +1294,8 @@ "id": "cab6911c", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb b/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb index 293f53eef..5665049d1 100644 --- a/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb +++ b/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb @@ -825,6 +825,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb index 74f4825af..1801b0d7c 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb @@ -345,6 +345,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb index 24d104ae1..056980c20 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb @@ -865,6 +865,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb index 4a7b0919d..f0e321f68 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb @@ -1510,6 +1510,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb index 7884fc9fd..f5ad09c8d 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb @@ -1960,6 +1960,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb b/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb index a82fc9ad4..bba44f438 100644 --- a/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb +++ b/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb @@ -969,6 +969,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb b/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb index 54fa6e64f..897541235 100644 --- a/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb +++ b/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb @@ -1058,6 +1058,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb b/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb index 24bbfd048..7b1417dfc 100644 --- a/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb +++ b/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb @@ -958,6 +958,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb b/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb index 88186539c..f91c759b1 100644 --- a/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb +++ b/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb @@ -1839,6 +1839,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb b/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb index d7b72d369..2ff9e7eef 100644 --- a/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb @@ -461,6 +461,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb index 795b8e199..5807cb645 100644 --- a/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb @@ -508,6 +508,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb b/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb index 6a016f168..27b686021 100644 --- a/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb @@ -445,6 +445,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb index 70546c7cf..f6aeacb36 100644 --- a/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb @@ -466,6 +466,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb index 6e49cc111..2e45b1e35 100644 --- a/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb @@ -983,6 +983,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb b/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb index d2d87974e..cc305d9c8 100644 --- a/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb @@ -516,6 +516,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb b/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb index f36b0689e..2dbd36a41 100644 --- a/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb @@ -1820,6 +1820,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb b/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb index a90116e9b..60090eb16 100644 --- a/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb @@ -1644,6 +1644,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb b/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb index 64d494c61..a6957f059 100644 --- a/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb +++ b/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb @@ -1322,6 +1322,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb b/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb index 020e5deea..cc730ad5f 100644 --- a/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb +++ b/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb @@ -849,6 +849,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb b/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb index 59af877cb..d233cf04a 100644 --- a/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb +++ b/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb @@ -550,6 +550,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb b/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb index 6b6f644a7..0f1b611ab 100644 --- a/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb +++ b/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb @@ -712,6 +712,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb b/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb index f45144cb8..ad9bd4997 100644 --- a/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb +++ b/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb @@ -970,6 +970,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb b/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb index fdc67c0e3..1ece1c15c 100644 --- a/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb +++ b/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb @@ -451,6 +451,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb b/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb index 7a7242330..2238ebf3b 100644 --- a/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb +++ b/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb @@ -739,6 +739,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb b/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb index 9608d3c09..a1968716e 100644 --- a/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb +++ b/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb @@ -314,6 +314,8 @@ "id": "0a827b65", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb b/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb index 3777d6216..1929e0a5d 100644 --- a/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb +++ b/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb @@ -504,6 +504,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb b/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb index 061f13b81..48d860478 100644 --- a/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb +++ b/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb @@ -1434,6 +1434,8 @@ "id": "f7176334", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/llm/rag_llamaindex.ipynb b/notebooks/code_sharing/llm/rag_llamaindex.ipynb index 131f6ce82..7eeffbd86 100644 --- a/notebooks/code_sharing/llm/rag_llamaindex.ipynb +++ b/notebooks/code_sharing/llm/rag_llamaindex.ipynb @@ -554,6 +554,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb b/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb index 818c0fb11..2466d6b40 100644 --- a/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb @@ -300,6 +300,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb b/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb index 8fa203141..e6746b00a 100644 --- a/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb @@ -807,6 +807,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb b/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb index b8a6907c6..be6de753b 100644 --- a/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb @@ -1099,6 +1099,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb b/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb index e98880ebc..dfc494b30 100644 --- a/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb +++ b/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb @@ -1413,6 +1413,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb b/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb index c421d3e45..c6be300b1 100644 --- a/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb +++ b/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb @@ -511,6 +511,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb b/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb index 5529f2712..36ea66d13 100644 --- a/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb +++ b/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb @@ -346,6 +346,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb b/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb index 82e82aabc..d15c60429 100644 --- a/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb +++ b/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb @@ -1162,6 +1162,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb b/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb index 4b76dbb01..fb7c4d9d9 100644 --- a/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb +++ b/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb @@ -166,6 +166,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb b/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb index 2139c25ee..4b3da0479 100644 --- a/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb +++ b/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb @@ -820,6 +820,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/post_processing_functions.ipynb b/notebooks/code_sharing/post_processing_functions.ipynb index 74429b4fe..cdc2b792a 100644 --- a/notebooks/code_sharing/post_processing_functions.ipynb +++ b/notebooks/code_sharing/post_processing_functions.ipynb @@ -612,6 +612,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/regression/regression_unit_metrics.ipynb b/notebooks/code_sharing/regression/regression_unit_metrics.ipynb index b405c71fa..bcf287279 100644 --- a/notebooks/code_sharing/regression/regression_unit_metrics.ipynb +++ b/notebooks/code_sharing/regression/regression_unit_metrics.ipynb @@ -763,6 +763,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/code_sharing/test_configuration_updates_demo.ipynb b/notebooks/code_sharing/test_configuration_updates_demo.ipynb index 4310a68c9..ec3e8f1db 100644 --- a/notebooks/code_sharing/test_configuration_updates_demo.ipynb +++ b/notebooks/code_sharing/test_configuration_updates_demo.ipynb @@ -58,6 +58,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/configure_dataset_features.ipynb b/notebooks/how_to/configure_dataset_features.ipynb index 7685c647b..70bef799a 100644 --- a/notebooks/how_to/configure_dataset_features.ipynb +++ b/notebooks/how_to/configure_dataset_features.ipynb @@ -434,6 +434,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/customize_test_result_descriptions.ipynb b/notebooks/how_to/customize_test_result_descriptions.ipynb index fd83794fc..eb6ad34e7 100644 --- a/notebooks/how_to/customize_test_result_descriptions.ipynb +++ b/notebooks/how_to/customize_test_result_descriptions.ipynb @@ -1004,6 +1004,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb b/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb index 46b3241f2..8f0b6bc2a 100644 --- a/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb +++ b/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb @@ -585,6 +585,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/enable_pii_detection.ipynb b/notebooks/how_to/enable_pii_detection.ipynb index 06a584d2c..0ad69ac0a 100644 --- a/notebooks/how_to/enable_pii_detection.ipynb +++ b/notebooks/how_to/enable_pii_detection.ipynb @@ -603,6 +603,8 @@ "id": "f1e696a9", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/explore_test_suites.ipynb b/notebooks/how_to/explore_test_suites.ipynb index 4a8f2d6e6..cebf3977b 100644 --- a/notebooks/how_to/explore_test_suites.ipynb +++ b/notebooks/how_to/explore_test_suites.ipynb @@ -690,6 +690,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/filter_input_columns.ipynb b/notebooks/how_to/filter_input_columns.ipynb index 2ed43196c..20fc2875e 100644 --- a/notebooks/how_to/filter_input_columns.ipynb +++ b/notebooks/how_to/filter_input_columns.ipynb @@ -66,6 +66,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/load_datasets_predictions.ipynb b/notebooks/how_to/load_datasets_predictions.ipynb index 092170f25..0a2b2d2b9 100644 --- a/notebooks/how_to/load_datasets_predictions.ipynb +++ b/notebooks/how_to/load_datasets_predictions.ipynb @@ -1019,6 +1019,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/log_metrics_over_time.ipynb b/notebooks/how_to/log_metrics_over_time.ipynb index e0518a15f..2cc5da6a3 100644 --- a/notebooks/how_to/log_metrics_over_time.ipynb +++ b/notebooks/how_to/log_metrics_over_time.ipynb @@ -921,6 +921,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/run_documentation_sections.ipynb b/notebooks/how_to/run_documentation_sections.ipynb index 2fc438286..44e27ca20 100644 --- a/notebooks/how_to/run_documentation_sections.ipynb +++ b/notebooks/how_to/run_documentation_sections.ipynb @@ -557,6 +557,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/run_documentation_tests_with_config.ipynb b/notebooks/how_to/run_documentation_tests_with_config.ipynb index 41b5f980b..0795316e3 100644 --- a/notebooks/how_to/run_documentation_tests_with_config.ipynb +++ b/notebooks/how_to/run_documentation_tests_with_config.ipynb @@ -682,6 +682,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb b/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb index 5c5a70756..1b909773b 100644 --- a/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb +++ b/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb @@ -569,6 +569,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb b/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb index f0f72d781..7c27e48de 100644 --- a/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb +++ b/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb @@ -753,6 +753,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb b/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb index 8ef2e12c3..6a5d57476 100644 --- a/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb +++ b/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb @@ -534,6 +534,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/run_unit_metrics.ipynb b/notebooks/how_to/run_unit_metrics.ipynb index 6f75f82bb..7242c67a9 100644 --- a/notebooks/how_to/run_unit_metrics.ipynb +++ b/notebooks/how_to/run_unit_metrics.ipynb @@ -758,6 +758,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/understand_utilize_rawdata.ipynb b/notebooks/how_to/understand_utilize_rawdata.ipynb index 5901232b6..72425f2c2 100644 --- a/notebooks/how_to/understand_utilize_rawdata.ipynb +++ b/notebooks/how_to/understand_utilize_rawdata.ipynb @@ -705,6 +705,8 @@ "id": "16d292fe", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/how_to/use_dataset_model_objects.ipynb b/notebooks/how_to/use_dataset_model_objects.ipynb index 50e3c9d23..402542003 100644 --- a/notebooks/how_to/use_dataset_model_objects.ipynb +++ b/notebooks/how_to/use_dataset_model_objects.ipynb @@ -968,6 +968,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/quickstart/quickstart_model_documentation.ipynb b/notebooks/quickstart/quickstart_model_documentation.ipynb index adca7b2c3..6369bb8ec 100644 --- a/notebooks/quickstart/quickstart_model_documentation.ipynb +++ b/notebooks/quickstart/quickstart_model_documentation.ipynb @@ -855,6 +855,8 @@ "id": "09c906f3", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/quickstart/quickstart_model_validation.ipynb b/notebooks/quickstart/quickstart_model_validation.ipynb index 0744b8d4c..367263b9b 100644 --- a/notebooks/quickstart/quickstart_model_validation.ipynb +++ b/notebooks/quickstart/quickstart_model_validation.ipynb @@ -1176,6 +1176,8 @@ "id": "f33d4214", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/templates/_copyright.ipynb b/notebooks/templates/_copyright.ipynb index 1c5cb574e..52531a279 100644 --- a/notebooks/templates/_copyright.ipynb +++ b/notebooks/templates/_copyright.ipynb @@ -5,6 +5,8 @@ "id": "copyright", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "\n", diff --git a/notebooks/templates/e2e-notebook.ipynb b/notebooks/templates/e2e-notebook.ipynb index 7ea585be6..01b964ccd 100644 --- a/notebooks/templates/e2e-notebook.ipynb +++ b/notebooks/templates/e2e-notebook.ipynb @@ -266,6 +266,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/tutorials/model_development/1-set_up_validmind.ipynb b/notebooks/tutorials/model_development/1-set_up_validmind.ipynb index 95b53729c..832b69ad5 100644 --- a/notebooks/tutorials/model_development/1-set_up_validmind.ipynb +++ b/notebooks/tutorials/model_development/1-set_up_validmind.ipynb @@ -443,6 +443,8 @@ "id": "39c974f0", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/tutorials/model_development/2-start_development_process.ipynb b/notebooks/tutorials/model_development/2-start_development_process.ipynb index 6f7ff2093..26010cb73 100644 --- a/notebooks/tutorials/model_development/2-start_development_process.ipynb +++ b/notebooks/tutorials/model_development/2-start_development_process.ipynb @@ -991,6 +991,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb b/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb index 85dd9fca5..18f7e8348 100644 --- a/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb +++ b/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb @@ -970,6 +970,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb b/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb index 15c3b55ca..5b0521886 100644 --- a/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb +++ b/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb @@ -969,6 +969,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb b/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb index ced975536..7275fd2ea 100644 --- a/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb +++ b/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb @@ -494,6 +494,8 @@ "id": "47e90901", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/tutorials/model_validation/2-start_validation_process.ipynb b/notebooks/tutorials/model_validation/2-start_validation_process.ipynb index 6a36ea4d8..b5d10b3ac 100644 --- a/notebooks/tutorials/model_validation/2-start_validation_process.ipynb +++ b/notebooks/tutorials/model_validation/2-start_validation_process.ipynb @@ -864,6 +864,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb b/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb index e99a2a936..70ab94083 100644 --- a/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb +++ b/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb @@ -867,6 +867,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", diff --git a/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb b/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb index 4d7c8cbb9..0566bddba 100644 --- a/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb +++ b/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb @@ -1202,6 +1202,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "\n", "\n", "***\n", From 854c4c9d3ef6b2b9a300e1766968c76ea6fcf5cd Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:16:10 -0800 Subject: [PATCH 08/31] Save point --- .../r_demo/r-customer-churn-model.ipynb | 15 +++++++++++++++ notebooks/templates/_copyright.ipynb | 2 -- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb b/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb index ac31e6e70..756e08952 100644 --- a/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb +++ b/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb @@ -481,6 +481,21 @@ "# save the model\n", "saveRDS(lg_reg_model, \"r_log_reg_churn_model.rds\")" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] } ], "metadata": { diff --git a/notebooks/templates/_copyright.ipynb b/notebooks/templates/_copyright.ipynb index 52531a279..1c5cb574e 100644 --- a/notebooks/templates/_copyright.ipynb +++ b/notebooks/templates/_copyright.ipynb @@ -5,8 +5,6 @@ "id": "copyright", "metadata": {}, "source": [ - "\n", - "\n", "\n", "\n", "\n", From dfd8ce84595ed17a4bce48592a23d751f1835790 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:22:37 -0800 Subject: [PATCH 09/31] Copyright insertion test --- notebooks/templates/e2e-notebook.ipynb | 15 +++++++++ notebooks/templates/e2e_template.py | 44 ++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/notebooks/templates/e2e-notebook.ipynb b/notebooks/templates/e2e-notebook.ipynb index 01b964ccd..061c3548d 100644 --- a/notebooks/templates/e2e-notebook.ipynb +++ b/notebooks/templates/e2e-notebook.ipynb @@ -209,6 +209,21 @@ " tm.add_upgrade(filepath)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Insert copyright information" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "if filepath:\n", + " tm.add_copyright(filepath)" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/notebooks/templates/e2e_template.py b/notebooks/templates/e2e_template.py index 7899d27be..2c4dc72b0 100644 --- a/notebooks/templates/e2e_template.py +++ b/notebooks/templates/e2e_template.py @@ -452,6 +452,50 @@ def add_upgrade(filepath): except Exception as e: print(f"Error appending notebooks: {e}") +import os +import uuid +import nbformat + + +def add_copyright(filepath: str) -> None: + """Append the contents of '_copyright.ipynb' to the specified notebook.""" + source_notebook_path = os.path.join(os.path.dirname(__file__), "_copyright.ipynb") + + if not os.path.exists(source_notebook_path): + print(f"Source notebook '{source_notebook_path}' does not exist") + return + + try: + with open(filepath, "r", encoding="utf-8") as target_file: + target_notebook = nbformat.read(target_file, as_version=4) + + with open(source_notebook_path, "r", encoding="utf-8") as source_file: + source_notebook = nbformat.read(source_file, as_version=4) + except Exception as e: + print(f"Error reading notebooks: {e}") + return + + # Make sure every appended cell has a unique ID. + for cell in source_notebook.cells: + original_id = cell.get("id") or f"cell-{uuid.uuid4()}" + cell["id"] = f"{original_id}-{uuid.uuid4()}" + + target_notebook.cells.extend(source_notebook.cells) + + # If you have a helper that enforces IDs, keep it. + try: + target_notebook = ensure_ids(target_notebook) # type: ignore[name-defined] + except NameError: + pass + + try: + with open(filepath, "w", encoding="utf-8") as target_file: + nbformat.write(target_notebook, target_file) + print(f"'_copyright.ipynb' appended to '{filepath}'") + except Exception as e: + print(f"Error appending notebooks: {e}") + + if __name__ == "__main__": script_dir = os.path.dirname(os.path.abspath(__file__)) From 2f4faa758b92cd47782cfa06619fa8f9221e8f1b Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:24:00 -0800 Subject: [PATCH 10/31] Save point --- notebooks/templates/e2e-notebook.ipynb | 4 +++- notebooks/templates/e2e_template.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/notebooks/templates/e2e-notebook.ipynb b/notebooks/templates/e2e-notebook.ipynb index 061c3548d..c6641a3a8 100644 --- a/notebooks/templates/e2e-notebook.ipynb +++ b/notebooks/templates/e2e-notebook.ipynb @@ -217,8 +217,10 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ "if filepath:\n", " tm.add_copyright(filepath)" diff --git a/notebooks/templates/e2e_template.py b/notebooks/templates/e2e_template.py index 2c4dc72b0..767426041 100644 --- a/notebooks/templates/e2e_template.py +++ b/notebooks/templates/e2e_template.py @@ -508,3 +508,4 @@ def add_copyright(filepath: str) -> None: add_install(filepath) next_steps(filepath) add_upgrade(filepath) + add_copyright(filepath) From 3bd187f00ca6e32431941cca2e86937ca9126064 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:27:30 -0800 Subject: [PATCH 11/31] Save point --- notebooks/templates/e2e_template.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/notebooks/templates/e2e_template.py b/notebooks/templates/e2e_template.py index 767426041..3f3a08db5 100644 --- a/notebooks/templates/e2e_template.py +++ b/notebooks/templates/e2e_template.py @@ -452,11 +452,6 @@ def add_upgrade(filepath): except Exception as e: print(f"Error appending notebooks: {e}") -import os -import uuid -import nbformat - - def add_copyright(filepath: str) -> None: """Append the contents of '_copyright.ipynb' to the specified notebook.""" source_notebook_path = os.path.join(os.path.dirname(__file__), "_copyright.ipynb") From 0eaf639b3909738d837270ed1c87ff2cc13329ce Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:30:25 -0800 Subject: [PATCH 12/31] Updating README --- notebooks/templates/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/notebooks/templates/README.md b/notebooks/templates/README.md index 6864fefc4..b4d9db71e 100644 --- a/notebooks/templates/README.md +++ b/notebooks/templates/README.md @@ -16,6 +16,7 @@ The template generation script/notebook draws from the following mini-templates, - [`_install-initialize-validmind.ipynb`](_install-initialize-validmind.ipynb): ValidMind Library installation & initialization instructions. - [`_next-steps.ipynb`](_next-steps.ipynb): Directions to review the generated documentation within the ValidMind Platform & additional learning resources. - [`_upgrade-validmind.ipynb`](_upgrade-validmind.ipynb): Instructions for comparing & upgrading versions of the ValidMind Library. +- [`_copyright.ipynb`](_copyright.ipynb): Copyright and licensing info — mandatory at the bottom of every notebook owned by ValidMind. ## Add table of contents From 5f1b9685df167cde0c39e29804891e4c6f6a943c Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:33:05 -0800 Subject: [PATCH 13/31] Testing notebook copyright script --- Makefile | 3 + scripts/copyright_notebooks.py | 236 +++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100644 scripts/copyright_notebooks.py diff --git a/Makefile b/Makefile index 23aec29cf..0a511591c 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,9 @@ generate-test-id-types: copyright: poetry run python scripts/copyright_files.py +copyright-nb: + poetry run python scripts/copyright_notebooks.py + verify-copyright: poetry run python scripts/verify_copyright.py diff --git a/scripts/copyright_notebooks.py b/scripts/copyright_notebooks.py new file mode 100644 index 000000000..2b18dff2d --- /dev/null +++ b/scripts/copyright_notebooks.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +""" +sync_copyright.py + +- Scans notebooks under repo_root/notebooks by default +- Only processes .ipynb files whose *filename does NOT* start with "_" +- Looks for a markdown cell containing: +- If missing, appends the canonical copyright cell from repo_root/_copyright.ipynb +- If present but different, replaces it with the canonical content + +Usage examples: + python scripts/sync_copyright.py + python scripts/sync_copyright.py --dry-run + python scripts/sync_copyright.py --check + python scripts/sync_copyright.py --root path/to/other/notebooks + python scripts/sync_copyright.py --copyright path/to/_copyright.ipynb +""" + +from __future__ import annotations + +import argparse +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Optional, Tuple + +import nbformat + +MARKER = "" + +# Assumes: +# repo/ +# notebooks/ +# scripts/ +# sync_copyright.py +# _copyright.ipynb +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class Result: + path: Path + status: str # "unchanged" | "updated" | "appended" | "would-update" | "would-append" | "error" + detail: str = "" + + +def _read_notebook(path: Path) -> nbformat.NotebookNode: + return nbformat.read(str(path), as_version=4) + + +def _write_notebook(path: Path, nb: nbformat.NotebookNode) -> None: + nbformat.write(nb, str(path)) + + +def _normalize_source(s: str) -> str: + """ + Normalize for stable comparisons: + - Convert Windows/Mac newlines to \n + - Strip trailing whitespace on each line + - Strip trailing newlines at end + """ + s = s.replace("\r\n", "\n").replace("\r", "\n") + s = "\n".join(line.rstrip() for line in s.split("\n")) + return s.rstrip("\n") + + +def _find_marked_markdown_cell_index(nb: nbformat.NotebookNode) -> Optional[int]: + for i, cell in enumerate(nb.cells): + if cell.get("cell_type") == "markdown": + src = cell.get("source", "") + if isinstance(src, str) and MARKER in src: + return i + return None + + +def _load_canonical_cell_source(copyright_nb_path: Path) -> str: + if not copyright_nb_path.exists(): + raise FileNotFoundError(f"Copyright notebook not found: {copyright_nb_path}") + + nb = _read_notebook(copyright_nb_path) + idx = _find_marked_markdown_cell_index(nb) + if idx is None: + raise ValueError( + f"Could not find a markdown cell containing marker {MARKER!r} in {copyright_nb_path}" + ) + + canonical = nb.cells[idx].get("source", "") + if not isinstance(canonical, str) or not canonical.strip(): + raise ValueError(f"Canonical cell in {copyright_nb_path} is empty or invalid.") + + return canonical + + +def _iter_ipynb_files(root: Path, exclude_dirs: Tuple[str, ...]) -> Iterable[Path]: + for path in root.rglob("*.ipynb"): + # Skip typical noisy directories. + if any(ex in path.parts for ex in exclude_dirs): + continue + + # Only check notebooks whose filename does NOT start with "_" + if path.name.startswith("_"): + continue + + yield path + + +def process_notebook( + nb_path: Path, + canonical_source: str, + *, + dry_run: bool, + check_only: bool, +) -> Result: + try: + nb = _read_notebook(nb_path) + except Exception as e: + return Result(nb_path, "error", f"failed to read: {e}") + + canonical_norm = _normalize_source(canonical_source) + + idx = _find_marked_markdown_cell_index(nb) + if idx is None: + if check_only: + return Result(nb_path, "would-append", "marker not found") + + if not dry_run: + nb.cells.append(nbformat.v4.new_markdown_cell(source=canonical_source)) + try: + _write_notebook(nb_path, nb) + except Exception as e: + return Result(nb_path, "error", f"failed to write: {e}") + + return Result(nb_path, "appended" if not dry_run else "would-append", "marker not found") + + existing = nb.cells[idx].get("source", "") + existing_norm = _normalize_source(existing if isinstance(existing, str) else "") + + if existing_norm == canonical_norm: + return Result(nb_path, "unchanged", "already matches canonical") + + if check_only: + return Result(nb_path, "would-update", "marker found but content differs") + + if not dry_run: + nb.cells[idx]["source"] = canonical_source + try: + _write_notebook(nb_path, nb) + except Exception as e: + return Result(nb_path, "error", f"failed to write: {e}") + + return Result( + nb_path, + "updated" if not dry_run else "would-update", + "marker found but content differed", + ) + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument( + "--root", + type=Path, + default=REPO_ROOT / "notebooks", + help="Root directory to scan (default: repo_root/notebooks)", + ) + p.add_argument( + "--copyright", + type=Path, + default=REPO_ROOT / "_copyright.ipynb", + help="Notebook containing the canonical copyright cell (default: repo_root/_copyright.ipynb)", + ) + p.add_argument( + "--exclude-dirs", + nargs="*", + default=[".ipynb_checkpoints", ".git", ".venv", "venv", "node_modules", "__pycache__"], + help="Directory names to exclude anywhere in the path", + ) + p.add_argument("--dry-run", action="store_true", help="Show what would change but don't write") + p.add_argument( + "--check", + action="store_true", + help="Exit non-zero if any notebook would be updated/appended (no writes)", + ) + p.add_argument( + "--include-copyright-notebook", + action="store_true", + help="Also process the copyright notebook itself (default: skip it)", + ) + args = p.parse_args() + + root = args.root.resolve() + copyright_nb = args.copyright.resolve() + + try: + canonical_source = _load_canonical_cell_source(copyright_nb) + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + return 2 + + results: list[Result] = [] + for nb_path in _iter_ipynb_files(root, tuple(args.exclude_dirs)): + if not args.include_copyright_notebook and nb_path.resolve() == copyright_nb: + continue + results.append( + process_notebook( + nb_path, + canonical_source, + dry_run=args.dry_run, + check_only=args.check, + ) + ) + + # Print per-file changes/errors + counts: dict[str, int] = {} + for r in results: + counts[r.status] = counts.get(r.status, 0) + 1 + if r.status in {"appended", "updated", "would-append", "would-update", "error"}: + print(f"{r.status:12} {r.path} ({r.detail})") + + print("\nSummary:") + for k in sorted(counts.keys()): + print(f" {k:12}: {counts[k]}") + + if args.check: + # Any diff means failure (useful for CI) + if counts.get("would-append", 0) or counts.get("would-update", 0) or counts.get("error", 0): + return 1 + + if counts.get("error", 0): + return 3 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 6ae5fb43eac8ca229775a86517c297c4f7a955dd Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:38:24 -0800 Subject: [PATCH 14/31] Save point --- scripts/copyright_notebooks.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/copyright_notebooks.py b/scripts/copyright_notebooks.py index 2b18dff2d..444d17761 100644 --- a/scripts/copyright_notebooks.py +++ b/scripts/copyright_notebooks.py @@ -5,7 +5,7 @@ - Scans notebooks under repo_root/notebooks by default - Only processes .ipynb files whose *filename does NOT* start with "_" - Looks for a markdown cell containing: -- If missing, appends the canonical copyright cell from repo_root/_copyright.ipynb +- If missing, appends the canonical copyright cell from repo_root/notebook/templates/_copyright.ipynb - If present but different, replaces it with the canonical content Usage examples: @@ -13,7 +13,7 @@ python scripts/sync_copyright.py --dry-run python scripts/sync_copyright.py --check python scripts/sync_copyright.py --root path/to/other/notebooks - python scripts/sync_copyright.py --copyright path/to/_copyright.ipynb + python scripts/sync_copyright.py --copyright path/to/notebook/templates/_copyright.ipynb """ from __future__ import annotations @@ -31,9 +31,10 @@ # Assumes: # repo/ # notebooks/ +# templates/ +# _copyright.ipynb # scripts/ # sync_copyright.py -# _copyright.ipynb REPO_ROOT = Path(__file__).resolve().parents[1] @@ -166,8 +167,8 @@ def main() -> int: p.add_argument( "--copyright", type=Path, - default=REPO_ROOT / "_copyright.ipynb", - help="Notebook containing the canonical copyright cell (default: repo_root/_copyright.ipynb)", + default=REPO_ROOT / "notebooks" / "templates" / "_copyright.ipynb", + help="Notebook containing the canonical copyright cell (default: repo_root/notebooks/templates/_copyright.ipynb)", ) p.add_argument( "--exclude-dirs", From 4969115b9b7804a11a00db31c0554d6348090ea0 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:39:51 -0800 Subject: [PATCH 15/31] Ran the script --- .../deepeval_integration_demo.ipynb | 2191 ++-- .../geval_deepeval_integration_demo.ipynb | 1035 +- .../code_sharing/plots_and_stats_demo.ipynb | 1435 +-- .../code_sharing/r_demo/r-ecm-demo.ipynb | 16 + .../code_sharing/r_demo/r-ecm-model.ipynb | 16 + .../assign_scores_complete_tutorial.ipynb | 1619 +-- notebooks/how_to/explore_tests.ipynb | 8793 +++++++++-------- 7 files changed, 7576 insertions(+), 7529 deletions(-) diff --git a/notebooks/code_sharing/deepeval_integration_demo.ipynb b/notebooks/code_sharing/deepeval_integration_demo.ipynb index c97c00bec..c2698e31f 100644 --- a/notebooks/code_sharing/deepeval_integration_demo.ipynb +++ b/notebooks/code_sharing/deepeval_integration_demo.ipynb @@ -1,1099 +1,1102 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "# DeepEval Integration with ValidMind\n", - "\n", - "Let's learn how to integrate [DeepEval](https://github.com/confident-ai/deepeval) with the ValidMind Library to evaluate Large Language Models (LLMs) and AI agents. This notebook demonstrates how to use DeepEval's summarization metrics within ValidMind's testing infrastructure.\n", - "\n", - "To integrate DeepEval with ValidMind, we'll:\n", - " 1. Set up both frameworks and install required dependencies\n", - " 2. Create a dataset with source texts and generated summaries\n", - " 3. Use ValidMind's Summarization scorer to evaluate summary quality\n", - " 4. Analyze the evaluation results and reasons\n", - " 5. Apply the evaluation pipeline to multiple examples\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## Contents \n", - "- [Introduction](#toc1_) \n", - "- [About DeepEval Integration](#toc2_) \n", - " - [Before you begin](#toc2_1_) \n", - " - [Key concepts](#toc2_2_) \n", - "- [Setting up](#toc3_) \n", - " - [Install required packages](#toc3_1_) \n", - " - [Initialize ValidMind](#toc3_2_) \n", - "- [Basic Usage - Simple Q&A Evaluation](#toc4_) \n", - "- [RAG System Evaluation](#toc5_) \n", - " - [Create test cases](#toc5_1_) \n", - " - [Build dataset](#toc5_2_) \n", - " - [Evaluation metrics](#toc5_3_) \n", - " - [Contextual Relevancy](#toc5_3_1_) \n", - " - [Contextual Precision](#toc5_3_2_) \n", - " - [Contextual Recall](#toc5_3_3_) \n", - "- [LLM Agent Evaluation](#toc6_) \n", - " - [Create test cases](#toc6_1_) \n", - " - [Build dataset](#toc6_2_) \n", - " - [Evaluation metrics](#toc6_3_) \n", - " - [Faithfulness](#toc6_3_1_) \n", - " - [Hallucination](#toc6_3_2_) \n", - " - [Summarization](#toc6_3_3_) \n", - " - [AI Agent Evaluation Metrics](#toc6_3_4_) \n", - "- [In summary](#toc10_) \n", - "- [Next steps](#toc11_) \n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## Introduction\n", - "\n", - "Large Language Model (LLM) evaluation is critical for understanding model performance across different tasks and scenarios. This notebook demonstrates how to integrate DeepEval's comprehensive evaluation framework with ValidMind's testing infrastructure to create a robust LLM evaluation pipeline.\n", - "\n", - "DeepEval provides over 30 evaluation metrics specifically designed for LLMs, covering scenarios from simple Q&A to complex agent interactions. By integrating with ValidMind, you can leverage these metrics within a structured testing framework that supports documentation, collaboration, and compliance requirements.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## About DeepEval Integration\n", - "\n", - "DeepEval is a comprehensive evaluation framework for LLMs that provides metrics for various scenarios including hallucination detection, answer relevancy, faithfulness, and custom evaluation criteria. ValidMind is a platform for managing model risk and documentation through automated testing.\n", - "\n", - "Together, these tools enable comprehensive LLM evaluation within a structured, compliant framework.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "### Before you begin\n", - "\n", - "This notebook assumes you have basic familiarity with Python and Large Language Models. You'll need:\n", - "\n", - "- Python 3.8 or higher\n", - "- Access to OpenAI API (for DeepEval metrics evaluation)\n", - "- ValidMind account and model registration\n", - "\n", - "If you encounter errors due to missing modules, install them with `pip install` and re-run the notebook.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "### Key concepts\n", - "\n", - "**LLMTestCase**: A DeepEval object that represents a single test case with input, expected output, actual output, and optional context.\n", - "\n", - "**LLMAgentDataset**: A ValidMind dataset class that bridges DeepEval test cases with ValidMind's testing infrastructure.\n", - "\n", - "**RAG Evaluation**: Testing retrieval-augmented generation systems that combine document retrieval with generation.\n", - "\n", - "**Agent Evaluation**: Testing LLM agents that can use tools and perform multi-step reasoning.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## Setting up\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "### Install required packages\n", - "\n", - "First, let's install the required packages and set up our environment.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -q validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "### Initialize ValidMind\n", - "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", - "\n", - "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", - "

\n", - "Register with ValidMind
\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load your model identifier credentials from an `.env` file\n", - "%load_ext dotenv\n", - "%dotenv .env\n", - "\n", - "# Or replace with your code snippet\n", - "import validmind as vm\n", - "\n", - "vm.init(\n", - " api_host=\"...\",\n", - " api_key=\"...\",\n", - " api_secret=\"...\",\n", - " model=\"...\",\n", - ")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Core imports\n", - "import pandas as pd\n", - "import warnings\n", - "from deepeval.test_case import LLMTestCase, ToolCall\n", - "from validmind.datasets.llm import LLMAgentDataset\n", - "\n", - "warnings.filterwarnings('ignore')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## Basic Usage - Simple Q&A Evaluation\n", - "\n", - "Let's start with the simplest use case: evaluating a basic question-and-answer interaction with an LLM. This demonstrates how to create LLMTestCase objects and integrate them with ValidMind's dataset infrastructure.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create a simple LLM test case" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "simple_test_cases = [\n", - " LLMTestCase(\n", - " input=\"What is machine learning?\",\n", - " actual_output=\"\"\"Machine learning is a subset of artificial intelligence (AI) that enables \n", - " computers to learn and make decisions from data without being explicitly programmed for every task. \n", - " It uses algorithms to find patterns in data and make predictions or decisions based on those patterns.\"\"\",\n", - " expected_output=\"\"\"Machine learning is a method of data analysis that automates analytical \n", - " model building. It uses algorithms that iteratively learn from data, allowing computers to find \n", - " hidden insights without being explicitly programmed where to look.\"\"\",\n", - " context=[\"Machine learning is a branch of AI that focuses on algorithms that can learn from data.\"],\n", - " retrieval_context=[\"Machine learning is a branch of AI that focuses on algorithms that can learn from data.\"],\n", - " tools_called=[\n", - " ToolCall(\n", - " name=\"search_docs\",\n", - " input={\"query\": \"machine learning definition\"},\n", - " response=\"Found definition of machine learning in documentation.\"\n", - " )\n", - " ],\n", - " expected_tools=[\n", - " ToolCall(\n", - " name=\"search_docs\",\n", - " input={\"query\": \"machine learning definition\"},\n", - " response=\"Found definition of machine learning in documentation.\"\n", - " )\n", - " ]\n", - " ),\n", - " LLMTestCase(\n", - " input=\"What is deep learning?\",\n", - " actual_output=\"\"\"Bananas are yellow fruits that grow on trees in tropical climates. \n", - " They are rich in potassium and make a great healthy snack. You can also use them \n", - " in smoothies and baking.\"\"\",\n", - " expected_output=\"\"\"Deep learning is an advanced machine learning technique that uses neural networks\n", - " with many layers to automatically learn representations of data with multiple levels of abstraction.\n", - " It has enabled major breakthroughs in AI applications.\"\"\",\n", - " context=[\"Deep learning is a specialized machine learning approach that uses deep neural networks to learn from data.\"],\n", - " retrieval_context=[\"Deep learning is a specialized machine learning approach that uses deep neural networks to learn from data.\"],\n", - " tools_called=[\n", - " ToolCall(\n", - " name=\"search_docs\", \n", - " args={\"query\": \"deep learning definition\"},\n", - " response=\"Found definition of deep learning in documentation.\"\n", - " )\n", - " ],\n", - " expected_tools=[\n", - " ToolCall(\n", - " name=\"summarize_docs\", \n", - " args={\"query\": \"deep learning definition\"},\n", - " response=\"Generated summary of deep learning from documentation.\"\n", - " )\n", - " ]\n", - " )\n", - "]\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create LLMAgentDataset from the test case\n", - "Let's create ValidMind dataset from Deepeval's test cases" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nCreating ValidMind dataset...\")\n", - "\n", - "simple_dataset = LLMAgentDataset.from_test_cases(\n", - " test_cases=simple_test_cases,\n", - " input_id=\"simple_qa_dataset\"\n", - ")\n", - "\n", - "print(simple_dataset)\n", - "# Display the dataset\n", - "pd.set_option('display.max_colwidth', 40)\n", - "pd.set_option('display.width', 120)\n", - "pd.set_option('display.max_colwidth', None)\n", - "print(\"\\nDataset preview:\")\n", - "display(simple_dataset.df)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "simple_dataset._df.columns" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Compute metrics using ValidMind scorer interface\n", - "\n", - "Now we'll compute metrics on our dataset using ValidMind's scorer interface. This will help us evaluate how well our model is performing by calculating various metrics like answer relevancy. The scorer interface provides a standardized way to assess model outputs against expected results.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "simple_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.AnswerRelevancy\",\n", - " input_column = \"input\",\n", - " actual_output_column = \"actual_output\",\n", - ")\n", - "simple_dataset._df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "simple_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.Bias\",\n", - " input_column = \"input\",\n", - " actual_output_column = \"actual_output\",\n", - ")\n", - "simple_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## RAG System Evaluation\n", - "\n", - "Now let's evaluate a more complex use case: a Retrieval-Augmented Generation (RAG) system that retrieves relevant documents and generates responses based on them. RAG systems combine document retrieval with text generation, requiring specialized evaluation approaches.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Create test cases" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "print(\"Creating RAG evaluation test cases...\")\n", - "rag_test_cases = [\n", - " LLMTestCase(\n", - " input=\"How do I return a product that doesn't fit?\",\n", - " actual_output=\"\"\"You can return any product within 30 days of purchase for a full refund. \n", - " Simply visit our returns page on the website and follow the step-by-step instructions. \n", - " You'll need your order number and email address. No questions asked!\"\"\",\n", - " expected_output=\"We offer a 30-day return policy for full refunds. Visit our returns page to start the process.\",\n", - " context=[\"Company policy allows 30-day returns for full refund with no restocking fees.\"],\n", - " retrieval_context=[\n", - " \"Return Policy: All items can be returned within 30 days of purchase for a full refund.\",\n", - " \"Return Process: Visit our website's returns page and enter your order details.\",\n", - " \"Customer Service: Available 24/7 to help with returns and refunds.\",\n", - " \"No restocking fees apply to returns within the 30-day window.\"\n", - " ]\n", - " ),\n", - " LLMTestCase(\n", - " input=\"What are your shipping options and costs?\",\n", - " actual_output=\"\"\"We offer three shipping options: Standard (5-7 days, $5.99), \n", - " Express (2-3 days, $9.99), and Overnight (next day, $19.99). \n", - " Free shipping is available on orders over $50 with Standard delivery.\"\"\",\n", - " expected_output=\"Multiple shipping options available with costs ranging from $5.99 to $19.99. Free shipping on orders over $50.\",\n", - " context=[\"Shipping information includes various speed and cost options.\"],\n", - " retrieval_context=[\n", - " \"Standard Shipping: 5-7 business days, $5.99\",\n", - " \"Express Shipping: 2-3 business days, $9.99\", \n", - " \"Overnight Shipping: Next business day, $19.99\",\n", - " \"Free Standard Shipping on orders over $50\"\n", - " ]\n", - " ),\n", - " LLMTestCase(\n", - " input=\"Do you have a warranty on electronics?\",\n", - " actual_output=\"\"\"Yes, all electronics come with a manufacturer's warranty. \n", - " Most items have a 1-year warranty, while premium products may have 2-3 years. \n", - " We also offer extended warranty options for purchase.\"\"\",\n", - " expected_output=\"Electronics include manufacturer warranty, typically 1-year, with extended options available.\",\n", - " context=[\"Electronics warranty information varies by product type and manufacturer.\"],\n", - " retrieval_context=[\n", - " \"Electronics Warranty: Manufacturer warranty included with all electronic items\",\n", - " \"Standard Coverage: 1 year for most electronics\",\n", - " \"Premium Products: May include 2-3 year coverage\",\n", - " \"Extended Warranty: Available for purchase at checkout\"\n", - " ]\n", - " )\n", - "]\n", - "\n", - "print(f\"Created {len(rag_test_cases)} RAG test cases\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Build dataset\n", - "\n", - "In this section, we'll convert our Deepeval LLMTestCase objects into a ValidMind dataset format.\n", - "This allows us to leverage ValidMind's powerful evaluation capabilities while maintaining \n", - "compatibility with Deepeval's test case structure.\n", - "\n", - "The dataset will contain:\n", - "- Input queries\n", - "- Actual model outputs \n", - "- Expected outputs\n", - "- Context information\n", - "- Retrieved context passages\n", - "\n", - "This structured format enables detailed analysis of the RAG system's performance\n", - "across multiple evaluation dimensions.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "rag_dataset = LLMAgentDataset.from_test_cases(\n", - " test_cases=rag_test_cases,\n", - " input_id=\"rag_evaluation_dataset\"\n", - ")\n", - "\n", - "print(f\"RAG Dataset: {rag_dataset}\")\n", - "print(f\"Shape: {rag_dataset.df.shape}\")\n", - "\n", - "# Show dataset structure\n", - "print(\"\\nRAG Dataset Preview:\")\n", - "display(rag_dataset.df[['input', 'actual_output', 'context', 'retrieval_context']].head())\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Evaluation metrics" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Contextual Relevancy\n", - "The Contextual Relevancy metric evaluates how well the retrieved context aligns with the input query.\n", - "It measures whether the context contains the necessary information to answer the query accurately.\n", - "A high relevancy score indicates that the retrieved context is highly relevant and contains the key information needed.\n", - "This helps validate that the RAG system is retrieving appropriate context for the given queries.\n", - "\n", - "\n", - "\n", - "#### Contextual Precision\n", - "The Contextual Precision metric evaluates how well a RAG system ranks retrieved context nodes by relevance to the input query. \n", - "It checks if the most relevant nodes are ranked at the top of the retrieval results.\n", - "A high precision score indicates that the retrieved context is highly relevant to the query and properly ranked.\n", - "This is particularly useful for evaluating RAG systems and ensuring they surface the most relevant information first.\n", - "\n", - "\n", - "\n", - "#### Contextual Recall\n", - "The Contextual Recall metric evaluates how well the retrieved context covers all the information needed to generate the expected output.\n", - "It extracts statements from the expected output and checks how many of them can be attributed to the retrieved context.\n", - "A high recall score indicates that the retrieved context contains all the key information needed to generate the expected response.\n", - "This helps ensure the RAG system retrieves comprehensive context that covers all aspects of the expected answer.\n", - "\n", - "Now we'll evaluate the RAG system's performance using multiple metrics at once. The `assign_scores()` method accepts a list of metrics to evaluate different aspects of the system's behavior. The metrics will add score and reason columns to the dataset, providing quantitative and qualitative feedback on the system's performance. This multi-metric evaluation gives us comprehensive insights into the strengths and potential areas for improvement.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "rag_dataset.assign_scores(\n", - " metrics = [\"validmind.scorer.llm.deepeval.ContextualRelevancy\",\n", - " \"validmind.scorer.llm.deepeval.ContextualPrecision\",\n", - " \"validmind.scorer.llm.deepeval.ContextualRecall\"],\n", - " input_column = \"input\",\n", - " expected_output_column = \"expected_output\",\n", - " retrieval_context_column = \"retrieval_context\",\n", - ")\n", - "display(rag_dataset._df.head(2))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## LLM Agent Evaluation\n", - "\n", - "Let's evaluate LLM agents that can use tools to accomplish tasks. This is one of the most advanced evaluation scenarios, requiring assessment of both response quality and tool usage appropriateness." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "### Create test cases\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Create LLM Agent test cases with tool usage\n", - "print(\"Creating Agent evaluation test cases...\")\n", - "\n", - "# Create test cases\n", - "agent_test_cases = [\n", - " LLMTestCase(\n", - " input=\"What's the weather like in New York City today?\",\n", - " actual_output=\"\"\"Based on current weather data, New York City is experiencing partly cloudy skies \n", - " with a temperature of 72°F (22°C). The humidity is at 60% and there's a light breeze from the west at 8 mph. \n", - " No precipitation is expected today.\"\"\",\n", - " expected_output=\"Current weather in New York shows mild temperatures with partly cloudy conditions.\",\n", - " tools_called=[\n", - " ToolCall(\n", - " name=\"WeatherAPI\",\n", - " description=\"Fetches current weather information for a specified location\",\n", - " input_parameters={\"city\": \"New York City\", \"units\": \"fahrenheit\", \"include_forecast\": False},\n", - " output={\n", - " \"temperature\": 72,\n", - " \"condition\": \"partly_cloudy\", \n", - " \"humidity\": 60,\n", - " \"wind_speed\": 8,\n", - " \"wind_direction\": \"west\"\n", - " },\n", - " reasoning=\"User asked for current weather in NYC, so I need to call the weather API\"\n", - " )\n", - " ],\n", - " expected_tools=[\n", - " ToolCall(\n", - " name=\"WeatherAPI\",\n", - " description=\"Should fetch weather information for New York City\",\n", - " input_parameters={\"city\": \"New York City\"}\n", - " )\n", - " ],\n", - " retrieval_context=[\n", - " \"Temperature: 72°F, Condition: Partly Cloudy, Humidity: 60%, Wind: 8mph from west\",\n", - " \"No precipitation in forecast for today\",\n", - " \"Historical average temperature for this date: 70°F\"\n", - " ]\n", - " ),\n", - " LLMTestCase(\n", - " input=\"Calculate the compound interest on $1000 invested at 5% annually for 3 years\",\n", - " actual_output=\"\"\"Using the compound interest formula A = P(1 + r)^t, where P = $1000, r = 0.05, and t = 3:\n", - " A = 1000(1 + 0.05)^3 = 1000(1.05)^3 = 1000 × 1.157625 = $1,157.63\n", - " The investment will grow to $1,157.63, earning $157.63 in compound interest.\"\"\",\n", - " expected_output=\"The compound interest calculation shows the investment grows to approximately $1,157.63.\",\n", - " tools_called=[\n", - " ToolCall(\n", - " name=\"Calculator\",\n", - " description=\"Performs mathematical calculations\",\n", - " input_parameters={\"expression\": \"1000 * (1.05 ** 3)\", \"precision\": 2},\n", - " output=1157.63,\n", - " reasoning=\"Need to calculate compound interest using the standard formula\"\n", - " )\n", - " ],\n", - " expected_tools=[\n", - " ToolCall(\n", - " name=\"Calculator\", \n", - " description=\"Should perform compound interest calculation\",\n", - " input_parameters={\"calculation_type\": \"compound_interest\"}\n", - " )\n", - " ],\n", - " retrieval_context=[\n", - " \"Calculation result: $1,157.63\",\n", - " \"Formula used: A = P(1 + r)^t\",\n", - " \"Parameters: Principal=$1000, Rate=5%, Time=3 years\"\n", - " ]\n", - " ),\n", - " LLMTestCase(\n", - " input=\"Send an email to john@example.com about our meeting tomorrow at 2 PM\",\n", - " actual_output=\"\"\"I've sent an email to john@example.com with the subject \"Meeting Reminder - Tomorrow at 2 PM\". \n", - " The email confirms your meeting scheduled for tomorrow at 2:00 PM and asks John to confirm his attendance. \n", - " The email was successfully delivered to the recipient.\"\"\",\n", - " expected_output=\"Email sent successfully to john@example.com about the 2 PM meeting tomorrow.\",\n", - " tools_called=[\n", - " ToolCall(\n", - " name=\"EmailSender\",\n", - " description=\"Sends emails to specified recipients\",\n", - " input_parameters={\n", - " \"to\": \"john@example.com\",\n", - " \"subject\": \"Meeting Reminder - Tomorrow at 2 PM\", \n", - " \"body\": \"Hi John,\\n\\nThis is a reminder about our meeting scheduled for tomorrow at 2:00 PM. Please confirm your attendance.\\n\\nBest regards\"\n", - " },\n", - " output={\"status\": \"sent\", \"message_id\": \"msg_12345\", \"timestamp\": \"2024-01-15T10:30:00Z\"},\n", - " reasoning=\"User requested to send email, so I need to use the email tool with appropriate content\"\n", - " )\n", - " ],\n", - " expected_tools=[\n", - " ToolCall(\n", - " name=\"EmailSender\",\n", - " description=\"Should send an email about the meeting\",\n", - " input_parameters={\"recipient\": \"john@example.com\"}\n", - " )\n", - " ],\n", - " retrieval_context=[\n", - " \"Email sent successfully (msg_12345)\",\n", - " \"Recipient: john@example.com\",\n", - " \"Subject: Meeting Reminder - Tomorrow at 2 PM\",\n", - " \"Timestamp: 2024-01-15T10:30:00Z\"\n", - " ]\n", - " )\n", - "]\n", - "print(f\"Created {len(agent_test_cases)} Agent test cases\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Build dataset\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Create Agent dataset\n", - "agent_dataset = LLMAgentDataset.from_test_cases(\n", - " test_cases=agent_test_cases,\n", - " input_id=\"agent_evaluation_dataset\"\n", - ")\n", - "\n", - "print(f\"Agent Dataset: {agent_dataset}\")\n", - "print(f\"Shape: {agent_dataset.df.shape}\")\n", - "\n", - "# Analyze tool usage\n", - "tool_usage = {}\n", - "for case in agent_test_cases:\n", - " if case.tools_called:\n", - " for tool in case.tools_called:\n", - " tool_usage[tool.name] = tool_usage.get(tool.name, 0) + 1\n", - "\n", - "print(\"\\nTool Usage Analysis:\")\n", - "for tool, count in tool_usage.items():\n", - " print(f\" - {tool}: {count} times\")\n", - "\n", - "print(\"\\nAgent Dataset Preview:\")\n", - "display(agent_dataset.df[['input', 'actual_output', 'tools_called']].head())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_dataset.df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Evaluation metrics\n", - "\n", - "\n", - "#### Faithfulness\n", - "The Faithfulness metric evaluates whether the model's output contains any contradictions or hallucinations compared to the provided context. It ensures that the model's response is grounded in and consistent with the given information, rather than making up facts or contradicting the context. A high faithfulness score indicates that the model's output aligns well with the source material.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.Faithfulness\",\n", - " user_input_column = \"input\",\n", - " response_column = \"actual_output\",\n", - " retrieved_contexts_column = \"retrieval_context\",\n", - " )\n", - "agent_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Hallucination\n", - "The Hallucination metric evaluates whether the model's output contains information that is not supported by or contradicts the provided context. It helps identify cases where the model makes up facts or includes details that aren't grounded in the source material. A low hallucination score indicates that the model's response stays faithful to the given context without introducing unsupported information.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.Hallucination\",\n", - " input_column = \"input\",\n", - " actual_output_column = \"actual_output\",\n", - " context_column = \"retrieval_context\",\n", - ")\n", - "agent_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Summarization\n", - "The Summarization metric evaluates how well a model's output summarizes the given context by generating assessment questions to check if the summary is factually aligned with and sufficiently covers the source text. It helps ensure that summaries are accurate, complete, and maintain the key information from the original content without introducing unsupported details or omitting critical points. A high summarization score indicates that the model effectively condenses the source material while preserving its essential meaning.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.Summarization\",\n", - " input_column = \"input\",\n", - " actual_output_column = \"actual_output\",\n", - ")\n", - "agent_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### AI Agent Evaluation Metrics\n", - "\n", - "AI agent evaluation metrics are specialized measurements designed to assess how well autonomous LLM-based agents reason, plan, select and execute tools, and ultimately complete user tasks by analyzing the **full execution trace**—including reasoning steps, tool calls, intermediate decisions, and outcomes—rather than just single input–output pairs.\n", - "\n", - "These metrics are essential because agent failures often occur in ways traditional LLM metrics miss (e.g., choosing the right tool with wrong arguments, creating a good plan but not following it, or completing a task inefficiently).\n", - "\n", - "**DeepEval’s AI agent evaluation framework** breaks evaluation into three layers with corresponding metric categories:\n", - "\n", - "1. **Reasoning Layer** – Evaluates planning and strategy generation:\n", - "\n", - " * *PlanQualityMetric* – how logical, complete, and efficient the agent’s plan is\n", - " * *PlanAdherenceMetric* – whether the agent follows its own plan during execution \n", - "\n", - "2. **Action Layer** – Assesses tool usage and argument generation:\n", - "\n", - " * *ToolCorrectnessMetric* – whether the agent selects and calls the right tools\n", - " * *ArgumentCorrectnessMetric* – whether the agent generates correct tool arguments\n", - "\n", - "3. **Execution Layer** – Measures end-to-end performance:\n", - "\n", - " * *TaskCompletionMetric* – whether the agent successfully completes the intended task\n", - " * *StepEfficiencyMetric* – whether the agent avoids unnecessary or redundant steps\n", - "\n", - "Together, these metrics enable granular diagnosis of agent behavior, help pinpoint where failures occur (reasoning, action, or execution), and support both development benchmarking and production monitoring." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "#### **Reasoning Layer**\n", - "#### PlanQualityMetric\n", - "Let's measures how well the agent generates a plan before acting. A high score means the plan is logical, complete, and efficient." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.PlanQuality\",\n", - " input_column = \"input\",\n", - " actual_output_column = \"actual_output\",\n", - " tools_called_column = \"tools_called\",\n", - ")\n", - "agent_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### PlanAdherenceMetric\n", - "Let's checks whether the agent follows the plan it created. Deviations lower this score and indicate gaps between reasoning and execution." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.PlanAdherence\",\n", - " input_column = \"input\",\n", - " # actual_output_column = \"actual_output\",\n", - " agent_output_column = \"actual_output\",\n", - " tools_called_column = \"tools_called\",\n", - ")\n", - "agent_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "#### **Action Layer**\n", - "#### ToolCorrectnessMetric\n", - "Let's evaluates if the agent selects the appropriate tool for the task. Choosing the wrong tool reduces performance even if reasoning was correct." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.ToolCorrectness\",\n", - " input_column = \"input\",\n", - " actual_output_column = \"actual_output\",\n", - " agent_output_column = \"actual_output\",\n", - " tools_called_column = \"tools_called\",\n", - " expected_tools_column = \"expected_tools\",\n", - ")\n", - "agent_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### ArgumentCorrectnessMetric\n", - "Let's assesses whether the agent provides correct inputs or arguments to the selected tool. Incorrect arguments can lead to failed or unexpected results." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.ArgumentCorrectness\",\n", - " input_column = \"input\",\n", - " actual_output_column = \"actual_output\",\n", - " tools_called_column = \"tools_called\",\n", - ")\n", - "agent_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "#### **Execution Layer**\n", - "#### TaskCompletionMetric\n", - "Let's measures whether the agent successfully completes the overall task. This is the ultimate indicator of success." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.TaskCompletion\",\n", - " input_column = \"input\",\n", - " actual_output_column = \"actual_output\",\n", - " agent_output_column = \"agent_output\",\n", - " tools_called_column = \"tools_called\",\n", - "\n", - ")\n", - "agent_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "#### StepEfficiencyMetric\n", - "Let's looks at how efficiently the agent executes the task, penalizing unnecessary or redundant steps. High efficiency indicates optimized behavior." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# simple_dataset.assign_scores(\n", - "# metrics = \"validmind.scorer.llm.deepeval.StepEfficiency\",\n", - "# input_column = \"input\",\n", - "# actual_output_column = \"actual_output\",\n", - "# agent_output_column = \"actual_output\",\n", - "# tools_called_column = \"tools_called\",\n", - "# )\n", - "# simple_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## In summary\n", - "\n", - "This notebook demonstrated the comprehensive integration between DeepEval and ValidMind for LLM evaluation:\n", - "\n", - "**Key Achievements:**\n", - "- Successfully created and evaluated different types of LLM test cases (Q&A, RAG, Agents)\n", - "- Integrated DeepEval metrics with ValidMind's testing infrastructure\n", - "- Showed how to handle complex agent scenarios with tool usage\n", - "\n", - "**Integration Benefits:**\n", - "- **Comprehensive Coverage**: Evaluate LLMs across 30+ specialized metrics\n", - "- **Structured Documentation**: Leverage ValidMind's compliance and documentation features\n", - "- **Flexibility**: Support for custom metrics and domain-specific evaluation criteria\n", - "- **Production Ready**: Handle real-world LLM evaluation scenarios at scale\n", - "\n", - "The `LLMAgentDataset` class provides a seamless bridge between DeepEval's evaluation capabilities and ValidMind's testing infrastructure, enabling robust LLM evaluation within a structured, compliant framework.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## Next steps\n", - "\n", - "**Explore Advanced Features:**\n", - "- **Continuous Evaluation**: Set up automated LLM evaluation pipelines\n", - "- **A/B Testing**: Compare different LLM models and configurations\n", - "- **Metrics Customization**: Create domain-specific evaluation criteria\n", - "- **Integration Patterns**: Embed evaluation into your LLM development workflow\n", - "\n", - "**Additional Resources:**\n", - "- [ValidMind Library Documentation](https://docs.validmind.ai/developer/validmind-library.html) - Complete API reference and tutorials\n", - "\n", - "**Try These Examples:**\n", - "- Implement custom business-specific evaluation metrics\n", - "- Create automated evaluation pipelines for model deployment\n", - "- Integrate with your existing ML infrastructure and workflows\n", - "- Explore multi-modal evaluation scenarios (text, code, images)\n", - "\n", - "Start building comprehensive LLM evaluation workflows that combine the power of DeepEval's specialized metrics with ValidMind's structured testing and documentation framework.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "***\n", - "\n", - "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", - "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" } - ], - "metadata": { - "kernelspec": { - "display_name": "validmind-1QuffXMV-py3.11", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + }, + "source": [ + "# DeepEval Integration with ValidMind\n", + "\n", + "Let's learn how to integrate [DeepEval](https://github.com/confident-ai/deepeval) with the ValidMind Library to evaluate Large Language Models (LLMs) and AI agents. This notebook demonstrates how to use DeepEval's summarization metrics within ValidMind's testing infrastructure.\n", + "\n", + "To integrate DeepEval with ValidMind, we'll:\n", + " 1. Set up both frameworks and install required dependencies\n", + " 2. Create a dataset with source texts and generated summaries\n", + " 3. Use ValidMind's Summarization scorer to evaluate summary quality\n", + " 4. Analyze the evaluation results and reasons\n", + " 5. Apply the evaluation pipeline to multiple examples\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "## Contents \n", + "- [Introduction](#toc1_) \n", + "- [About DeepEval Integration](#toc2_) \n", + " - [Before you begin](#toc2_1_) \n", + " - [Key concepts](#toc2_2_) \n", + "- [Setting up](#toc3_) \n", + " - [Install required packages](#toc3_1_) \n", + " - [Initialize ValidMind](#toc3_2_) \n", + "- [Basic Usage - Simple Q&A Evaluation](#toc4_) \n", + "- [RAG System Evaluation](#toc5_) \n", + " - [Create test cases](#toc5_1_) \n", + " - [Build dataset](#toc5_2_) \n", + " - [Evaluation metrics](#toc5_3_) \n", + " - [Contextual Relevancy](#toc5_3_1_) \n", + " - [Contextual Precision](#toc5_3_2_) \n", + " - [Contextual Recall](#toc5_3_3_) \n", + "- [LLM Agent Evaluation](#toc6_) \n", + " - [Create test cases](#toc6_1_) \n", + " - [Build dataset](#toc6_2_) \n", + " - [Evaluation metrics](#toc6_3_) \n", + " - [Faithfulness](#toc6_3_1_) \n", + " - [Hallucination](#toc6_3_2_) \n", + " - [Summarization](#toc6_3_3_) \n", + " - [AI Agent Evaluation Metrics](#toc6_3_4_) \n", + "- [In summary](#toc10_) \n", + "- [Next steps](#toc11_) \n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "## Introduction\n", + "\n", + "Large Language Model (LLM) evaluation is critical for understanding model performance across different tasks and scenarios. This notebook demonstrates how to integrate DeepEval's comprehensive evaluation framework with ValidMind's testing infrastructure to create a robust LLM evaluation pipeline.\n", + "\n", + "DeepEval provides over 30 evaluation metrics specifically designed for LLMs, covering scenarios from simple Q&A to complex agent interactions. By integrating with ValidMind, you can leverage these metrics within a structured testing framework that supports documentation, collaboration, and compliance requirements.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "## About DeepEval Integration\n", + "\n", + "DeepEval is a comprehensive evaluation framework for LLMs that provides metrics for various scenarios including hallucination detection, answer relevancy, faithfulness, and custom evaluation criteria. ValidMind is a platform for managing model risk and documentation through automated testing.\n", + "\n", + "Together, these tools enable comprehensive LLM evaluation within a structured, compliant framework.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "### Before you begin\n", + "\n", + "This notebook assumes you have basic familiarity with Python and Large Language Models. You'll need:\n", + "\n", + "- Python 3.8 or higher\n", + "- Access to OpenAI API (for DeepEval metrics evaluation)\n", + "- ValidMind account and model registration\n", + "\n", + "If you encounter errors due to missing modules, install them with `pip install` and re-run the notebook.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "### Key concepts\n", + "\n", + "**LLMTestCase**: A DeepEval object that represents a single test case with input, expected output, actual output, and optional context.\n", + "\n", + "**LLMAgentDataset**: A ValidMind dataset class that bridges DeepEval test cases with ValidMind's testing infrastructure.\n", + "\n", + "**RAG Evaluation**: Testing retrieval-augmented generation systems that combine document retrieval with generation.\n", + "\n", + "**Agent Evaluation**: Testing LLM agents that can use tools and perform multi-step reasoning.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "## Setting up\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "### Install required packages\n", + "\n", + "First, let's install the required packages and set up our environment.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q validmind" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "### Initialize ValidMind\n", + "\n", + "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "\n", + "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", + "

\n", + "Register with ValidMind
\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load your model identifier credentials from an `.env` file\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " api_host=\"...\",\n", + " api_key=\"...\",\n", + " api_secret=\"...\",\n", + " model=\"...\",\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Core imports\n", + "import pandas as pd\n", + "import warnings\n", + "from deepeval.test_case import LLMTestCase, ToolCall\n", + "from validmind.datasets.llm import LLMAgentDataset\n", + "\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "## Basic Usage - Simple Q&A Evaluation\n", + "\n", + "Let's start with the simplest use case: evaluating a basic question-and-answer interaction with an LLM. This demonstrates how to create LLMTestCase objects and integrate them with ValidMind's dataset infrastructure.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a simple LLM test case" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "simple_test_cases = [\n", + " LLMTestCase(\n", + " input=\"What is machine learning?\",\n", + " actual_output=\"\"\"Machine learning is a subset of artificial intelligence (AI) that enables \n", + " computers to learn and make decisions from data without being explicitly programmed for every task. \n", + " It uses algorithms to find patterns in data and make predictions or decisions based on those patterns.\"\"\",\n", + " expected_output=\"\"\"Machine learning is a method of data analysis that automates analytical \n", + " model building. It uses algorithms that iteratively learn from data, allowing computers to find \n", + " hidden insights without being explicitly programmed where to look.\"\"\",\n", + " context=[\"Machine learning is a branch of AI that focuses on algorithms that can learn from data.\"],\n", + " retrieval_context=[\"Machine learning is a branch of AI that focuses on algorithms that can learn from data.\"],\n", + " tools_called=[\n", + " ToolCall(\n", + " name=\"search_docs\",\n", + " input={\"query\": \"machine learning definition\"},\n", + " response=\"Found definition of machine learning in documentation.\"\n", + " )\n", + " ],\n", + " expected_tools=[\n", + " ToolCall(\n", + " name=\"search_docs\",\n", + " input={\"query\": \"machine learning definition\"},\n", + " response=\"Found definition of machine learning in documentation.\"\n", + " )\n", + " ]\n", + " ),\n", + " LLMTestCase(\n", + " input=\"What is deep learning?\",\n", + " actual_output=\"\"\"Bananas are yellow fruits that grow on trees in tropical climates. \n", + " They are rich in potassium and make a great healthy snack. You can also use them \n", + " in smoothies and baking.\"\"\",\n", + " expected_output=\"\"\"Deep learning is an advanced machine learning technique that uses neural networks\n", + " with many layers to automatically learn representations of data with multiple levels of abstraction.\n", + " It has enabled major breakthroughs in AI applications.\"\"\",\n", + " context=[\"Deep learning is a specialized machine learning approach that uses deep neural networks to learn from data.\"],\n", + " retrieval_context=[\"Deep learning is a specialized machine learning approach that uses deep neural networks to learn from data.\"],\n", + " tools_called=[\n", + " ToolCall(\n", + " name=\"search_docs\", \n", + " args={\"query\": \"deep learning definition\"},\n", + " response=\"Found definition of deep learning in documentation.\"\n", + " )\n", + " ],\n", + " expected_tools=[\n", + " ToolCall(\n", + " name=\"summarize_docs\", \n", + " args={\"query\": \"deep learning definition\"},\n", + " response=\"Generated summary of deep learning from documentation.\"\n", + " )\n", + " ]\n", + " )\n", + "]\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create LLMAgentDataset from the test case\n", + "Let's create ValidMind dataset from Deepeval's test cases" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nCreating ValidMind dataset...\")\n", + "\n", + "simple_dataset = LLMAgentDataset.from_test_cases(\n", + " test_cases=simple_test_cases,\n", + " input_id=\"simple_qa_dataset\"\n", + ")\n", + "\n", + "print(simple_dataset)\n", + "# Display the dataset\n", + "pd.set_option('display.max_colwidth', 40)\n", + "pd.set_option('display.width', 120)\n", + "pd.set_option('display.max_colwidth', None)\n", + "print(\"\\nDataset preview:\")\n", + "display(simple_dataset.df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "simple_dataset._df.columns" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Compute metrics using ValidMind scorer interface\n", + "\n", + "Now we'll compute metrics on our dataset using ValidMind's scorer interface. This will help us evaluate how well our model is performing by calculating various metrics like answer relevancy. The scorer interface provides a standardized way to assess model outputs against expected results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "simple_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.AnswerRelevancy\",\n", + " input_column = \"input\",\n", + " actual_output_column = \"actual_output\",\n", + ")\n", + "simple_dataset._df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "simple_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.Bias\",\n", + " input_column = \"input\",\n", + " actual_output_column = \"actual_output\",\n", + ")\n", + "simple_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "## RAG System Evaluation\n", + "\n", + "Now let's evaluate a more complex use case: a Retrieval-Augmented Generation (RAG) system that retrieves relevant documents and generates responses based on them. RAG systems combine document retrieval with text generation, requiring specialized evaluation approaches.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Create test cases" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "print(\"Creating RAG evaluation test cases...\")\n", + "rag_test_cases = [\n", + " LLMTestCase(\n", + " input=\"How do I return a product that doesn't fit?\",\n", + " actual_output=\"\"\"You can return any product within 30 days of purchase for a full refund. \n", + " Simply visit our returns page on the website and follow the step-by-step instructions. \n", + " You'll need your order number and email address. No questions asked!\"\"\",\n", + " expected_output=\"We offer a 30-day return policy for full refunds. Visit our returns page to start the process.\",\n", + " context=[\"Company policy allows 30-day returns for full refund with no restocking fees.\"],\n", + " retrieval_context=[\n", + " \"Return Policy: All items can be returned within 30 days of purchase for a full refund.\",\n", + " \"Return Process: Visit our website's returns page and enter your order details.\",\n", + " \"Customer Service: Available 24/7 to help with returns and refunds.\",\n", + " \"No restocking fees apply to returns within the 30-day window.\"\n", + " ]\n", + " ),\n", + " LLMTestCase(\n", + " input=\"What are your shipping options and costs?\",\n", + " actual_output=\"\"\"We offer three shipping options: Standard (5-7 days, $5.99), \n", + " Express (2-3 days, $9.99), and Overnight (next day, $19.99). \n", + " Free shipping is available on orders over $50 with Standard delivery.\"\"\",\n", + " expected_output=\"Multiple shipping options available with costs ranging from $5.99 to $19.99. Free shipping on orders over $50.\",\n", + " context=[\"Shipping information includes various speed and cost options.\"],\n", + " retrieval_context=[\n", + " \"Standard Shipping: 5-7 business days, $5.99\",\n", + " \"Express Shipping: 2-3 business days, $9.99\", \n", + " \"Overnight Shipping: Next business day, $19.99\",\n", + " \"Free Standard Shipping on orders over $50\"\n", + " ]\n", + " ),\n", + " LLMTestCase(\n", + " input=\"Do you have a warranty on electronics?\",\n", + " actual_output=\"\"\"Yes, all electronics come with a manufacturer's warranty. \n", + " Most items have a 1-year warranty, while premium products may have 2-3 years. \n", + " We also offer extended warranty options for purchase.\"\"\",\n", + " expected_output=\"Electronics include manufacturer warranty, typically 1-year, with extended options available.\",\n", + " context=[\"Electronics warranty information varies by product type and manufacturer.\"],\n", + " retrieval_context=[\n", + " \"Electronics Warranty: Manufacturer warranty included with all electronic items\",\n", + " \"Standard Coverage: 1 year for most electronics\",\n", + " \"Premium Products: May include 2-3 year coverage\",\n", + " \"Extended Warranty: Available for purchase at checkout\"\n", + " ]\n", + " )\n", + "]\n", + "\n", + "print(f\"Created {len(rag_test_cases)} RAG test cases\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Build dataset\n", + "\n", + "In this section, we'll convert our Deepeval LLMTestCase objects into a ValidMind dataset format.\n", + "This allows us to leverage ValidMind's powerful evaluation capabilities while maintaining \n", + "compatibility with Deepeval's test case structure.\n", + "\n", + "The dataset will contain:\n", + "- Input queries\n", + "- Actual model outputs \n", + "- Expected outputs\n", + "- Context information\n", + "- Retrieved context passages\n", + "\n", + "This structured format enables detailed analysis of the RAG system's performance\n", + "across multiple evaluation dimensions.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rag_dataset = LLMAgentDataset.from_test_cases(\n", + " test_cases=rag_test_cases,\n", + " input_id=\"rag_evaluation_dataset\"\n", + ")\n", + "\n", + "print(f\"RAG Dataset: {rag_dataset}\")\n", + "print(f\"Shape: {rag_dataset.df.shape}\")\n", + "\n", + "# Show dataset structure\n", + "print(\"\\nRAG Dataset Preview:\")\n", + "display(rag_dataset.df[['input', 'actual_output', 'context', 'retrieval_context']].head())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Evaluation metrics" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Contextual Relevancy\n", + "The Contextual Relevancy metric evaluates how well the retrieved context aligns with the input query.\n", + "It measures whether the context contains the necessary information to answer the query accurately.\n", + "A high relevancy score indicates that the retrieved context is highly relevant and contains the key information needed.\n", + "This helps validate that the RAG system is retrieving appropriate context for the given queries.\n", + "\n", + "\n", + "\n", + "#### Contextual Precision\n", + "The Contextual Precision metric evaluates how well a RAG system ranks retrieved context nodes by relevance to the input query. \n", + "It checks if the most relevant nodes are ranked at the top of the retrieval results.\n", + "A high precision score indicates that the retrieved context is highly relevant to the query and properly ranked.\n", + "This is particularly useful for evaluating RAG systems and ensuring they surface the most relevant information first.\n", + "\n", + "\n", + "\n", + "#### Contextual Recall\n", + "The Contextual Recall metric evaluates how well the retrieved context covers all the information needed to generate the expected output.\n", + "It extracts statements from the expected output and checks how many of them can be attributed to the retrieved context.\n", + "A high recall score indicates that the retrieved context contains all the key information needed to generate the expected response.\n", + "This helps ensure the RAG system retrieves comprehensive context that covers all aspects of the expected answer.\n", + "\n", + "Now we'll evaluate the RAG system's performance using multiple metrics at once. The `assign_scores()` method accepts a list of metrics to evaluate different aspects of the system's behavior. The metrics will add score and reason columns to the dataset, providing quantitative and qualitative feedback on the system's performance. This multi-metric evaluation gives us comprehensive insights into the strengths and potential areas for improvement.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rag_dataset.assign_scores(\n", + " metrics = [\"validmind.scorer.llm.deepeval.ContextualRelevancy\",\n", + " \"validmind.scorer.llm.deepeval.ContextualPrecision\",\n", + " \"validmind.scorer.llm.deepeval.ContextualRecall\"],\n", + " input_column = \"input\",\n", + " expected_output_column = \"expected_output\",\n", + " retrieval_context_column = \"retrieval_context\",\n", + ")\n", + "display(rag_dataset._df.head(2))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "## LLM Agent Evaluation\n", + "\n", + "Let's evaluate LLM agents that can use tools to accomplish tasks. This is one of the most advanced evaluation scenarios, requiring assessment of both response quality and tool usage appropriateness." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "### Create test cases\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create LLM Agent test cases with tool usage\n", + "print(\"Creating Agent evaluation test cases...\")\n", + "\n", + "# Create test cases\n", + "agent_test_cases = [\n", + " LLMTestCase(\n", + " input=\"What's the weather like in New York City today?\",\n", + " actual_output=\"\"\"Based on current weather data, New York City is experiencing partly cloudy skies \n", + " with a temperature of 72°F (22°C). The humidity is at 60% and there's a light breeze from the west at 8 mph. \n", + " No precipitation is expected today.\"\"\",\n", + " expected_output=\"Current weather in New York shows mild temperatures with partly cloudy conditions.\",\n", + " tools_called=[\n", + " ToolCall(\n", + " name=\"WeatherAPI\",\n", + " description=\"Fetches current weather information for a specified location\",\n", + " input_parameters={\"city\": \"New York City\", \"units\": \"fahrenheit\", \"include_forecast\": False},\n", + " output={\n", + " \"temperature\": 72,\n", + " \"condition\": \"partly_cloudy\", \n", + " \"humidity\": 60,\n", + " \"wind_speed\": 8,\n", + " \"wind_direction\": \"west\"\n", + " },\n", + " reasoning=\"User asked for current weather in NYC, so I need to call the weather API\"\n", + " )\n", + " ],\n", + " expected_tools=[\n", + " ToolCall(\n", + " name=\"WeatherAPI\",\n", + " description=\"Should fetch weather information for New York City\",\n", + " input_parameters={\"city\": \"New York City\"}\n", + " )\n", + " ],\n", + " retrieval_context=[\n", + " \"Temperature: 72°F, Condition: Partly Cloudy, Humidity: 60%, Wind: 8mph from west\",\n", + " \"No precipitation in forecast for today\",\n", + " \"Historical average temperature for this date: 70°F\"\n", + " ]\n", + " ),\n", + " LLMTestCase(\n", + " input=\"Calculate the compound interest on $1000 invested at 5% annually for 3 years\",\n", + " actual_output=\"\"\"Using the compound interest formula A = P(1 + r)^t, where P = $1000, r = 0.05, and t = 3:\n", + " A = 1000(1 + 0.05)^3 = 1000(1.05)^3 = 1000 × 1.157625 = $1,157.63\n", + " The investment will grow to $1,157.63, earning $157.63 in compound interest.\"\"\",\n", + " expected_output=\"The compound interest calculation shows the investment grows to approximately $1,157.63.\",\n", + " tools_called=[\n", + " ToolCall(\n", + " name=\"Calculator\",\n", + " description=\"Performs mathematical calculations\",\n", + " input_parameters={\"expression\": \"1000 * (1.05 ** 3)\", \"precision\": 2},\n", + " output=1157.63,\n", + " reasoning=\"Need to calculate compound interest using the standard formula\"\n", + " )\n", + " ],\n", + " expected_tools=[\n", + " ToolCall(\n", + " name=\"Calculator\", \n", + " description=\"Should perform compound interest calculation\",\n", + " input_parameters={\"calculation_type\": \"compound_interest\"}\n", + " )\n", + " ],\n", + " retrieval_context=[\n", + " \"Calculation result: $1,157.63\",\n", + " \"Formula used: A = P(1 + r)^t\",\n", + " \"Parameters: Principal=$1000, Rate=5%, Time=3 years\"\n", + " ]\n", + " ),\n", + " LLMTestCase(\n", + " input=\"Send an email to john@example.com about our meeting tomorrow at 2 PM\",\n", + " actual_output=\"\"\"I've sent an email to john@example.com with the subject \"Meeting Reminder - Tomorrow at 2 PM\". \n", + " The email confirms your meeting scheduled for tomorrow at 2:00 PM and asks John to confirm his attendance. \n", + " The email was successfully delivered to the recipient.\"\"\",\n", + " expected_output=\"Email sent successfully to john@example.com about the 2 PM meeting tomorrow.\",\n", + " tools_called=[\n", + " ToolCall(\n", + " name=\"EmailSender\",\n", + " description=\"Sends emails to specified recipients\",\n", + " input_parameters={\n", + " \"to\": \"john@example.com\",\n", + " \"subject\": \"Meeting Reminder - Tomorrow at 2 PM\", \n", + " \"body\": \"Hi John,\\n\\nThis is a reminder about our meeting scheduled for tomorrow at 2:00 PM. Please confirm your attendance.\\n\\nBest regards\"\n", + " },\n", + " output={\"status\": \"sent\", \"message_id\": \"msg_12345\", \"timestamp\": \"2024-01-15T10:30:00Z\"},\n", + " reasoning=\"User requested to send email, so I need to use the email tool with appropriate content\"\n", + " )\n", + " ],\n", + " expected_tools=[\n", + " ToolCall(\n", + " name=\"EmailSender\",\n", + " description=\"Should send an email about the meeting\",\n", + " input_parameters={\"recipient\": \"john@example.com\"}\n", + " )\n", + " ],\n", + " retrieval_context=[\n", + " \"Email sent successfully (msg_12345)\",\n", + " \"Recipient: john@example.com\",\n", + " \"Subject: Meeting Reminder - Tomorrow at 2 PM\",\n", + " \"Timestamp: 2024-01-15T10:30:00Z\"\n", + " ]\n", + " )\n", + "]\n", + "print(f\"Created {len(agent_test_cases)} Agent test cases\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Build dataset\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create Agent dataset\n", + "agent_dataset = LLMAgentDataset.from_test_cases(\n", + " test_cases=agent_test_cases,\n", + " input_id=\"agent_evaluation_dataset\"\n", + ")\n", + "\n", + "print(f\"Agent Dataset: {agent_dataset}\")\n", + "print(f\"Shape: {agent_dataset.df.shape}\")\n", + "\n", + "# Analyze tool usage\n", + "tool_usage = {}\n", + "for case in agent_test_cases:\n", + " if case.tools_called:\n", + " for tool in case.tools_called:\n", + " tool_usage[tool.name] = tool_usage.get(tool.name, 0) + 1\n", + "\n", + "print(\"\\nTool Usage Analysis:\")\n", + "for tool, count in tool_usage.items():\n", + " print(f\" - {tool}: {count} times\")\n", + "\n", + "print(\"\\nAgent Dataset Preview:\")\n", + "display(agent_dataset.df[['input', 'actual_output', 'tools_called']].head())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agent_dataset.df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Evaluation metrics\n", + "\n", + "\n", + "#### Faithfulness\n", + "The Faithfulness metric evaluates whether the model's output contains any contradictions or hallucinations compared to the provided context. It ensures that the model's response is grounded in and consistent with the given information, rather than making up facts or contradicting the context. A high faithfulness score indicates that the model's output aligns well with the source material.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agent_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.Faithfulness\",\n", + " user_input_column = \"input\",\n", + " response_column = \"actual_output\",\n", + " retrieved_contexts_column = \"retrieval_context\",\n", + " )\n", + "agent_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Hallucination\n", + "The Hallucination metric evaluates whether the model's output contains information that is not supported by or contradicts the provided context. It helps identify cases where the model makes up facts or includes details that aren't grounded in the source material. A low hallucination score indicates that the model's response stays faithful to the given context without introducing unsupported information.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agent_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.Hallucination\",\n", + " input_column = \"input\",\n", + " actual_output_column = \"actual_output\",\n", + " context_column = \"retrieval_context\",\n", + ")\n", + "agent_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Summarization\n", + "The Summarization metric evaluates how well a model's output summarizes the given context by generating assessment questions to check if the summary is factually aligned with and sufficiently covers the source text. It helps ensure that summaries are accurate, complete, and maintain the key information from the original content without introducing unsupported details or omitting critical points. A high summarization score indicates that the model effectively condenses the source material while preserving its essential meaning.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agent_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.Summarization\",\n", + " input_column = \"input\",\n", + " actual_output_column = \"actual_output\",\n", + ")\n", + "agent_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### AI Agent Evaluation Metrics\n", + "\n", + "AI agent evaluation metrics are specialized measurements designed to assess how well autonomous LLM-based agents reason, plan, select and execute tools, and ultimately complete user tasks by analyzing the **full execution trace**—including reasoning steps, tool calls, intermediate decisions, and outcomes—rather than just single input–output pairs.\n", + "\n", + "These metrics are essential because agent failures often occur in ways traditional LLM metrics miss (e.g., choosing the right tool with wrong arguments, creating a good plan but not following it, or completing a task inefficiently).\n", + "\n", + "**DeepEval’s AI agent evaluation framework** breaks evaluation into three layers with corresponding metric categories:\n", + "\n", + "1. **Reasoning Layer** – Evaluates planning and strategy generation:\n", + "\n", + " * *PlanQualityMetric* – how logical, complete, and efficient the agent’s plan is\n", + " * *PlanAdherenceMetric* – whether the agent follows its own plan during execution \n", + "\n", + "2. **Action Layer** – Assesses tool usage and argument generation:\n", + "\n", + " * *ToolCorrectnessMetric* – whether the agent selects and calls the right tools\n", + " * *ArgumentCorrectnessMetric* – whether the agent generates correct tool arguments\n", + "\n", + "3. **Execution Layer** – Measures end-to-end performance:\n", + "\n", + " * *TaskCompletionMetric* – whether the agent successfully completes the intended task\n", + " * *StepEfficiencyMetric* – whether the agent avoids unnecessary or redundant steps\n", + "\n", + "Together, these metrics enable granular diagnosis of agent behavior, help pinpoint where failures occur (reasoning, action, or execution), and support both development benchmarking and production monitoring." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "#### **Reasoning Layer**\n", + "#### PlanQualityMetric\n", + "Let's measures how well the agent generates a plan before acting. A high score means the plan is logical, complete, and efficient." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agent_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.PlanQuality\",\n", + " input_column = \"input\",\n", + " actual_output_column = \"actual_output\",\n", + " tools_called_column = \"tools_called\",\n", + ")\n", + "agent_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### PlanAdherenceMetric\n", + "Let's checks whether the agent follows the plan it created. Deviations lower this score and indicate gaps between reasoning and execution." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agent_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.PlanAdherence\",\n", + " input_column = \"input\",\n", + " # actual_output_column = \"actual_output\",\n", + " agent_output_column = \"actual_output\",\n", + " tools_called_column = \"tools_called\",\n", + ")\n", + "agent_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "#### **Action Layer**\n", + "#### ToolCorrectnessMetric\n", + "Let's evaluates if the agent selects the appropriate tool for the task. Choosing the wrong tool reduces performance even if reasoning was correct." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agent_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.ToolCorrectness\",\n", + " input_column = \"input\",\n", + " actual_output_column = \"actual_output\",\n", + " agent_output_column = \"actual_output\",\n", + " tools_called_column = \"tools_called\",\n", + " expected_tools_column = \"expected_tools\",\n", + ")\n", + "agent_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### ArgumentCorrectnessMetric\n", + "Let's assesses whether the agent provides correct inputs or arguments to the selected tool. Incorrect arguments can lead to failed or unexpected results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agent_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.ArgumentCorrectness\",\n", + " input_column = \"input\",\n", + " actual_output_column = \"actual_output\",\n", + " tools_called_column = \"tools_called\",\n", + ")\n", + "agent_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "#### **Execution Layer**\n", + "#### TaskCompletionMetric\n", + "Let's measures whether the agent successfully completes the overall task. This is the ultimate indicator of success." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agent_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.TaskCompletion\",\n", + " input_column = \"input\",\n", + " actual_output_column = \"actual_output\",\n", + " agent_output_column = \"agent_output\",\n", + " tools_called_column = \"tools_called\",\n", + "\n", + ")\n", + "agent_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "#### StepEfficiencyMetric\n", + "Let's looks at how efficiently the agent executes the task, penalizing unnecessary or redundant steps. High efficiency indicates optimized behavior." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# simple_dataset.assign_scores(\n", + "# metrics = \"validmind.scorer.llm.deepeval.StepEfficiency\",\n", + "# input_column = \"input\",\n", + "# actual_output_column = \"actual_output\",\n", + "# agent_output_column = \"actual_output\",\n", + "# tools_called_column = \"tools_called\",\n", + "# )\n", + "# simple_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "## In summary\n", + "\n", + "This notebook demonstrated the comprehensive integration between DeepEval and ValidMind for LLM evaluation:\n", + "\n", + "**Key Achievements:**\n", + "- Successfully created and evaluated different types of LLM test cases (Q&A, RAG, Agents)\n", + "- Integrated DeepEval metrics with ValidMind's testing infrastructure\n", + "- Showed how to handle complex agent scenarios with tool usage\n", + "\n", + "**Integration Benefits:**\n", + "- **Comprehensive Coverage**: Evaluate LLMs across 30+ specialized metrics\n", + "- **Structured Documentation**: Leverage ValidMind's compliance and documentation features\n", + "- **Flexibility**: Support for custom metrics and domain-specific evaluation criteria\n", + "- **Production Ready**: Handle real-world LLM evaluation scenarios at scale\n", + "\n", + "The `LLMAgentDataset` class provides a seamless bridge between DeepEval's evaluation capabilities and ValidMind's testing infrastructure, enabling robust LLM evaluation within a structured, compliant framework.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" } + }, + "source": [ + "\n", + "\n", + "## Next steps\n", + "\n", + "**Explore Advanced Features:**\n", + "- **Continuous Evaluation**: Set up automated LLM evaluation pipelines\n", + "- **A/B Testing**: Compare different LLM models and configurations\n", + "- **Metrics Customization**: Create domain-specific evaluation criteria\n", + "- **Integration Patterns**: Embed evaluation into your LLM development workflow\n", + "\n", + "**Additional Resources:**\n", + "- [ValidMind Library Documentation](https://docs.validmind.ai/developer/validmind-library.html) - Complete API reference and tutorials\n", + "\n", + "**Try These Examples:**\n", + "- Implement custom business-specific evaluation metrics\n", + "- Create automated evaluation pipelines for model deployment\n", + "- Integrate with your existing ML infrastructure and workflows\n", + "- Explore multi-modal evaluation scenarios (text, code, images)\n", + "\n", + "Start building comprehensive LLM evaluation workflows that combine the power of DeepEval's specialized metrics with ValidMind's structured testing and documentation framework.\n" + ] + }, + { + "cell_type": "markdown", + "id": "22f216ac", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "validmind-1QuffXMV-py3.11", + "language": "python", + "name": "python3" }, - "nbformat": 4, - "nbformat_minor": 2 + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb b/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb index fa302b110..f9c068dc5 100644 --- a/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb +++ b/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb @@ -1,521 +1,524 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "# G-Eval Integration for DeepEval within ValidMind\n", - "\n", - "Let's learn how to integrate [DeepEval](https://github.com/confident-ai/deepeval) with the ValidMind Library to evaluate Large Language Models (LLMs) and AI agents. \n", - "Large Language Model (LLM) evaluation requires robust metrics to assess model outputs. G-Eval, a key feature of DeepEval, uses LLMs themselves to evaluate model responses across dimensions like factual accuracy, coherence, and relevance, etc. This notebook demonstrates how to leverage G-Eval metrics within ValidMind's testing infrastructure to create comprehensive, automated evaluations of LLM outputs.\n", - "\n", - "To integrate DeepEval with ValidMind, we'll:\n", - " 1. Set up both frameworks and install required dependencies\n", - " 2. Create a dataset with source texts and generated summaries\n", - " 3. Analyze the evaluation results using G-eval custom metrics\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## Contents \n", - "- [Introduction](#toc1_) \n", - " - [Before you begin](#toc2_1_) \n", - " - [Key concepts](#toc2_2_) \n", - "- [Setting up](#toc3_) \n", - " - [Install required packages](#toc3_1_) \n", - " - [Initialize ValidMind](#toc3_2_) \n", - "- [Custom Metrics with G-Eval](#toc4_) \n", - " - [Technical accuracy](#toc4_1_) \n", - " - [Clarity and Comprehensiveness](#toc4_2_) \n", - " - [Business Context Appropriateness](#toc4_3_) \n", - " - [Tool Usage Appropriateness](#toc4_4_) \n", - " - [Coherence Evaluation](#toc4_5_) \n", - "- [In summary](#toc5_) \n", - "- [Next steps](#toc6_) \n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## Introduction\n", - "**G-Eval** is a framework that uses large language models (LLMs) as evaluators—essentially treating an LLM as a “judge” to assess the quality of other LLM outputs. Instead of relying on traditional metrics like BLEU or ROUGE, G-Eval enables natural-language evaluation criteria (e.g., “rate how factual this summary is”). The framework guides the judge model through structured reasoning steps, producing more consistent, transparent, and interpretable scoring results. It is particularly effective for subjective or open-ended tasks such as summarization, dialogue generation, and content evaluation.\n", - "\n", - "Key advantages of G-Eval include:\n", - "\n", - "* **Structured reasoning:** Uses a step-by-step approach to improve reliability and reduce bias.\n", - "* **Custom evaluation criteria:** Supports diverse factors like accuracy, tone, safety, or style.\n", - "* **Enhanced consistency:** Provides more repeatable judgments than earlier LLM-as-a-judge methods.\n", - "* **Production scalability:** Integrates easily with CI/CD pipelines via tools like *DeepEval*.\n", - "* **Broader applicability:** Works across multiple domains and task types, from creative writing to factual QA." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "### Before you begin\n", - "\n", - "This notebook assumes you have basic familiarity with Python and Large Language Models. You'll need:\n", - "\n", - "- Python 3.8 or higher\n", - "- Access to OpenAI API (for DeepEval metrics evaluation)\n", - "- ValidMind account and model registration\n", - "\n", - "If you encounter errors due to missing modules, install them with `pip install` and re-run the notebook.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "### Key concepts\n", - "\n", - "**LLMTestCase**: A DeepEval object that represents a single test case with input, expected output, actual output, and optional context.\n", - "\n", - "**G-Eval**: Generative evaluation using LLMs to assess response quality based on custom criteria.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## Setting up\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "### Install required packages\n", - "\n", - "First, let's install the required packages and set up our environment.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -q validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "### Initialize ValidMind\n", - "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", - "\n", - "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", - "

\n", - "Register with ValidMind
\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load your model identifier credentials from an `.env` file\n", - "%load_ext dotenv\n", - "%dotenv .env\n", - "\n", - "# # Or replace with your code snippet\n", - "import validmind as vm\n", - "\n", - "vm.init(\n", - " api_host=\"...\",\n", - " api_key=\"...\",\n", - " api_secret=\"...\",\n", - " model=\"...\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Core imports\n", - "import warnings\n", - "from deepeval.test_case import LLMTestCase\n", - "from deepeval.metrics.g_eval.utils import Rubric\n", - "from deepeval.test_case import LLMTestCaseParams\n", - "from validmind.datasets.llm import LLMAgentDataset\n", - "import pandas as pd\n", - "\n", - "warnings.filterwarnings('ignore')\n", - "\n", - "pd.set_option('display.max_columns', None)\n", - "pd.set_option('display.max_colwidth', None)\n", - "pd.set_option('display.width', None)\n", - "pd.set_option('display.max_rows', None)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "## Create test cases\n", - "\n", - "Let's create test cases to demonstrate the G-Eval custom metrics functionality." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Create a test dataset for evaluating the custom metrics\n", - "test_cases = [\n", - " LLMTestCase(\n", - " input=\"What is machine learning?\",\n", - " actual_output=\"Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed. It uses statistical techniques to allow computers to find patterns in data.\",\n", - " context=[\"Machine learning is a branch of AI that focuses on building applications that learn from data and improve their accuracy over time without being programmed to do so.\"],\n", - " expected_output=\"Machine learning is a method of data analysis that automates analytical model building. It is a branch of artificial intelligence based on the idea that systems can learn from data, identify patterns and make decisions with minimal human intervention.\"\n", - " ), \n", - " LLMTestCase(\n", - " input=\"How do I implement a neural network?\",\n", - " actual_output=\"To implement a neural network, you need to: 1) Define the network architecture (layers, neurons), 2) Initialize weights and biases, 3) Implement forward propagation, 4) Calculate loss, 5) Perform backpropagation, and 6) Update weights using gradient descent.\",\n", - " context=[\"Neural networks are computing systems inspired by biological neural networks. They consist of layers of interconnected nodes that process and transmit signals.\"],\n", - " expected_output=\"Neural network implementation involves defining network architecture, initializing parameters, implementing forward and backward propagation, and using optimization algorithms for training.\"\n", - " )\n", - "]\n", - "\n", - "# Create Agent dataset\n", - "geval_dataset = LLMAgentDataset.from_test_cases(\n", - " test_cases=test_cases,\n", - " input_id=\"geval_dataset\"\n", - ")\n", - "geval_dataset._df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Scorers in ValidMind\n", - "\n", - "Scorers are evaluation metrics that analyze model outputs and store their results in the dataset. When using `assign_scores()`:\n", - "\n", - "- For Geval scorer adds new columns (score, reason and criteria) to the dataset with format: `GEval_{metric_name}_score`, `GEval_{metric_name}_reason` and `GEval_{metric_name}_criteria`\n", - "- The column contains the numeric score (typically 0-1) for each example\n", - "- Multiple scorers can be run on the same dataset, each adding their own columns\n", - "- Scores are persisted in the dataset for later analysis and visualization" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Custom Metrics with G-Eval\n", - "One of DeepEval's most powerful features is the ability to create custom evaluation metrics using G-Eval (Generative Evaluation). This enables domain-specific evaluation criteria tailored to your use case.\n", - "\n", - "\n", - "\n", - "\n", - "### Technical accuracy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "name=\"Technical Accuracy\"\n", - "criteria=\"\"\"Evaluate whether the response is technically accurate and uses appropriate \n", - "terminology for the domain. Consider if the explanations are scientifically sound \n", - "and if technical concepts are explained correctly.\"\"\"\n", - "threshold=0.8\n", - "geval_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.GEval\",\n", - " metric_name=name, \n", - " criteria = criteria,\n", - " threshold=threshold,\n", - " evaluation_params={\n", - " LLMTestCaseParams.INPUT: \"input\",\n", - " LLMTestCaseParams.ACTUAL_OUTPUT: \"actual_output\",\n", - " }\n", - ")\n", - "geval_dataset._df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Clarity and Comprehensiveness\n", - "This evaluation assesses the clarity and comprehensiveness of responses, focusing on how well-structured and understandable they are. The criteria examines whether responses are logically organized, address all aspects of questions thoroughly, and maintain an appropriate level of detail without being overly verbose.\n", - " \n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "name=\"Clarity and Comprehensiveness\"\n", - "criteria=\"\"\"Evaluate the clarity, structure, and comprehensiveness of the actual output \n", - "in relation to the expected output. The response should be clear, well-organized, and \n", - "comparable in coverage to the expected output, addressing all relevant aspects without \n", - "being overly verbose. Deduct points if important points or details present in the expected \n", - "output are missing or inaccurately conveyed in the actual output.\"\"\"\n", - "threshold=0.75\n", - "\n", - "geval_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.GEval\",\n", - " metric_name=name, \n", - " criteria = criteria,\n", - " threshold=threshold,\n", - " evaluation_params={\n", - " LLMTestCaseParams.INPUT: \"input\",\n", - " LLMTestCaseParams.ACTUAL_OUTPUT: \"actual_output\",\n", - " LLMTestCaseParams.EXPECTED_OUTPUT: \"expected_output\",\n", - " }\n", - ")\n", - "geval_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Business Context Appropriateness\n", - "\n", - "This evaluation assesses whether responses are appropriate for a business context, considering factors like professional tone, business relevance, and actionable insights. The criteria focuses on ensuring content would be valuable and applicable for business users.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "name=\"Business Context Appropriateness\"\n", - "criteria=\"\"\"Evaluate whether the response is appropriate for a business context. \n", - "Consider if the tone is professional, if the content is relevant to business needs, \n", - "and if it provides actionable information that would be valuable to a business user.\"\"\"\n", - "threshold=0.7\n", - "geval_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.GEval\",\n", - " metric_name=name, \n", - " criteria = criteria,\n", - " threshold=threshold,\n", - " evaluation_params={\n", - " LLMTestCaseParams.INPUT: \"input\",\n", - " LLMTestCaseParams.ACTUAL_OUTPUT: \"actual_output\",\n", - " LLMTestCaseParams.EXPECTED_OUTPUT: \"expected_output\",\n", - " }\n", - ")\n", - "geval_dataset._df.head()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Conciseness Evaluation\n", - "This evaluation assesses how well the responses flow and connect logically. It examines whether the content builds naturally from sentence to sentence to form a coherent narrative, rather than just being a collection of related but disconnected information. The evaluation considers factors like fluency, logical progression, and overall readability." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "criteria = \"\"\"\n", - " Evaluate the conciseness of the generation on a continuous scale from 0 to 1.\n", - " A generation can be considered concise (Score: 1) if it directly and succinctly\n", - " answers the question posed, focusing specifically on the information requested\n", - " without including unnecessary, irrelevant, or excessive details.\"\"\"\n", - "\n", - "evaluation_steps=[\n", - " \"Read the input and identify which pieces of information need to be conveyed.\"\n", - " \"Read the actual_output and check if it includes all the required information.\",\n", - " \"Check if the actual_output excludes irrelevant details or redundancies.\",\n", - " \"Check if the wording is as brief as possible while still being clear and complete.\",\n", - " \"Assign a score (e.g., 0-10) based on how well the actual_output meets the above.\"\n", - " ]\n", - "\n", - "rubric=[\n", - " Rubric(score_range=(0, 1), expected_outcome=\"Very poor Conciseness\"),\n", - " Rubric(score_range=(2, 3), expected_outcome=\"Poor Conciseness\"),\n", - " Rubric(score_range=(4, 5), expected_outcome=\"Fair Conciseness\"),\n", - " Rubric(score_range=(6, 7), expected_outcome=\"Good Conciseness\"),\n", - " Rubric(score_range=(8, 10), expected_outcome=\"Excellent Conciseness\"),\n", - " ]\n", - "\n", - "geval_dataset.assign_scores(\n", - " metrics = \"validmind.scorer.llm.deepeval.GEval\",\n", - " metric_name=\"Conciseness\", \n", - " criteria = criteria,\n", - " rubric=rubric,\n", - " evaluation_steps=evaluation_steps,\n", - " evaluation_params={\n", - " LLMTestCaseParams.INPUT: \"input\",\n", - " LLMTestCaseParams.ACTUAL_OUTPUT: \"actual_output\",\n", - " }\n", - ")\n", - "geval_dataset._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's plot all of these metrics together in a Boxplot Test" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vm.tests.run_test(\n", - " \"validmind.plots.BoxPlot\",\n", - " inputs={\"dataset\": geval_dataset},\n", - " params={\n", - " \"columns\": [\n", - " \"GEval_Technical_Accuracy_score\",\n", - " \"GEval_Clarity_and_Comprehensiveness_score\",\n", - " \"GEval_Business_Context_Appropriateness_score\",\n", - " \"GEval_Conciseness_score\"\n", - " ],\n", - " \"title\": \"Distribution of G-Eval Scores\",\n", - " \"ylabel\": \"Score\",\n", - " }\n", - ").log()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "\n", - "\n", - "## Next steps\n", - "\n", - "**Explore Advanced Features:**\n", - "- **Continuous Evaluation**: Set up automated LLM evaluation pipelines\n", - "- **Metrics Customization**: Create domain-specific evaluation criteria\n", - "- **Integration Patterns**: Embed evaluation into your LLM development workflow\n", - "\n", - "**Additional Resources:**\n", - "- [ValidMind Library Documentation](https://docs.validmind.ai/developer/validmind-library.html) - Complete API reference and tutorials\n", - "\n", - "**Try These Examples:**\n", - "- Implement custom business-specific evaluation metrics\n", - "- Create automated evaluation pipelines for model deployment\n", - "- Integrate with your existing ML infrastructure and workflows\n", - "- Explore multi-modal evaluation scenarios (text, code, images)\n", - "\n", - "Start building comprehensive LLM evaluation workflows that combine the power of DeepEval's specialized metrics with ValidMind's structured testing and documentation framework.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "***\n", - "\n", - "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", - "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" } - ], - "metadata": { - "kernelspec": { - "display_name": "ValidMind Library", - "language": "python", - "name": "validmind" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + }, + "source": [ + "# G-Eval Integration for DeepEval within ValidMind\n", + "\n", + "Let's learn how to integrate [DeepEval](https://github.com/confident-ai/deepeval) with the ValidMind Library to evaluate Large Language Models (LLMs) and AI agents. \n", + "Large Language Model (LLM) evaluation requires robust metrics to assess model outputs. G-Eval, a key feature of DeepEval, uses LLMs themselves to evaluate model responses across dimensions like factual accuracy, coherence, and relevance, etc. This notebook demonstrates how to leverage G-Eval metrics within ValidMind's testing infrastructure to create comprehensive, automated evaluations of LLM outputs.\n", + "\n", + "To integrate DeepEval with ValidMind, we'll:\n", + " 1. Set up both frameworks and install required dependencies\n", + " 2. Create a dataset with source texts and generated summaries\n", + " 3. Analyze the evaluation results using G-eval custom metrics\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "## Contents \n", + "- [Introduction](#toc1_) \n", + " - [Before you begin](#toc2_1_) \n", + " - [Key concepts](#toc2_2_) \n", + "- [Setting up](#toc3_) \n", + " - [Install required packages](#toc3_1_) \n", + " - [Initialize ValidMind](#toc3_2_) \n", + "- [Custom Metrics with G-Eval](#toc4_) \n", + " - [Technical accuracy](#toc4_1_) \n", + " - [Clarity and Comprehensiveness](#toc4_2_) \n", + " - [Business Context Appropriateness](#toc4_3_) \n", + " - [Tool Usage Appropriateness](#toc4_4_) \n", + " - [Coherence Evaluation](#toc4_5_) \n", + "- [In summary](#toc5_) \n", + "- [Next steps](#toc6_) \n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "## Introduction\n", + "**G-Eval** is a framework that uses large language models (LLMs) as evaluators—essentially treating an LLM as a “judge” to assess the quality of other LLM outputs. Instead of relying on traditional metrics like BLEU or ROUGE, G-Eval enables natural-language evaluation criteria (e.g., “rate how factual this summary is”). The framework guides the judge model through structured reasoning steps, producing more consistent, transparent, and interpretable scoring results. It is particularly effective for subjective or open-ended tasks such as summarization, dialogue generation, and content evaluation.\n", + "\n", + "Key advantages of G-Eval include:\n", + "\n", + "* **Structured reasoning:** Uses a step-by-step approach to improve reliability and reduce bias.\n", + "* **Custom evaluation criteria:** Supports diverse factors like accuracy, tone, safety, or style.\n", + "* **Enhanced consistency:** Provides more repeatable judgments than earlier LLM-as-a-judge methods.\n", + "* **Production scalability:** Integrates easily with CI/CD pipelines via tools like *DeepEval*.\n", + "* **Broader applicability:** Works across multiple domains and task types, from creative writing to factual QA." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "### Before you begin\n", + "\n", + "This notebook assumes you have basic familiarity with Python and Large Language Models. You'll need:\n", + "\n", + "- Python 3.8 or higher\n", + "- Access to OpenAI API (for DeepEval metrics evaluation)\n", + "- ValidMind account and model registration\n", + "\n", + "If you encounter errors due to missing modules, install them with `pip install` and re-run the notebook.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "### Key concepts\n", + "\n", + "**LLMTestCase**: A DeepEval object that represents a single test case with input, expected output, actual output, and optional context.\n", + "\n", + "**G-Eval**: Generative evaluation using LLMs to assess response quality based on custom criteria.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "## Setting up\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "### Install required packages\n", + "\n", + "First, let's install the required packages and set up our environment.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q validmind" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "\n", + "\n", + "### Initialize ValidMind\n", + "\n", + "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "\n", + "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", + "

\n", + "Register with ValidMind
\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load your model identifier credentials from an `.env` file\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# # Or replace with your code snippet\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " api_host=\"...\",\n", + " api_key=\"...\",\n", + " api_secret=\"...\",\n", + " model=\"...\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Core imports\n", + "import warnings\n", + "from deepeval.test_case import LLMTestCase\n", + "from deepeval.metrics.g_eval.utils import Rubric\n", + "from deepeval.test_case import LLMTestCaseParams\n", + "from validmind.datasets.llm import LLMAgentDataset\n", + "import pandas as pd\n", + "\n", + "warnings.filterwarnings('ignore')\n", + "\n", + "pd.set_option('display.max_columns', None)\n", + "pd.set_option('display.max_colwidth', None)\n", + "pd.set_option('display.width', None)\n", + "pd.set_option('display.max_rows', None)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Create test cases\n", + "\n", + "Let's create test cases to demonstrate the G-Eval custom metrics functionality." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create a test dataset for evaluating the custom metrics\n", + "test_cases = [\n", + " LLMTestCase(\n", + " input=\"What is machine learning?\",\n", + " actual_output=\"Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed. It uses statistical techniques to allow computers to find patterns in data.\",\n", + " context=[\"Machine learning is a branch of AI that focuses on building applications that learn from data and improve their accuracy over time without being programmed to do so.\"],\n", + " expected_output=\"Machine learning is a method of data analysis that automates analytical model building. It is a branch of artificial intelligence based on the idea that systems can learn from data, identify patterns and make decisions with minimal human intervention.\"\n", + " ), \n", + " LLMTestCase(\n", + " input=\"How do I implement a neural network?\",\n", + " actual_output=\"To implement a neural network, you need to: 1) Define the network architecture (layers, neurons), 2) Initialize weights and biases, 3) Implement forward propagation, 4) Calculate loss, 5) Perform backpropagation, and 6) Update weights using gradient descent.\",\n", + " context=[\"Neural networks are computing systems inspired by biological neural networks. They consist of layers of interconnected nodes that process and transmit signals.\"],\n", + " expected_output=\"Neural network implementation involves defining network architecture, initializing parameters, implementing forward and backward propagation, and using optimization algorithms for training.\"\n", + " )\n", + "]\n", + "\n", + "# Create Agent dataset\n", + "geval_dataset = LLMAgentDataset.from_test_cases(\n", + " test_cases=test_cases,\n", + " input_id=\"geval_dataset\"\n", + ")\n", + "geval_dataset._df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scorers in ValidMind\n", + "\n", + "Scorers are evaluation metrics that analyze model outputs and store their results in the dataset. When using `assign_scores()`:\n", + "\n", + "- For Geval scorer adds new columns (score, reason and criteria) to the dataset with format: `GEval_{metric_name}_score`, `GEval_{metric_name}_reason` and `GEval_{metric_name}_criteria`\n", + "- The column contains the numeric score (typically 0-1) for each example\n", + "- Multiple scorers can be run on the same dataset, each adding their own columns\n", + "- Scores are persisted in the dataset for later analysis and visualization" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Custom Metrics with G-Eval\n", + "One of DeepEval's most powerful features is the ability to create custom evaluation metrics using G-Eval (Generative Evaluation). This enables domain-specific evaluation criteria tailored to your use case.\n", + "\n", + "\n", + "\n", + "\n", + "### Technical accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "name=\"Technical Accuracy\"\n", + "criteria=\"\"\"Evaluate whether the response is technically accurate and uses appropriate \n", + "terminology for the domain. Consider if the explanations are scientifically sound \n", + "and if technical concepts are explained correctly.\"\"\"\n", + "threshold=0.8\n", + "geval_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.GEval\",\n", + " metric_name=name, \n", + " criteria = criteria,\n", + " threshold=threshold,\n", + " evaluation_params={\n", + " LLMTestCaseParams.INPUT: \"input\",\n", + " LLMTestCaseParams.ACTUAL_OUTPUT: \"actual_output\",\n", + " }\n", + ")\n", + "geval_dataset._df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Clarity and Comprehensiveness\n", + "This evaluation assesses the clarity and comprehensiveness of responses, focusing on how well-structured and understandable they are. The criteria examines whether responses are logically organized, address all aspects of questions thoroughly, and maintain an appropriate level of detail without being overly verbose.\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "name=\"Clarity and Comprehensiveness\"\n", + "criteria=\"\"\"Evaluate the clarity, structure, and comprehensiveness of the actual output \n", + "in relation to the expected output. The response should be clear, well-organized, and \n", + "comparable in coverage to the expected output, addressing all relevant aspects without \n", + "being overly verbose. Deduct points if important points or details present in the expected \n", + "output are missing or inaccurately conveyed in the actual output.\"\"\"\n", + "threshold=0.75\n", + "\n", + "geval_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.GEval\",\n", + " metric_name=name, \n", + " criteria = criteria,\n", + " threshold=threshold,\n", + " evaluation_params={\n", + " LLMTestCaseParams.INPUT: \"input\",\n", + " LLMTestCaseParams.ACTUAL_OUTPUT: \"actual_output\",\n", + " LLMTestCaseParams.EXPECTED_OUTPUT: \"expected_output\",\n", + " }\n", + ")\n", + "geval_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Business Context Appropriateness\n", + "\n", + "This evaluation assesses whether responses are appropriate for a business context, considering factors like professional tone, business relevance, and actionable insights. The criteria focuses on ensuring content would be valuable and applicable for business users.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "name=\"Business Context Appropriateness\"\n", + "criteria=\"\"\"Evaluate whether the response is appropriate for a business context. \n", + "Consider if the tone is professional, if the content is relevant to business needs, \n", + "and if it provides actionable information that would be valuable to a business user.\"\"\"\n", + "threshold=0.7\n", + "geval_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.GEval\",\n", + " metric_name=name, \n", + " criteria = criteria,\n", + " threshold=threshold,\n", + " evaluation_params={\n", + " LLMTestCaseParams.INPUT: \"input\",\n", + " LLMTestCaseParams.ACTUAL_OUTPUT: \"actual_output\",\n", + " LLMTestCaseParams.EXPECTED_OUTPUT: \"expected_output\",\n", + " }\n", + ")\n", + "geval_dataset._df.head()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Conciseness Evaluation\n", + "This evaluation assesses how well the responses flow and connect logically. It examines whether the content builds naturally from sentence to sentence to form a coherent narrative, rather than just being a collection of related but disconnected information. The evaluation considers factors like fluency, logical progression, and overall readability." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "criteria = \"\"\"\n", + " Evaluate the conciseness of the generation on a continuous scale from 0 to 1.\n", + " A generation can be considered concise (Score: 1) if it directly and succinctly\n", + " answers the question posed, focusing specifically on the information requested\n", + " without including unnecessary, irrelevant, or excessive details.\"\"\"\n", + "\n", + "evaluation_steps=[\n", + " \"Read the input and identify which pieces of information need to be conveyed.\"\n", + " \"Read the actual_output and check if it includes all the required information.\",\n", + " \"Check if the actual_output excludes irrelevant details or redundancies.\",\n", + " \"Check if the wording is as brief as possible while still being clear and complete.\",\n", + " \"Assign a score (e.g., 0-10) based on how well the actual_output meets the above.\"\n", + " ]\n", + "\n", + "rubric=[\n", + " Rubric(score_range=(0, 1), expected_outcome=\"Very poor Conciseness\"),\n", + " Rubric(score_range=(2, 3), expected_outcome=\"Poor Conciseness\"),\n", + " Rubric(score_range=(4, 5), expected_outcome=\"Fair Conciseness\"),\n", + " Rubric(score_range=(6, 7), expected_outcome=\"Good Conciseness\"),\n", + " Rubric(score_range=(8, 10), expected_outcome=\"Excellent Conciseness\"),\n", + " ]\n", + "\n", + "geval_dataset.assign_scores(\n", + " metrics = \"validmind.scorer.llm.deepeval.GEval\",\n", + " metric_name=\"Conciseness\", \n", + " criteria = criteria,\n", + " rubric=rubric,\n", + " evaluation_steps=evaluation_steps,\n", + " evaluation_params={\n", + " LLMTestCaseParams.INPUT: \"input\",\n", + " LLMTestCaseParams.ACTUAL_OUTPUT: \"actual_output\",\n", + " }\n", + ")\n", + "geval_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's plot all of these metrics together in a Boxplot Test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.plots.BoxPlot\",\n", + " inputs={\"dataset\": geval_dataset},\n", + " params={\n", + " \"columns\": [\n", + " \"GEval_Technical_Accuracy_score\",\n", + " \"GEval_Clarity_and_Comprehensiveness_score\",\n", + " \"GEval_Business_Context_Appropriateness_score\",\n", + " \"GEval_Conciseness_score\"\n", + " ],\n", + " \"title\": \"Distribution of G-Eval Scores\",\n", + " \"ylabel\": \"Score\",\n", + " }\n", + ").log()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" } + }, + "source": [ + "\n", + "\n", + "## Next steps\n", + "\n", + "**Explore Advanced Features:**\n", + "- **Continuous Evaluation**: Set up automated LLM evaluation pipelines\n", + "- **Metrics Customization**: Create domain-specific evaluation criteria\n", + "- **Integration Patterns**: Embed evaluation into your LLM development workflow\n", + "\n", + "**Additional Resources:**\n", + "- [ValidMind Library Documentation](https://docs.validmind.ai/developer/validmind-library.html) - Complete API reference and tutorials\n", + "\n", + "**Try These Examples:**\n", + "- Implement custom business-specific evaluation metrics\n", + "- Create automated evaluation pipelines for model deployment\n", + "- Integrate with your existing ML infrastructure and workflows\n", + "- Explore multi-modal evaluation scenarios (text, code, images)\n", + "\n", + "Start building comprehensive LLM evaluation workflows that combine the power of DeepEval's specialized metrics with ValidMind's structured testing and documentation framework.\n" + ] + }, + { + "cell_type": "markdown", + "id": "bb785b65", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ValidMind Library", + "language": "python", + "name": "validmind" }, - "nbformat": 4, - "nbformat_minor": 2 + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/notebooks/code_sharing/plots_and_stats_demo.ipynb b/notebooks/code_sharing/plots_and_stats_demo.ipynb index 7e85a2d4e..02b2c29a3 100644 --- a/notebooks/code_sharing/plots_and_stats_demo.ipynb +++ b/notebooks/code_sharing/plots_and_stats_demo.ipynb @@ -1,721 +1,724 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "# Comprehensive Guide: ValidMind Plots and Statistics Tests\n", - "\n", - "This notebook demonstrates all the available tests from the `validmind.plots` and `validmind.stats` modules. Theseized tests provide powerful visualization and statistical analysis capabilities for any dataset.\n", - "\n", - "## What You'll Learn\n", - "\n", - "In this notebook, we'll explore:\n", - "\n", - "1. **Plotting Tests**: Visual analysis tools for data exploration\n", - " - CorrelationHeatmap\n", - " - HistogramPlot\n", - " - BoxPlot\n", - " - ViolinPlot\n", - "\n", - "2. **Statistical Tests**: Comprehensive statistical analysis tools\n", - " - DescriptiveStats\n", - " - CorrelationAnalysis\n", - " - NormalityTests\n", - " - OutlierDetection\n", - "\n", - "Each test is highly configurable and can be adapted to different datasets and use cases.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Comprehensive Guide: ValidMind Plots and Statistics Tests\n", - "\n", - "This notebook demonstrates all the available tests from the `validmind.plots` and `validmind.stats` modules. These generalized tests provide powerful visualization and statistical analysis capabilities for any dataset.\n", - "\n", - "## What You'll Learn\n", - "\n", - "In this notebook, we'll explore:\n", - "\n", - "1. **Plotting Tests**: Visual analysis tools for data exploration\n", - " - CorrelationHeatmap\n", - " - HistogramPlot\n", - " - BoxPlot\n", - " - ViolinPlot\n", - "\n", - "2. **Statistical Tests**: Comprehensive statistical analysis tools\n", - " - DescriptiveStats\n", - " - CorrelationAnalysis\n", - " - NormalityTests\n", - " - OutlierDetection\n", - "\n", - "Each test is highly configurable and can be adapted to different datasets and use cases.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## About ValidMind\n", - "\n", - "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models. You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## Setting up\n", - "\n", - "### Install the ValidMind Library\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -q validmind\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "### Initialize the ValidMind Library\n", - "\n", - "For this demonstration, we'll initialize ValidMind in demo mode.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load your model identifier credentials from an `.env` file\n", - "\n", - "%load_ext dotenv\n", - "%dotenv .env\n", - "\n", - "# Or replace with your code snippet\n", - "\n", - "import validmind as vm\n", - "\n", - "# Note: You need valid API credentials for this to work\n", - "# If you don't have credentials, use the standalone script: test_outlier_detection_standalone.py\n", - "\n", - "vm.init(\n", - " api_host=\"...\",\n", - " api_key=\"...\",\n", - " api_secret=\"...\",\n", - " model=\"...\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## Import and Prepare Sample Dataset\n", - "\n", - "We'll use the Bank Customer Churn dataset as our example data for demonstrating all the tests.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from validmind.datasets.classification import customer_churn\n", - "\n", - "print(\n", - " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{customer_churn.target_column}' \\n\\t• Class labels: {customer_churn.class_labels}\"\n", - ")\n", - "\n", - "# Load and preprocess the data\n", - "raw_df = customer_churn.load_data()\n", - "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)\n", - "\n", - "print(f\"\\nDataset shapes:\")\n", - "print(f\"• Training: {train_df.shape}\")\n", - "print(f\"• Validation: {validation_df.shape}\")\n", - "print(f\"• Test: {test_df.shape}\")\n", - "\n", - "raw_df.head()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "### Initialize ValidMind Datasets\n", - "\n", - "Initialize ValidMind dataset objects for our analysis:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize datasets for ValidMind\n", - "vm_raw_dataset = vm.init_dataset(\n", - " dataset=raw_df,\n", - " input_id=\"raw_dataset\",\n", - " target_column=customer_churn.target_column,\n", - " class_labels=customer_churn.class_labels,\n", - ")\n", - "\n", - "vm_train_ds = vm.init_dataset(\n", - " dataset=train_df,\n", - " input_id=\"train_dataset\",\n", - " target_column=customer_churn.target_column,\n", - ")\n", - "\n", - "print(\"✅ ValidMind datasets initialized successfully!\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "### Explore Dataset Structure\n", - "\n", - "Let's examine our dataset to understand what columns are available for analysis:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"📊 Dataset Information:\")\n", - "print(f\"\\nAll columns ({len(vm_train_ds.df.columns)}):\")\n", - "print(list(vm_train_ds.df.columns))\n", - "\n", - "print(f\"\\nNumerical columns ({len(vm_train_ds.feature_columns_numeric)}):\")\n", - "print(vm_train_ds.feature_columns_numeric)\n", - "\n", - "print(f\"\\nCategorical columns ({len(vm_train_ds.feature_columns_categorical) if hasattr(vm_train_ds, 'feature_columns_categorical') else 0}):\")\n", - "print(vm_train_ds.feature_columns_categorical if hasattr(vm_train_ds, 'feature_columns_categorical') else \"None detected\")\n", - "\n", - "print(f\"\\nTarget column: {vm_train_ds.target_column}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "# Part 1: Plotting Tests\n", - "\n", - "The ValidMind plotting tests provide powerful visualization capabilities for data exploration and analysis. All plots are interactive and built with Plotly.\n", - "\n", - "## 1. Correlation Heatmap\n", - "\n", - "Visualizes correlations between numerical features using a heatmap. Useful for identifying multicollinearity and feature relationships.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Basic correlation heatmap\n", - "vm.tests.run_test(\n", - " \"validmind.plots.CorrelationHeatmap\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\n", - " \"method\": \"pearson\",\n", - " \"show_values\": True,\n", - " \"colorscale\": \"RdBu\",\n", - " \"mask_upper\": False,\n", - " \"threshold\": None,\n", - " \"width\": 800,\n", - " \"height\": 600,\n", - " \"title\": \"Feature Correlation Heatmap\"\n", - " }\n", - ")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Advanced correlation heatmap with custom settings\n", - "vm.tests.run_test(\n", - " \"validmind.plots.CorrelationHeatmap\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\n", - " \"method\": \"spearman\", # Different correlation method\n", - " \"show_values\": True,\n", - " \"colorscale\": \"Viridis\",\n", - " \"mask_upper\": True, # Mask upper triangle\n", - " \"width\": 900,\n", - " \"height\": 700,\n", - " \"title\": \"Spearman Correlation (|r| > 0.3)\",\n", - " \"columns\": [\"CreditScore\", \"Age\", \"Balance\", \"EstimatedSalary\"] # Specific columns\n", - " }\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## 2. Histogram Plot\n", - "\n", - "Creates histogram distributions for numerical features with optional KDE overlay. Essential for understanding data distributions.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Basic histogram with KDE\n", - "vm.tests.run_test(\n", - " \"validmind.plots.HistogramPlot\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\n", - " \"columns\": [\"CreditScore\", \"Balance\", \"EstimatedSalary\", \"Age\"],\n", - " \"bins\": 30,\n", - " \"color\": \"steelblue\",\n", - " \"opacity\": 0.7,\n", - " \"show_kde\": True,\n", - " \"normalize\": False,\n", - " \"log_scale\": False,\n", - " \"width\": 1200,\n", - " \"height\": 800,\n", - " \"n_cols\": 2,\n", - " \"vertical_spacing\": 0.15,\n", - " \"horizontal_spacing\": 0.15,\n", - " \"title_prefix\": \"Distribution of\"\n", - " }\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## 3. Box Plot\n", - "\n", - "Displays box plots for numerical features, optionally grouped by a categorical variable. Excellent for outlier detection and comparing distributions.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Box plots grouped by target variable\n", - "vm.tests.run_test(\n", - " \"validmind.plots.BoxPlot\", \n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\n", - " \"columns\": [\"CreditScore\", \"Balance\", \"Age\"],\n", - " \"group_by\": \"Exited\", # Group by churn status\n", - " \"colors\": [\"lightblue\", \"salmon\"],\n", - " \"show_outliers\": True,\n", - " \"width\": 1200,\n", - " \"height\": 600\n", - " }\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## 4. Violin Plot\n", - "\n", - "Creates violin plots that combine box plots with kernel density estimation. Shows both summary statistics and distribution shape.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Violin plots grouped by target variable\n", - "vm.tests.run_test(\n", - " \"validmind.plots.ViolinPlot\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\n", - " \"columns\": [\"Age\", \"Balance\"], # Focus on key variables\n", - " \"group_by\": \"Exited\",\n", - " \"width\": 800,\n", - " \"height\": 600\n", - " }\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "# Part 2: Statistical Tests\n", - "\n", - "The ValidMind statistical tests provide comprehensive statistical analysis capabilities for understanding data characteristics and quality.\n", - "\n", - "## 1. Descriptive Statistics\n", - "\n", - "Provides comprehensive descriptive statistics including basic statistics, distribution measures, confidence intervals, and normality tests.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Advanced descriptive statistics with all measures\n", - "vm.tests.run_test(\n", - " \"validmind.stats.DescriptiveStats\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\n", - " \"include_advanced\": True, # Include skewness, kurtosis, normality tests, etc.\n", - " \"confidence_level\": 0.99, # 99% confidence intervals\n", - " \"columns\": [\"CreditScore\", \"Balance\", \"EstimatedSalary\", \"Age\"] # Specific columns\n", - " }\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## 2. Correlation Analysis\n", - "\n", - "Performs detailed correlation analysis with statistical significance testing and identifies highly correlated feature pairs.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Correlation analysis with significance testing\n", - "result = vm.tests.run_test(\n", - " \"validmind.stats.CorrelationAnalysis\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\n", - " \"method\": \"pearson\", # or \"spearman\", \"kendall\"\n", - " \"significance_level\": 0.05,\n", - " \"min_correlation\": 0.1 # Minimum correlation threshold\n", - " }\n", - ")\n", - "result.log()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## 3. Normality Tests\n", - "\n", - "Performs various normality tests to assess whether features follow a normal distribution.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Comprehensive normality testing\n", - "vm.tests.run_test(\n", - " \"validmind.stats.NormalityTests\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\n", - " \"tests\": [\"shapiro\", \"anderson\", \"kstest\"], # Multiple tests\n", - " \"alpha\": 0.05,\n", - " \"columns\": [\"CreditScore\", \"Balance\", \"Age\"] # Focus on key features\n", - " }\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "## 4. Outlier Detection\n", - "\n", - "Identifies outliers using various statistical methods including IQR, Z-score, and Isolation Forest.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Comprehensive outlier detection with multiple methods\n", - "vm.tests.run_test(\n", - " \"validmind.stats.OutlierDetection\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\n", - " \"methods\": [\"iqr\", \"zscore\", \"isolation_forest\"],\n", - " \"iqr_threshold\": 1.5,\n", - " \"zscore_threshold\": 3.0,\n", - " \"contamination\": 0.1,\n", - " \"columns\": [\"CreditScore\", \"Balance\", \"EstimatedSalary\"]\n", - " }\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "# Part 3: Complete EDA Workflow Example\n", - "\n", - "Let's demonstrate a complete exploratory data analysis workflow using all the tests together:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example: Complete EDA workflow using all tests\n", - "print(\"🔍 Complete Exploratory Data Analysis Workflow\")\n", - "print(\"=\" * 50)\n", - "\n", - "# 1. Start with descriptive statistics\n", - "print(\"\\n1. Descriptive Statistics:\")\n", - "desc_stats = vm.tests.run_test(\n", - " \"validmind.stats.DescriptiveStats\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\"include_advanced\": True}\n", - ")\n", - "\n", - "print(\"\\n2. Distribution Analysis:\")\n", - "# 2. Visualize distributions\n", - "hist_plot = vm.tests.run_test(\n", - " \"validmind.plots.HistogramPlot\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\"show_kde\": True, \"n_cols\": 3}\n", - ")\n", - "\n", - "print(\"\\n3. Correlation Analysis:\")\n", - "# 3. Check correlations\n", - "corr_heatmap = vm.tests.run_test(\n", - " \"validmind.plots.CorrelationHeatmap\",\n", - " inputs={\"dataset\": vm_train_ds}\n", - ")\n", - "\n", - "print(\"\\n4. Outlier Detection:\")\n", - "# 4. Detect outliers\n", - "outliers = vm.tests.run_test(\n", - " \"validmind.stats.OutlierDetection\",\n", - " inputs={\"dataset\": vm_train_ds},\n", - " params={\"methods\": [\"iqr\", \"zscore\"]}\n", - ")\n", - "\n", - "print(\"\\n✅ EDA Complete! Check the visualizations and tables above for insights.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Comprehensive Guide: ValidMind Plots and Statistics Tests\n", - "\n", - "This notebook demonstrates all the available tests from the `validmind.plots` and `validmind.stats` modules. These generalized tests provide powerful visualization and statistical analysis capabilities for any dataset.\n", - "\n", - "## What You'll Learn\n", - "\n", - "In this notebook, we'll explore:\n", - "\n", - "1. **Plotting Tests**: Visual analysis tools for data exploration\n", - " - GeneralCorrelationHeatmap\n", - " - GeneralHistogramPlot\n", - " - GeneralBoxPlot\n", - " - GeneralViolinPlot\n", - "\n", - "2. **Statistical Tests**: Comprehensive statistical analysis tools\n", - " - GeneralDescriptiveStats\n", - " - GeneralCorrelationAnalysis\n", - " - GeneralNormalityTests\n", - " - GeneralOutlierDetection\n", - "\n", - "Each test is highly configurable and can be adapted to different datasets and use cases.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "# Conclusion\n", - "\n", - "This notebook demonstrated all the plotting and statistical tests available in ValidMind:\n", - "\n", - "## Plotting Tests Covered:\n", - "✅ **GeneralCorrelationHeatmap** - Interactive correlation matrices \n", - "✅ **GeneralHistogramPlot** - Distribution analysis with KDE \n", - "✅ **GeneralBoxPlot** - Outlier detection and group comparisons \n", - "✅ **GeneralViolinPlot** - Distribution shape analysis \n", - "\n", - "## Statistical Tests Covered:\n", - "✅ **GeneralDescriptiveStats** - Comprehensive statistical profiling \n", - "✅ **GeneralCorrelationAnalysis** - Formal correlation testing \n", - "✅ **GeneralNormalityTests** - Distribution assumption checking \n", - "✅ **GeneralOutlierDetection** - Multi-method outlier identification \n", - "\n", - "## Key Benefits:\n", - "- **Highly Customizable**: All tests offer extensive parameter options\n", - "- **Interactive Visualizations**: Plotly-based plots with zoom, pan, hover\n", - "- **Statistical Rigor**: Formal testing with significance levels\n", - "- **Flexible Input**: Works with any ValidMind dataset\n", - "- **Comprehensive Output**: Tables, plots, and statistical summaries\n", - "\n", - "## Best Practices:\n", - "\n", - "### When to Use Each Test:\n", - "\n", - "**Plotting Tests:**\n", - "- **GeneralCorrelationHeatmap**: Initial data exploration, multicollinearity detection\n", - "- **GeneralHistogramPlot**: Understanding feature distributions, identifying skewness\n", - "- **GeneralBoxPlot**: Outlier detection, comparing groups\n", - "- **GeneralViolinPlot**: Detailed distribution analysis, especially for grouped data\n", - "\n", - "**Statistical Tests:**\n", - "- **GeneralDescriptiveStats**: Comprehensive data profiling, baseline statistics\n", - "- **GeneralCorrelationAnalysis**: Formal correlation testing with significance\n", - "- **GeneralNormalityTests**: Model assumption checking\n", - "- **GeneralOutlierDetection**: Data quality assessment, preprocessing decisions\n", - "\n", - "## Next Steps:\n", - "- Integrate these tests into your model documentation templates\n", - "- Customize parameters based on your specific data characteristics\n", - "- Use results to inform preprocessing and modeling decisions\n", - "- Combine with ValidMind's model validation tests for complete analysis\n", - "\n", - "These tests provide a solid foundation for exploratory data analysis, data quality assessment, and statistical validation in any data science workflow.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "***\n", - "\n", - "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", - "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" } - ], - "metadata": { - "kernelspec": { - "display_name": "ValidMind Library", - "language": "python", - "name": "validmind" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + }, + "source": [ + "# Comprehensive Guide: ValidMind Plots and Statistics Tests\n", + "\n", + "This notebook demonstrates all the available tests from the `validmind.plots` and `validmind.stats` modules. Theseized tests provide powerful visualization and statistical analysis capabilities for any dataset.\n", + "\n", + "## What You'll Learn\n", + "\n", + "In this notebook, we'll explore:\n", + "\n", + "1. **Plotting Tests**: Visual analysis tools for data exploration\n", + " - CorrelationHeatmap\n", + " - HistogramPlot\n", + " - BoxPlot\n", + " - ViolinPlot\n", + "\n", + "2. **Statistical Tests**: Comprehensive statistical analysis tools\n", + " - DescriptiveStats\n", + " - CorrelationAnalysis\n", + " - NormalityTests\n", + " - OutlierDetection\n", + "\n", + "Each test is highly configurable and can be adapted to different datasets and use cases.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Comprehensive Guide: ValidMind Plots and Statistics Tests\n", + "\n", + "This notebook demonstrates all the available tests from the `validmind.plots` and `validmind.stats` modules. These generalized tests provide powerful visualization and statistical analysis capabilities for any dataset.\n", + "\n", + "## What You'll Learn\n", + "\n", + "In this notebook, we'll explore:\n", + "\n", + "1. **Plotting Tests**: Visual analysis tools for data exploration\n", + " - CorrelationHeatmap\n", + " - HistogramPlot\n", + " - BoxPlot\n", + " - ViolinPlot\n", + "\n", + "2. **Statistical Tests**: Comprehensive statistical analysis tools\n", + " - DescriptiveStats\n", + " - CorrelationAnalysis\n", + " - NormalityTests\n", + " - OutlierDetection\n", + "\n", + "Each test is highly configurable and can be adapted to different datasets and use cases.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "## About ValidMind\n", + "\n", + "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models. You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "## Setting up\n", + "\n", + "### Install the ValidMind Library\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q validmind\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "### Initialize the ValidMind Library\n", + "\n", + "For this demonstration, we'll initialize ValidMind in demo mode.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load your model identifier credentials from an `.env` file\n", + "\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "\n", + "import validmind as vm\n", + "\n", + "# Note: You need valid API credentials for this to work\n", + "# If you don't have credentials, use the standalone script: test_outlier_detection_standalone.py\n", + "\n", + "vm.init(\n", + " api_host=\"...\",\n", + " api_key=\"...\",\n", + " api_secret=\"...\",\n", + " model=\"...\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "## Import and Prepare Sample Dataset\n", + "\n", + "We'll use the Bank Customer Churn dataset as our example data for demonstrating all the tests.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.datasets.classification import customer_churn\n", + "\n", + "print(\n", + " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{customer_churn.target_column}' \\n\\t• Class labels: {customer_churn.class_labels}\"\n", + ")\n", + "\n", + "# Load and preprocess the data\n", + "raw_df = customer_churn.load_data()\n", + "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)\n", + "\n", + "print(f\"\\nDataset shapes:\")\n", + "print(f\"• Training: {train_df.shape}\")\n", + "print(f\"• Validation: {validation_df.shape}\")\n", + "print(f\"• Test: {test_df.shape}\")\n", + "\n", + "raw_df.head()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "### Initialize ValidMind Datasets\n", + "\n", + "Initialize ValidMind dataset objects for our analysis:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize datasets for ValidMind\n", + "vm_raw_dataset = vm.init_dataset(\n", + " dataset=raw_df,\n", + " input_id=\"raw_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " class_labels=customer_churn.class_labels,\n", + ")\n", + "\n", + "vm_train_ds = vm.init_dataset(\n", + " dataset=train_df,\n", + " input_id=\"train_dataset\",\n", + " target_column=customer_churn.target_column,\n", + ")\n", + "\n", + "print(\"✅ ValidMind datasets initialized successfully!\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "### Explore Dataset Structure\n", + "\n", + "Let's examine our dataset to understand what columns are available for analysis:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"📊 Dataset Information:\")\n", + "print(f\"\\nAll columns ({len(vm_train_ds.df.columns)}):\")\n", + "print(list(vm_train_ds.df.columns))\n", + "\n", + "print(f\"\\nNumerical columns ({len(vm_train_ds.feature_columns_numeric)}):\")\n", + "print(vm_train_ds.feature_columns_numeric)\n", + "\n", + "print(f\"\\nCategorical columns ({len(vm_train_ds.feature_columns_categorical) if hasattr(vm_train_ds, 'feature_columns_categorical') else 0}):\")\n", + "print(vm_train_ds.feature_columns_categorical if hasattr(vm_train_ds, 'feature_columns_categorical') else \"None detected\")\n", + "\n", + "print(f\"\\nTarget column: {vm_train_ds.target_column}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "# Part 1: Plotting Tests\n", + "\n", + "The ValidMind plotting tests provide powerful visualization capabilities for data exploration and analysis. All plots are interactive and built with Plotly.\n", + "\n", + "## 1. Correlation Heatmap\n", + "\n", + "Visualizes correlations between numerical features using a heatmap. Useful for identifying multicollinearity and feature relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Basic correlation heatmap\n", + "vm.tests.run_test(\n", + " \"validmind.plots.CorrelationHeatmap\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\n", + " \"method\": \"pearson\",\n", + " \"show_values\": True,\n", + " \"colorscale\": \"RdBu\",\n", + " \"mask_upper\": False,\n", + " \"threshold\": None,\n", + " \"width\": 800,\n", + " \"height\": 600,\n", + " \"title\": \"Feature Correlation Heatmap\"\n", + " }\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Advanced correlation heatmap with custom settings\n", + "vm.tests.run_test(\n", + " \"validmind.plots.CorrelationHeatmap\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\n", + " \"method\": \"spearman\", # Different correlation method\n", + " \"show_values\": True,\n", + " \"colorscale\": \"Viridis\",\n", + " \"mask_upper\": True, # Mask upper triangle\n", + " \"width\": 900,\n", + " \"height\": 700,\n", + " \"title\": \"Spearman Correlation (|r| > 0.3)\",\n", + " \"columns\": [\"CreditScore\", \"Age\", \"Balance\", \"EstimatedSalary\"] # Specific columns\n", + " }\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "## 2. Histogram Plot\n", + "\n", + "Creates histogram distributions for numerical features with optional KDE overlay. Essential for understanding data distributions.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Basic histogram with KDE\n", + "vm.tests.run_test(\n", + " \"validmind.plots.HistogramPlot\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\n", + " \"columns\": [\"CreditScore\", \"Balance\", \"EstimatedSalary\", \"Age\"],\n", + " \"bins\": 30,\n", + " \"color\": \"steelblue\",\n", + " \"opacity\": 0.7,\n", + " \"show_kde\": True,\n", + " \"normalize\": False,\n", + " \"log_scale\": False,\n", + " \"width\": 1200,\n", + " \"height\": 800,\n", + " \"n_cols\": 2,\n", + " \"vertical_spacing\": 0.15,\n", + " \"horizontal_spacing\": 0.15,\n", + " \"title_prefix\": \"Distribution of\"\n", + " }\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" } + }, + "source": [ + "## 3. Box Plot\n", + "\n", + "Displays box plots for numerical features, optionally grouped by a categorical variable. Excellent for outlier detection and comparing distributions.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Box plots grouped by target variable\n", + "vm.tests.run_test(\n", + " \"validmind.plots.BoxPlot\", \n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\n", + " \"columns\": [\"CreditScore\", \"Balance\", \"Age\"],\n", + " \"group_by\": \"Exited\", # Group by churn status\n", + " \"colors\": [\"lightblue\", \"salmon\"],\n", + " \"show_outliers\": True,\n", + " \"width\": 1200,\n", + " \"height\": 600\n", + " }\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "## 4. Violin Plot\n", + "\n", + "Creates violin plots that combine box plots with kernel density estimation. Shows both summary statistics and distribution shape.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Violin plots grouped by target variable\n", + "vm.tests.run_test(\n", + " \"validmind.plots.ViolinPlot\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\n", + " \"columns\": [\"Age\", \"Balance\"], # Focus on key variables\n", + " \"group_by\": \"Exited\",\n", + " \"width\": 800,\n", + " \"height\": 600\n", + " }\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "# Part 2: Statistical Tests\n", + "\n", + "The ValidMind statistical tests provide comprehensive statistical analysis capabilities for understanding data characteristics and quality.\n", + "\n", + "## 1. Descriptive Statistics\n", + "\n", + "Provides comprehensive descriptive statistics including basic statistics, distribution measures, confidence intervals, and normality tests.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Advanced descriptive statistics with all measures\n", + "vm.tests.run_test(\n", + " \"validmind.stats.DescriptiveStats\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\n", + " \"include_advanced\": True, # Include skewness, kurtosis, normality tests, etc.\n", + " \"confidence_level\": 0.99, # 99% confidence intervals\n", + " \"columns\": [\"CreditScore\", \"Balance\", \"EstimatedSalary\", \"Age\"] # Specific columns\n", + " }\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "## 2. Correlation Analysis\n", + "\n", + "Performs detailed correlation analysis with statistical significance testing and identifies highly correlated feature pairs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Correlation analysis with significance testing\n", + "result = vm.tests.run_test(\n", + " \"validmind.stats.CorrelationAnalysis\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\n", + " \"method\": \"pearson\", # or \"spearman\", \"kendall\"\n", + " \"significance_level\": 0.05,\n", + " \"min_correlation\": 0.1 # Minimum correlation threshold\n", + " }\n", + ")\n", + "result.log()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "## 3. Normality Tests\n", + "\n", + "Performs various normality tests to assess whether features follow a normal distribution.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Comprehensive normality testing\n", + "vm.tests.run_test(\n", + " \"validmind.stats.NormalityTests\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\n", + " \"tests\": [\"shapiro\", \"anderson\", \"kstest\"], # Multiple tests\n", + " \"alpha\": 0.05,\n", + " \"columns\": [\"CreditScore\", \"Balance\", \"Age\"] # Focus on key features\n", + " }\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "## 4. Outlier Detection\n", + "\n", + "Identifies outliers using various statistical methods including IQR, Z-score, and Isolation Forest.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Comprehensive outlier detection with multiple methods\n", + "vm.tests.run_test(\n", + " \"validmind.stats.OutlierDetection\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\n", + " \"methods\": [\"iqr\", \"zscore\", \"isolation_forest\"],\n", + " \"iqr_threshold\": 1.5,\n", + " \"zscore_threshold\": 3.0,\n", + " \"contamination\": 0.1,\n", + " \"columns\": [\"CreditScore\", \"Balance\", \"EstimatedSalary\"]\n", + " }\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "# Part 3: Complete EDA Workflow Example\n", + "\n", + "Let's demonstrate a complete exploratory data analysis workflow using all the tests together:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example: Complete EDA workflow using all tests\n", + "print(\"🔍 Complete Exploratory Data Analysis Workflow\")\n", + "print(\"=\" * 50)\n", + "\n", + "# 1. Start with descriptive statistics\n", + "print(\"\\n1. Descriptive Statistics:\")\n", + "desc_stats = vm.tests.run_test(\n", + " \"validmind.stats.DescriptiveStats\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\"include_advanced\": True}\n", + ")\n", + "\n", + "print(\"\\n2. Distribution Analysis:\")\n", + "# 2. Visualize distributions\n", + "hist_plot = vm.tests.run_test(\n", + " \"validmind.plots.HistogramPlot\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\"show_kde\": True, \"n_cols\": 3}\n", + ")\n", + "\n", + "print(\"\\n3. Correlation Analysis:\")\n", + "# 3. Check correlations\n", + "corr_heatmap = vm.tests.run_test(\n", + " \"validmind.plots.CorrelationHeatmap\",\n", + " inputs={\"dataset\": vm_train_ds}\n", + ")\n", + "\n", + "print(\"\\n4. Outlier Detection:\")\n", + "# 4. Detect outliers\n", + "outliers = vm.tests.run_test(\n", + " \"validmind.stats.OutlierDetection\",\n", + " inputs={\"dataset\": vm_train_ds},\n", + " params={\"methods\": [\"iqr\", \"zscore\"]}\n", + ")\n", + "\n", + "print(\"\\n✅ EDA Complete! Check the visualizations and tables above for insights.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Comprehensive Guide: ValidMind Plots and Statistics Tests\n", + "\n", + "This notebook demonstrates all the available tests from the `validmind.plots` and `validmind.stats` modules. These generalized tests provide powerful visualization and statistical analysis capabilities for any dataset.\n", + "\n", + "## What You'll Learn\n", + "\n", + "In this notebook, we'll explore:\n", + "\n", + "1. **Plotting Tests**: Visual analysis tools for data exploration\n", + " - GeneralCorrelationHeatmap\n", + " - GeneralHistogramPlot\n", + " - GeneralBoxPlot\n", + " - GeneralViolinPlot\n", + "\n", + "2. **Statistical Tests**: Comprehensive statistical analysis tools\n", + " - GeneralDescriptiveStats\n", + " - GeneralCorrelationAnalysis\n", + " - GeneralNormalityTests\n", + " - GeneralOutlierDetection\n", + "\n", + "Each test is highly configurable and can be adapted to different datasets and use cases.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "# Conclusion\n", + "\n", + "This notebook demonstrated all the plotting and statistical tests available in ValidMind:\n", + "\n", + "## Plotting Tests Covered:\n", + "✅ **GeneralCorrelationHeatmap** - Interactive correlation matrices \n", + "✅ **GeneralHistogramPlot** - Distribution analysis with KDE \n", + "✅ **GeneralBoxPlot** - Outlier detection and group comparisons \n", + "✅ **GeneralViolinPlot** - Distribution shape analysis \n", + "\n", + "## Statistical Tests Covered:\n", + "✅ **GeneralDescriptiveStats** - Comprehensive statistical profiling \n", + "✅ **GeneralCorrelationAnalysis** - Formal correlation testing \n", + "✅ **GeneralNormalityTests** - Distribution assumption checking \n", + "✅ **GeneralOutlierDetection** - Multi-method outlier identification \n", + "\n", + "## Key Benefits:\n", + "- **Highly Customizable**: All tests offer extensive parameter options\n", + "- **Interactive Visualizations**: Plotly-based plots with zoom, pan, hover\n", + "- **Statistical Rigor**: Formal testing with significance levels\n", + "- **Flexible Input**: Works with any ValidMind dataset\n", + "- **Comprehensive Output**: Tables, plots, and statistical summaries\n", + "\n", + "## Best Practices:\n", + "\n", + "### When to Use Each Test:\n", + "\n", + "**Plotting Tests:**\n", + "- **GeneralCorrelationHeatmap**: Initial data exploration, multicollinearity detection\n", + "- **GeneralHistogramPlot**: Understanding feature distributions, identifying skewness\n", + "- **GeneralBoxPlot**: Outlier detection, comparing groups\n", + "- **GeneralViolinPlot**: Detailed distribution analysis, especially for grouped data\n", + "\n", + "**Statistical Tests:**\n", + "- **GeneralDescriptiveStats**: Comprehensive data profiling, baseline statistics\n", + "- **GeneralCorrelationAnalysis**: Formal correlation testing with significance\n", + "- **GeneralNormalityTests**: Model assumption checking\n", + "- **GeneralOutlierDetection**: Data quality assessment, preprocessing decisions\n", + "\n", + "## Next Steps:\n", + "- Integrate these tests into your model documentation templates\n", + "- Customize parameters based on your specific data characteristics\n", + "- Use results to inform preprocessing and modeling decisions\n", + "- Combine with ValidMind's model validation tests for complete analysis\n", + "\n", + "These tests provide a solid foundation for exploratory data analysis, data quality assessment, and statistical validation in any data science workflow.\n" + ] + }, + { + "cell_type": "markdown", + "id": "ce1de6a7", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ValidMind Library", + "language": "python", + "name": "validmind" }, - "nbformat": 4, - "nbformat_minor": 2 + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb b/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb index b394d1cce..c16e03eaf 100644 --- a/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb +++ b/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb @@ -120,6 +120,22 @@ "# and finally, lets delete the temporary file\n", "os.remove(temp_file.name)" ] + }, + { + "cell_type": "markdown", + "id": "8fb2b3e2", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] } ], "metadata": { diff --git a/notebooks/code_sharing/r_demo/r-ecm-model.ipynb b/notebooks/code_sharing/r_demo/r-ecm-model.ipynb index 4c1d1a3ed..e0a8a886a 100644 --- a/notebooks/code_sharing/r_demo/r-ecm-model.ipynb +++ b/notebooks/code_sharing/r_demo/r-ecm-model.ipynb @@ -208,6 +208,22 @@ "# save the model to an RDS file\n", "saveRDS(model, 'r-ecm-model.rds')" ] + }, + { + "cell_type": "markdown", + "id": "49991d76", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] } ], "metadata": { diff --git a/notebooks/how_to/assign_scores_complete_tutorial.ipynb b/notebooks/how_to/assign_scores_complete_tutorial.ipynb index 7339a9a43..24adfa9ce 100644 --- a/notebooks/how_to/assign_scores_complete_tutorial.ipynb +++ b/notebooks/how_to/assign_scores_complete_tutorial.ipynb @@ -1,810 +1,813 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Intro to Assign Scores" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `assign_scores()` method is a powerful feature that allows you to compute and add scorer scores as new columns in your dataset. This method takes a model and metric(s) as input, computes the specified metrics from the ValidMind scorer library, and adds them as new columns. The computed metrics provide per-row values, giving you granular insights into model performance at the individual prediction level.\n", - "\n", - "In this interactive notebook, we demonstrate how to use the `assign_scores()` method effectively. We'll walk through a complete example using a customer churn dataset, showing how to compute and assign row-level metrics (like Brier Score and Log Loss) that provide detailed performance insights for each prediction. You'll learn how to work with single and multiple scorers, pass custom parameters, and handle different metric types - all while maintaining a clean, organized dataset structure. Currently, assign_scores() supports all metrics available in the validmind.scorer module.\n", - "\n", - "**The Power of Row-Level Scoring**\n", - "\n", - "Traditional model evaluation workflows often focus on aggregate metrics that provide overall performance summaries. The `assign_scores()` method complements this by providing granular, row-level insights that help you:\n", - "\n", - "- **Identify Problematic Predictions**: Spot individual cases where your model performs poorly\n", - "- **Understand Model Behavior**: Analyze how model performance varies across different types of inputs\n", - "- **Enable Detailed Analysis**: Perform targeted investigations on specific subsets of your data\n", - "- **Support Model Debugging**: Pinpoint exactly where and why your model makes errors\n", - "\n", - "**Understanding assign_scores()**\n", - "\n", - "The `assign_scores()` method computes row metrics for a given model-dataset combination and adds the results as new columns to your dataset. Each new column follows the naming convention: `{model.input_id}_{metric_name}`, ensuring clear identification of which model and metric combination generated each score.\n", - "\n", - "Key features:\n", - "\n", - "- **Row-Level Focus**: Computes per-prediction metrics rather than aggregate scores\n", - "- **Flexible Input**: Accepts single metrics or lists of metrics\n", - "- **Parameter Support**: Allows passing additional parameters to underlying metric implementations\n", - "- **Multi-Model Support**: Can assign scores from multiple models to the same dataset\n", - "- **Type Agnostic**: Works with classification, regression, and other model types\n", - "\n", - "This approach provides detailed insights into your model's performance at the individual prediction level, enabling more sophisticated analysis and debugging workflows." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "::: {.content-hidden when-format=\"html\"}\n", - "## Contents \n", - "- [About ValidMind](#toc1__) \n", - " - [Before you begin](#toc1_1__) \n", - " - [New to ValidMind?](#toc1_2__) \n", - "- [Setting up](#toc2__) \n", - " - [Install the ValidMind Library](#toc2_1__) \n", - " - [Initialize the ValidMind Library](#toc2_2__) \n", - " - [Register sample model](#toc2_2_1__) \n", - " - [Apply documentation template](#toc2_2_2__) \n", - " - [Get your code snippet](#toc2_2_3__) \n", - "- [Load the demo dataset](#toc3__) \n", - "- [Train models for testing](#toc4__) \n", - "- [Initialize ValidMind objects](#toc5__) \n", - "- [Assign predictions](#toc6__) \n", - "- [Using assign_scores()](#toc7__) \n", - " - [Basic Usage](#toc7_1__) \n", - " - [Single Scorer Assignment](#toc7_2__) \n", - " - [A Scorer returns complex object](#toc7_3__) \n", - " - [Multiple Scorers Assignment](#toc7_4__) \n", - " - [Passing Parameters to Scorer](#toc7_5__) \n", - " - [Multi-Model scorers](#toc7_6__) \n", - " - [Scorer Metrics](#toc7_7__) \n", - " - [Custom Scorer](#toc7_8__) \n", - "- [Next steps](#toc8__) \n", - " - [Work with your model documentation](#toc8_1__) \n", - " - [Discover more learning resources](#toc8_2__) \n", - "- [Upgrade ValidMind](#toc9__) \n", - "\n", - ":::\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## About ValidMind\n", - "\n", - "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n", - "\n", - "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n", - "\n", - "\n", - "\n", - "### Before you begin\n", - "\n", - "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n", - "\n", - "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n", - "\n", - "\n", - "\n", - "### New to ValidMind?\n", - "\n", - "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n", - "\n", - "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", - "

\n", - "Register with ValidMind
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Setting up" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Install the ValidMind Library\n", - "\n", - "To install the library:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -q validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the ValidMind Library" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Register sample model\n", - "\n", - "Let's first register a sample model for use with this notebook:\n", - "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", - "\n", - "3. Enter the model details and click **Next >** to continue to assignment of model stakeholders. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", - "\n", - " For example, to register a model for use with this notebook, select the following use case: `Marketing/Sales - Analytics`\n", - "\n", - "4. Select your own name under the **MODEL OWNER** drop-down.\n", - "\n", - "5. Click **Register Model** to add the model to your inventory." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Apply documentation template\n", - "\n", - "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", - "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", - "\n", - "2. Under **TEMPLATE**, select `Binary classification`.\n", - "\n", - "3. Click **Use Template** to apply the template." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Get your code snippet\n", - "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", - "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load your model identifier credentials from an `.env` file\n", - "\n", - "%load_ext dotenv\n", - "%dotenv .env\n", - "\n", - "# Or replace with your code snippet\n", - "\n", - "import validmind as vm\n", - "\n", - "vm.init(\n", - " api_host=\"...\",\n", - " api_key=\"...\",\n", - " api_secret=\"...\",\n", - " model=\"...\",\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Load the demo dataset\n", - "\n", - "In this example, we load a demo dataset to demonstrate the assign_scores functionality with customer churn prediction models." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from validmind.datasets.classification import customer_churn as demo_dataset\n", - "\n", - "print(\n", - " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n", - ")\n", - "\n", - "raw_df = demo_dataset.load_data()\n", - "raw_df.head()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Train models for testing\n", - "\n", - "We'll train two different customer churn models to demonstrate the assign_scores functionality with multiple models." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import xgboost as xgb\n", - "from sklearn.ensemble import RandomForestClassifier\n", - "\n", - "# Preprocess the data\n", - "train_df, validation_df, test_df = demo_dataset.preprocess(raw_df)\n", - "\n", - "# Prepare training data\n", - "x_train = train_df.drop(demo_dataset.target_column, axis=1)\n", - "y_train = train_df[demo_dataset.target_column]\n", - "x_val = validation_df.drop(demo_dataset.target_column, axis=1)\n", - "y_val = validation_df[demo_dataset.target_column]\n", - "\n", - "# Train XGBoost model\n", - "xgb_model = xgb.XGBClassifier(early_stopping_rounds=10, random_state=42)\n", - "xgb_model.set_params(\n", - " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", - ")\n", - "xgb_model.fit(\n", - " x_train,\n", - " y_train,\n", - " eval_set=[(x_val, y_val)],\n", - " verbose=False,\n", - ")\n", - "\n", - "# Train Random Forest model\n", - "rf_model = RandomForestClassifier(n_estimators=100, random_state=42)\n", - "rf_model.fit(x_train, y_train)\n", - "\n", - "print(\"Models trained successfully!\")\n", - "print(f\"XGBoost training accuracy: {xgb_model.score(x_train, y_train):.3f}\")\n", - "print(f\"Random Forest training accuracy: {rf_model.score(x_train, y_train):.3f}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Initialize ValidMind objects\n", - "\n", - "We initialize ValidMind `dataset` and `model` objects. The `input_id` parameter is crucial for the assign_scores functionality as it determines the column naming convention for assigned scores." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize datasets\n", - "vm_train_ds = vm.init_dataset(\n", - " input_id=\"train_dataset\",\n", - " dataset=train_df,\n", - " target_column=demo_dataset.target_column,\n", - ")\n", - "vm_test_ds = vm.init_dataset(\n", - " input_id=\"test_dataset\",\n", - " dataset=test_df,\n", - " target_column=demo_dataset.target_column,\n", - ")\n", - "\n", - "# Initialize models with descriptive input_ids\n", - "vm_xgb_model = vm.init_model(model=xgb_model, input_id=\"xgboost_model\")\n", - "vm_rf_model = vm.init_model(model=rf_model, input_id=\"random_forest_model\")\n", - "\n", - "print(\"ValidMind objects initialized successfully!\")\n", - "print(f\"XGBoost model ID: {vm_xgb_model.input_id}\")\n", - "print(f\"Random Forest model ID: {vm_rf_model.input_id}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Assign predictions\n", - "\n", - "Before we can use assign_scores(), we need to assign predictions to our datasets. This step is essential as many unit metrics require both actual and predicted values." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Assign predictions for both models to both datasets\n", - "vm_train_ds.assign_predictions(model=vm_xgb_model)\n", - "vm_train_ds.assign_predictions(model=vm_rf_model)\n", - "\n", - "vm_test_ds.assign_predictions(model=vm_xgb_model)\n", - "vm_test_ds.assign_predictions(model=vm_rf_model)\n", - "\n", - "print(\"Predictions assigned successfully!\")\n", - "print(f\"Test dataset now has {len(vm_test_ds.df.columns)} columns\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Using assign_scores()\n", - "\n", - "Now we'll explore the various ways to use the assign_scores() method to integrate performance metrics directly into your dataset." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Basic Usage\n", - "\n", - "The assign_scores() method has a simple interface:\n", - "\n", - "```python\n", - "dataset.assign_scores(model, metrics, **kwargs)\n", - "```\n", - "\n", - "- **model**: A ValidMind model object\n", - "- **metrics**: Single metric ID or list of metric IDs (can use short names or full IDs)\n", - "- **kwargs**: Additional parameters passed to the underlying metric implementations\n", - "\n", - "Let's first check what columns we currently have in our test dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Current columns in test dataset:\")\n", - "for i, col in enumerate(vm_test_ds.df.columns, 1):\n", - " print(f\"{i:2d}. {col}\")\n", - "\n", - "print(f\"\\nDataset shape: {vm_test_ds.df.shape}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Single Scorer Assignment\n", - " \n", - "Let's start by assigning a single Scorer - the Brier Score - for our XGBoost model on the test dataset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Assign Brier Score for XGBoost model\n", - "vm_test_ds.assign_scores(metrics = \"validmind.scorer.classification.BrierScore\", model = vm_xgb_model)\n", - "\n", - "print(\"After assigning Brier Score:\")\n", - "print(f\"New column added: {vm_test_ds.df.columns}\")\n", - "# Display the metric values\n", - "vm_test_ds.df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### A Scorer returns complex object\n", - " The OutlierScore scorer demonstrates how scorers can return complex objects. It returns a dictionary containing per-row outlier detection results. For each row, it includes:\n", - " - is_outlier: Boolean indicating if the row is an outlier\n", - " - anomaly_score: Numerical score indicating degree of outlierness\n", - " - isolation_path: Length of isolation path in the tree\n", - "\n", - "When assigned to a dataset, these dictionary values are automatically unpacked into separate columns with appropriate prefixes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Assign Brier Score for XGBoost model\n", - "vm_test_ds.assign_scores(metrics = \"validmind.scorer.classification.OutlierScore\", model = vm_xgb_model)\n", - "\n", - "print(\"After assigning Score With Confidence:\")\n", - "print(f\"New column added: {vm_test_ds.df.columns}\")\n", - "# Display the metric values\n", - "vm_test_ds.df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Assign Brier Score for XGBoost model\n", - "vm_test_ds.assign_scores(\"validmind.scorer.classification.OutlierScore\")\n", - "\n", - "print(\"After assigning Score With Confidence:\")\n", - "print(f\"New column added: {vm_test_ds.df.columns}\")\n", - "# Display the metric values\n", - "vm_test_ds.df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Multiple Scorers Assignment\n", - "\n", - "We can assign multiple metrics at once by passing a list of Scorer names. This is more efficient than calling assign_scores() multiple times." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Assign multiple classification metrics for the Random Forest model\n", - "scorer = [\n", - " \"validmind.scorer.classification.BrierScore\",\n", - " \"validmind.scorer.classification.LogLoss\",\n", - " \"validmind.scorer.classification.Confidence\"\n", - "]\n", - "\n", - "vm_test_ds.assign_scores(metrics = scorer, model = vm_rf_model)\n", - "\n", - "print(\"After assigning multiple row metrics for Random Forest:\")\n", - "rf_columns = [col for col in vm_test_ds.df.columns if 'random_forest_model' in col]\n", - "print(f\"Random Forest columns: {rf_columns}\")\n", - "\n", - "# Display the metric values\n", - "vm_test_ds.df[rf_columns].head()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Passing Parameters to Scorer\n", - "\n", - "Many row metrics accept additional parameters that are passed through to the underlying implementations. Let's demonstrate this with the LogLoss metric." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Assign LogLoss\n", - "vm_test_ds.assign_scores(metrics = \"validmind.scorer.classification.LogLoss\", model = vm_xgb_model, eps = 1e-16)\n", - "\n", - "# We can also assign with different parameters by calling assign_scores again\n", - "# Note: This will overwrite the previous column with the same name\n", - "print(\"LogLoss assigned successfully\")\n", - "\n", - "# Let's also assign BrierScore and Confidence\n", - "vm_test_ds.assign_scores(metrics = [\"validmind.scorer.classification.BrierScore\",\"validmind.scorer.classification.Confidence\"], model = vm_xgb_model)\n", - "\n", - "print(\"BrierScore and Confidence assigned successfully\")\n", - "\n", - "# Display current XGBoost metric columns\n", - "xgb_columns = [col for col in vm_test_ds.df.columns if 'xgboost_model' in col]\n", - "print(f\"\\nXGBoost model columns: {xgb_columns}\")\n", - "\n", - "vm_test_ds.df[xgb_columns].head()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Multi-Model scorers\n", - "\n", - "One of the powerful features of assign_scores() is the ability to assign scores from multiple models to the same dataset, enabling detailed model comparison at the prediction level." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Let's assign a comprehensive set of metrics for both models\n", - "comprehensive_metrics = [\n", - " \"validmind.scorer.classification.BrierScore\",\n", - " \"validmind.scorer.classification.LogLoss\",\n", - " \"validmind.scorer.classification.Confidence\",\n", - " \"validmind.scorer.classification.Correctness\"\n", - "]\n", - "\n", - "# Assign for XGBoost model\n", - "vm_test_ds.assign_scores(metrics = comprehensive_metrics, model = vm_xgb_model)\n", - "\n", - "# Assign for Random Forest model}\n", - "vm_test_ds.assign_scores(metrics = comprehensive_metrics, model = vm_rf_model)\n", - "\n", - "print(\"Row-level metrics assigned for both models!\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Scorer Metrics\n", - "The next section demonstrates how to assign individual metrics that compute scores per row, rather than aggregate metrics.\n", - "We'll use several important row metrics:\n", - " \n", - "- Brier Score: Measures how well calibrated the model's probability predictions are for each individual prediction\n", - "- Log Loss: Evaluates how well the predicted probabilities match the true labels on a per-prediction basis\n", - "- Confidence: Measures the model's confidence in its predictions for each row\n", - "- Correctness: Indicates whether each prediction is correct (1) or incorrect (0)\n", - "\n", - "All these metrics provide granular insights into model performance at the individual prediction level." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Let's add some individual metrics that compute per-row scores\n", - "print(\"Adding individual metrics...\")\n", - "\n", - "# Add Brier Score - measures accuracy of probabilistic predictions per row\n", - "vm_test_ds.assign_scores(metrics = \"validmind.scorer.classification.BrierScore\", model = vm_xgb_model)\n", - "print(\"Added Brier Score - lower values indicate better calibrated probabilities\")\n", - "\n", - "# Add Log Loss - measures how well the predicted probabilities match true labels per row\n", - "vm_test_ds.assign_scores(metrics = \"validmind.scorer.classification.LogLoss\", model = vm_xgb_model)\n", - "print(\"Added Log Loss - lower values indicate better probability estimates\")\n", - "\n", - "# Create a comparison summary showing first few rows of individual metrics\n", - "print(\"\\nFirst few rows of individual metrics:\")\n", - "individual_metrics = [col for col in vm_test_ds.df.columns if any(m in col for m in ['BrierScore', 'LogLoss', 'Confidence', 'Correctness'])]\n", - "print(vm_test_ds.df[individual_metrics].head())\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vm_test_ds._df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Custom Scorer\n", - "Let's see how to create your own custom scorers using the `@scorer` decorator.\n", - " \n", - "The example below demonstrates a scorer that looks at the class balance in the neighborhood around each data point. For each row, it will give you a score from 0 to 1, where a score closer to 1 means there's a nice even balance of classes in that area of your data. This can help you identify regions where your classes are well-mixed vs regions dominated by a single class." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from validmind.scorer import scorer\n", - "import numpy as np\n", - "\n", - "@scorer(\"my_scorers.TestScorer\") \n", - "def test_scorer(model, dataset):\n", - " \"\"\"Custom scorer that calculates class balance ratio.\n", - " \n", - " Args:\n", - " model: Not used in this scorer\n", - " dataset: The dataset to analyze\n", - " \n", - " Returns:\n", - " numpy.ndarray: Array of class balance ratios between 0 and 1,\n", - " where values closer to 1 indicate better class balance in the local neighborhood\n", - " \"\"\"\n", - " # Get target values\n", - " y = dataset.df[dataset.target_column].values\n", - " \n", - " # Calculate local class balance in sliding windows\n", - " window_size = 100\n", - " balance_scores = []\n", - " \n", - " for i in range(len(y)):\n", - " start_idx = max(0, i - window_size//2)\n", - " end_idx = min(len(y), i + window_size//2)\n", - " window = y[start_idx:end_idx]\n", - " \n", - " # Calculate ratio of minority class\n", - " class_ratio = np.mean(window)\n", - " # Adjust to be symmetric around 0.5\n", - " balance_score = 1 - abs(0.5 - class_ratio) * 2\n", - " \n", - " balance_scores.append(balance_score)\n", - " \n", - " return np.array(balance_scores)\n", - "\n", - "# Assign the class balance scores to the dataset\n", - "vm_test_ds.assign_scores(metrics = \"my_scorers.TestScorer\", model = vm_xgb_model)\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Next steps\n", - "\n", - "You can explore the assigned scores right in the notebook as demonstrated above. However, there's even more value in using the ValidMind Platform to work with your model documentation and monitoring.\n", - "\n", - "\n", - "\n", - "### Work with your model documentation\n", - "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", - "\n", - "2. Click and expand the **Model Development** section.\n", - "\n", - "The scores you've assigned using `assign_scores()` become part of your model's documentation and can be used in ongoing monitoring workflows. You can view these metrics over time, set up alerts for performance drift, and compare models systematically. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", - "\n", - "\n", - "\n", - "### Discover more learning resources\n", - "\n", - "We offer many interactive notebooks to help you work with model scoring and evaluation:\n", - "\n", - "- [Run unit metrics](https://docs.validmind.ai/developer/model-testing/testing-overview.html)\n", - "- [Assign predictions](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)\n", - "- [Model comparison workflows](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)\n", - "\n", - "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Upgrade ValidMind\n", - "\n", - "
After installing ValidMind, you'll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n", - "\n", - "Retrieve the information for the currently installed version of ValidMind:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "%pip show validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n", - "\n", - "```bash\n", - "%pip install --upgrade validmind\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You may need to restart your kernel after running the upgrade package for changes to be applied." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "***\n", - "\n", - "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", - "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "ValidMind Library", - "language": "python", - "name": "validmind" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Intro to Assign Scores" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `assign_scores()` method is a powerful feature that allows you to compute and add scorer scores as new columns in your dataset. This method takes a model and metric(s) as input, computes the specified metrics from the ValidMind scorer library, and adds them as new columns. The computed metrics provide per-row values, giving you granular insights into model performance at the individual prediction level.\n", + "\n", + "In this interactive notebook, we demonstrate how to use the `assign_scores()` method effectively. We'll walk through a complete example using a customer churn dataset, showing how to compute and assign row-level metrics (like Brier Score and Log Loss) that provide detailed performance insights for each prediction. You'll learn how to work with single and multiple scorers, pass custom parameters, and handle different metric types - all while maintaining a clean, organized dataset structure. Currently, assign_scores() supports all metrics available in the validmind.scorer module.\n", + "\n", + "**The Power of Row-Level Scoring**\n", + "\n", + "Traditional model evaluation workflows often focus on aggregate metrics that provide overall performance summaries. The `assign_scores()` method complements this by providing granular, row-level insights that help you:\n", + "\n", + "- **Identify Problematic Predictions**: Spot individual cases where your model performs poorly\n", + "- **Understand Model Behavior**: Analyze how model performance varies across different types of inputs\n", + "- **Enable Detailed Analysis**: Perform targeted investigations on specific subsets of your data\n", + "- **Support Model Debugging**: Pinpoint exactly where and why your model makes errors\n", + "\n", + "**Understanding assign_scores()**\n", + "\n", + "The `assign_scores()` method computes row metrics for a given model-dataset combination and adds the results as new columns to your dataset. Each new column follows the naming convention: `{model.input_id}_{metric_name}`, ensuring clear identification of which model and metric combination generated each score.\n", + "\n", + "Key features:\n", + "\n", + "- **Row-Level Focus**: Computes per-prediction metrics rather than aggregate scores\n", + "- **Flexible Input**: Accepts single metrics or lists of metrics\n", + "- **Parameter Support**: Allows passing additional parameters to underlying metric implementations\n", + "- **Multi-Model Support**: Can assign scores from multiple models to the same dataset\n", + "- **Type Agnostic**: Works with classification, regression, and other model types\n", + "\n", + "This approach provides detailed insights into your model's performance at the individual prediction level, enabling more sophisticated analysis and debugging workflows." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [About ValidMind](#toc1__) \n", + " - [Before you begin](#toc1_1__) \n", + " - [New to ValidMind?](#toc1_2__) \n", + "- [Setting up](#toc2__) \n", + " - [Install the ValidMind Library](#toc2_1__) \n", + " - [Initialize the ValidMind Library](#toc2_2__) \n", + " - [Register sample model](#toc2_2_1__) \n", + " - [Apply documentation template](#toc2_2_2__) \n", + " - [Get your code snippet](#toc2_2_3__) \n", + "- [Load the demo dataset](#toc3__) \n", + "- [Train models for testing](#toc4__) \n", + "- [Initialize ValidMind objects](#toc5__) \n", + "- [Assign predictions](#toc6__) \n", + "- [Using assign_scores()](#toc7__) \n", + " - [Basic Usage](#toc7_1__) \n", + " - [Single Scorer Assignment](#toc7_2__) \n", + " - [A Scorer returns complex object](#toc7_3__) \n", + " - [Multiple Scorers Assignment](#toc7_4__) \n", + " - [Passing Parameters to Scorer](#toc7_5__) \n", + " - [Multi-Model scorers](#toc7_6__) \n", + " - [Scorer Metrics](#toc7_7__) \n", + " - [Custom Scorer](#toc7_8__) \n", + "- [Next steps](#toc8__) \n", + " - [Work with your model documentation](#toc8_1__) \n", + " - [Discover more learning resources](#toc8_2__) \n", + "- [Upgrade ValidMind](#toc9__) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## About ValidMind\n", + "\n", + "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n", + "\n", + "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n", + "\n", + "\n", + "\n", + "### Before you begin\n", + "\n", + "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n", + "\n", + "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n", + "\n", + "\n", + "\n", + "### New to ValidMind?\n", + "\n", + "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n", + "\n", + "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", + "

\n", + "Register with ValidMind
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Setting up" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Install the ValidMind Library\n", + "\n", + "To install the library:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q validmind" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind Library" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Register sample model\n", + "\n", + "Let's first register a sample model for use with this notebook:\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", + "\n", + "3. Enter the model details and click **Next >** to continue to assignment of model stakeholders. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", + "\n", + " For example, to register a model for use with this notebook, select the following use case: `Marketing/Sales - Analytics`\n", + "\n", + "4. Select your own name under the **MODEL OWNER** drop-down.\n", + "\n", + "5. Click **Register Model** to add the model to your inventory." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Apply documentation template\n", + "\n", + "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", + "\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "\n", + "2. Under **TEMPLATE**, select `Binary classification`.\n", + "\n", + "3. Click **Use Template** to apply the template." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Get your code snippet\n", + "\n", + "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "\n", + "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", + "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load your model identifier credentials from an `.env` file\n", + "\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " api_host=\"...\",\n", + " api_key=\"...\",\n", + " api_secret=\"...\",\n", + " model=\"...\",\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Load the demo dataset\n", + "\n", + "In this example, we load a demo dataset to demonstrate the assign_scores functionality with customer churn prediction models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.datasets.classification import customer_churn as demo_dataset\n", + "\n", + "print(\n", + " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n", + ")\n", + "\n", + "raw_df = demo_dataset.load_data()\n", + "raw_df.head()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Train models for testing\n", + "\n", + "We'll train two different customer churn models to demonstrate the assign_scores functionality with multiple models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import xgboost as xgb\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "\n", + "# Preprocess the data\n", + "train_df, validation_df, test_df = demo_dataset.preprocess(raw_df)\n", + "\n", + "# Prepare training data\n", + "x_train = train_df.drop(demo_dataset.target_column, axis=1)\n", + "y_train = train_df[demo_dataset.target_column]\n", + "x_val = validation_df.drop(demo_dataset.target_column, axis=1)\n", + "y_val = validation_df[demo_dataset.target_column]\n", + "\n", + "# Train XGBoost model\n", + "xgb_model = xgb.XGBClassifier(early_stopping_rounds=10, random_state=42)\n", + "xgb_model.set_params(\n", + " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", + ")\n", + "xgb_model.fit(\n", + " x_train,\n", + " y_train,\n", + " eval_set=[(x_val, y_val)],\n", + " verbose=False,\n", + ")\n", + "\n", + "# Train Random Forest model\n", + "rf_model = RandomForestClassifier(n_estimators=100, random_state=42)\n", + "rf_model.fit(x_train, y_train)\n", + "\n", + "print(\"Models trained successfully!\")\n", + "print(f\"XGBoost training accuracy: {xgb_model.score(x_train, y_train):.3f}\")\n", + "print(f\"Random Forest training accuracy: {rf_model.score(x_train, y_train):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Initialize ValidMind objects\n", + "\n", + "We initialize ValidMind `dataset` and `model` objects. The `input_id` parameter is crucial for the assign_scores functionality as it determines the column naming convention for assigned scores." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize datasets\n", + "vm_train_ds = vm.init_dataset(\n", + " input_id=\"train_dataset\",\n", + " dataset=train_df,\n", + " target_column=demo_dataset.target_column,\n", + ")\n", + "vm_test_ds = vm.init_dataset(\n", + " input_id=\"test_dataset\",\n", + " dataset=test_df,\n", + " target_column=demo_dataset.target_column,\n", + ")\n", + "\n", + "# Initialize models with descriptive input_ids\n", + "vm_xgb_model = vm.init_model(model=xgb_model, input_id=\"xgboost_model\")\n", + "vm_rf_model = vm.init_model(model=rf_model, input_id=\"random_forest_model\")\n", + "\n", + "print(\"ValidMind objects initialized successfully!\")\n", + "print(f\"XGBoost model ID: {vm_xgb_model.input_id}\")\n", + "print(f\"Random Forest model ID: {vm_rf_model.input_id}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Assign predictions\n", + "\n", + "Before we can use assign_scores(), we need to assign predictions to our datasets. This step is essential as many unit metrics require both actual and predicted values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assign predictions for both models to both datasets\n", + "vm_train_ds.assign_predictions(model=vm_xgb_model)\n", + "vm_train_ds.assign_predictions(model=vm_rf_model)\n", + "\n", + "vm_test_ds.assign_predictions(model=vm_xgb_model)\n", + "vm_test_ds.assign_predictions(model=vm_rf_model)\n", + "\n", + "print(\"Predictions assigned successfully!\")\n", + "print(f\"Test dataset now has {len(vm_test_ds.df.columns)} columns\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Using assign_scores()\n", + "\n", + "Now we'll explore the various ways to use the assign_scores() method to integrate performance metrics directly into your dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Basic Usage\n", + "\n", + "The assign_scores() method has a simple interface:\n", + "\n", + "```python\n", + "dataset.assign_scores(model, metrics, **kwargs)\n", + "```\n", + "\n", + "- **model**: A ValidMind model object\n", + "- **metrics**: Single metric ID or list of metric IDs (can use short names or full IDs)\n", + "- **kwargs**: Additional parameters passed to the underlying metric implementations\n", + "\n", + "Let's first check what columns we currently have in our test dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Current columns in test dataset:\")\n", + "for i, col in enumerate(vm_test_ds.df.columns, 1):\n", + " print(f\"{i:2d}. {col}\")\n", + "\n", + "print(f\"\\nDataset shape: {vm_test_ds.df.shape}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Single Scorer Assignment\n", + " \n", + "Let's start by assigning a single Scorer - the Brier Score - for our XGBoost model on the test dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assign Brier Score for XGBoost model\n", + "vm_test_ds.assign_scores(metrics = \"validmind.scorer.classification.BrierScore\", model = vm_xgb_model)\n", + "\n", + "print(\"After assigning Brier Score:\")\n", + "print(f\"New column added: {vm_test_ds.df.columns}\")\n", + "# Display the metric values\n", + "vm_test_ds.df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### A Scorer returns complex object\n", + " The OutlierScore scorer demonstrates how scorers can return complex objects. It returns a dictionary containing per-row outlier detection results. For each row, it includes:\n", + " - is_outlier: Boolean indicating if the row is an outlier\n", + " - anomaly_score: Numerical score indicating degree of outlierness\n", + " - isolation_path: Length of isolation path in the tree\n", + "\n", + "When assigned to a dataset, these dictionary values are automatically unpacked into separate columns with appropriate prefixes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assign Brier Score for XGBoost model\n", + "vm_test_ds.assign_scores(metrics = \"validmind.scorer.classification.OutlierScore\", model = vm_xgb_model)\n", + "\n", + "print(\"After assigning Score With Confidence:\")\n", + "print(f\"New column added: {vm_test_ds.df.columns}\")\n", + "# Display the metric values\n", + "vm_test_ds.df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assign Brier Score for XGBoost model\n", + "vm_test_ds.assign_scores(\"validmind.scorer.classification.OutlierScore\")\n", + "\n", + "print(\"After assigning Score With Confidence:\")\n", + "print(f\"New column added: {vm_test_ds.df.columns}\")\n", + "# Display the metric values\n", + "vm_test_ds.df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Multiple Scorers Assignment\n", + "\n", + "We can assign multiple metrics at once by passing a list of Scorer names. This is more efficient than calling assign_scores() multiple times." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assign multiple classification metrics for the Random Forest model\n", + "scorer = [\n", + " \"validmind.scorer.classification.BrierScore\",\n", + " \"validmind.scorer.classification.LogLoss\",\n", + " \"validmind.scorer.classification.Confidence\"\n", + "]\n", + "\n", + "vm_test_ds.assign_scores(metrics = scorer, model = vm_rf_model)\n", + "\n", + "print(\"After assigning multiple row metrics for Random Forest:\")\n", + "rf_columns = [col for col in vm_test_ds.df.columns if 'random_forest_model' in col]\n", + "print(f\"Random Forest columns: {rf_columns}\")\n", + "\n", + "# Display the metric values\n", + "vm_test_ds.df[rf_columns].head()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Passing Parameters to Scorer\n", + "\n", + "Many row metrics accept additional parameters that are passed through to the underlying implementations. Let's demonstrate this with the LogLoss metric." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assign LogLoss\n", + "vm_test_ds.assign_scores(metrics = \"validmind.scorer.classification.LogLoss\", model = vm_xgb_model, eps = 1e-16)\n", + "\n", + "# We can also assign with different parameters by calling assign_scores again\n", + "# Note: This will overwrite the previous column with the same name\n", + "print(\"LogLoss assigned successfully\")\n", + "\n", + "# Let's also assign BrierScore and Confidence\n", + "vm_test_ds.assign_scores(metrics = [\"validmind.scorer.classification.BrierScore\",\"validmind.scorer.classification.Confidence\"], model = vm_xgb_model)\n", + "\n", + "print(\"BrierScore and Confidence assigned successfully\")\n", + "\n", + "# Display current XGBoost metric columns\n", + "xgb_columns = [col for col in vm_test_ds.df.columns if 'xgboost_model' in col]\n", + "print(f\"\\nXGBoost model columns: {xgb_columns}\")\n", + "\n", + "vm_test_ds.df[xgb_columns].head()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Multi-Model scorers\n", + "\n", + "One of the powerful features of assign_scores() is the ability to assign scores from multiple models to the same dataset, enabling detailed model comparison at the prediction level." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Let's assign a comprehensive set of metrics for both models\n", + "comprehensive_metrics = [\n", + " \"validmind.scorer.classification.BrierScore\",\n", + " \"validmind.scorer.classification.LogLoss\",\n", + " \"validmind.scorer.classification.Confidence\",\n", + " \"validmind.scorer.classification.Correctness\"\n", + "]\n", + "\n", + "# Assign for XGBoost model\n", + "vm_test_ds.assign_scores(metrics = comprehensive_metrics, model = vm_xgb_model)\n", + "\n", + "# Assign for Random Forest model}\n", + "vm_test_ds.assign_scores(metrics = comprehensive_metrics, model = vm_rf_model)\n", + "\n", + "print(\"Row-level metrics assigned for both models!\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Scorer Metrics\n", + "The next section demonstrates how to assign individual metrics that compute scores per row, rather than aggregate metrics.\n", + "We'll use several important row metrics:\n", + " \n", + "- Brier Score: Measures how well calibrated the model's probability predictions are for each individual prediction\n", + "- Log Loss: Evaluates how well the predicted probabilities match the true labels on a per-prediction basis\n", + "- Confidence: Measures the model's confidence in its predictions for each row\n", + "- Correctness: Indicates whether each prediction is correct (1) or incorrect (0)\n", + "\n", + "All these metrics provide granular insights into model performance at the individual prediction level." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Let's add some individual metrics that compute per-row scores\n", + "print(\"Adding individual metrics...\")\n", + "\n", + "# Add Brier Score - measures accuracy of probabilistic predictions per row\n", + "vm_test_ds.assign_scores(metrics = \"validmind.scorer.classification.BrierScore\", model = vm_xgb_model)\n", + "print(\"Added Brier Score - lower values indicate better calibrated probabilities\")\n", + "\n", + "# Add Log Loss - measures how well the predicted probabilities match true labels per row\n", + "vm_test_ds.assign_scores(metrics = \"validmind.scorer.classification.LogLoss\", model = vm_xgb_model)\n", + "print(\"Added Log Loss - lower values indicate better probability estimates\")\n", + "\n", + "# Create a comparison summary showing first few rows of individual metrics\n", + "print(\"\\nFirst few rows of individual metrics:\")\n", + "individual_metrics = [col for col in vm_test_ds.df.columns if any(m in col for m in ['BrierScore', 'LogLoss', 'Confidence', 'Correctness'])]\n", + "print(vm_test_ds.df[individual_metrics].head())\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vm_test_ds._df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Custom Scorer\n", + "Let's see how to create your own custom scorers using the `@scorer` decorator.\n", + " \n", + "The example below demonstrates a scorer that looks at the class balance in the neighborhood around each data point. For each row, it will give you a score from 0 to 1, where a score closer to 1 means there's a nice even balance of classes in that area of your data. This can help you identify regions where your classes are well-mixed vs regions dominated by a single class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.scorer import scorer\n", + "import numpy as np\n", + "\n", + "@scorer(\"my_scorers.TestScorer\") \n", + "def test_scorer(model, dataset):\n", + " \"\"\"Custom scorer that calculates class balance ratio.\n", + " \n", + " Args:\n", + " model: Not used in this scorer\n", + " dataset: The dataset to analyze\n", + " \n", + " Returns:\n", + " numpy.ndarray: Array of class balance ratios between 0 and 1,\n", + " where values closer to 1 indicate better class balance in the local neighborhood\n", + " \"\"\"\n", + " # Get target values\n", + " y = dataset.df[dataset.target_column].values\n", + " \n", + " # Calculate local class balance in sliding windows\n", + " window_size = 100\n", + " balance_scores = []\n", + " \n", + " for i in range(len(y)):\n", + " start_idx = max(0, i - window_size//2)\n", + " end_idx = min(len(y), i + window_size//2)\n", + " window = y[start_idx:end_idx]\n", + " \n", + " # Calculate ratio of minority class\n", + " class_ratio = np.mean(window)\n", + " # Adjust to be symmetric around 0.5\n", + " balance_score = 1 - abs(0.5 - class_ratio) * 2\n", + " \n", + " balance_scores.append(balance_score)\n", + " \n", + " return np.array(balance_scores)\n", + "\n", + "# Assign the class balance scores to the dataset\n", + "vm_test_ds.assign_scores(metrics = \"my_scorers.TestScorer\", model = vm_xgb_model)\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Next steps\n", + "\n", + "You can explore the assigned scores right in the notebook as demonstrated above. However, there's even more value in using the ValidMind Platform to work with your model documentation and monitoring.\n", + "\n", + "\n", + "\n", + "### Work with your model documentation\n", + "\n", + "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "\n", + "2. Click and expand the **Model Development** section.\n", + "\n", + "The scores you've assigned using `assign_scores()` become part of your model's documentation and can be used in ongoing monitoring workflows. You can view these metrics over time, set up alerts for performance drift, and compare models systematically. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "\n", + "\n", + "\n", + "### Discover more learning resources\n", + "\n", + "We offer many interactive notebooks to help you work with model scoring and evaluation:\n", + "\n", + "- [Run unit metrics](https://docs.validmind.ai/developer/model-testing/testing-overview.html)\n", + "- [Assign predictions](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)\n", + "- [Model comparison workflows](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)\n", + "\n", + "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Upgrade ValidMind\n", + "\n", + "
After installing ValidMind, you'll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n", + "\n", + "Retrieve the information for the currently installed version of ValidMind:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "%pip show validmind" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n", + "\n", + "```bash\n", + "%pip install --upgrade validmind\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You may need to restart your kernel after running the upgrade package for changes to be applied." + ] + }, + { + "cell_type": "markdown", + "id": "c1b1dd04", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ValidMind Library", + "language": "python", + "name": "validmind" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/notebooks/how_to/explore_tests.ipynb b/notebooks/how_to/explore_tests.ipynb index 2076662b0..6ced61976 100644 --- a/notebooks/how_to/explore_tests.ipynb +++ b/notebooks/how_to/explore_tests.ipynb @@ -1,4423 +1,4426 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Explore tests\n", - "\n", - "Explore the individual out-the-box tests available in the ValidMind Library, and identify which tests to run to evaluate different aspects of your model. Browse available tests, view their descriptions, and filter by tags or task type to find tests relevant to your use case.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "::: {.content-hidden when-format=\"html\"}\n", - "## Contents \n", - "- [About ValidMind](#toc1_) \n", - " - [Before you begin](#toc1_1_) \n", - " - [New to ValidMind?](#toc1_2_) \n", - " - [Key concepts](#toc1_3_) \n", - "- [Install the ValidMind Library](#toc2_) \n", - "- [List all available tests](#toc3_) \n", - "- [Understand tags and task types](#toc4_) \n", - "- [Filter tests by tags and task types](#toc5_)\n", - "- [Store test sets for use](#toc6_) \n", - "- [Next steps](#toc7_) \n", - " - [Discover more learning resources](#toc7_1_) \n", - "\n", - ":::\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## About ValidMind\n", - "\n", - "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n", - "\n", - "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n", - "\n", - "\n", - "\n", - "### Before you begin\n", - "\n", - "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n", - "\n", - "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n", - "\n", - "\n", - "\n", - "\n", - "### New to ValidMind?\n", - "\n", - "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n", - "\n", - "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", - "

\n", - "Register with ValidMind
\n", - "\n", - "\n", - "\n", - "### Key concepts\n", - "\n", - "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n", - "\n", - "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", - "\n", - "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n", - "\n", - "**Custom tests**: Custom tests are functions that you define to evaluate your model or dataset. These functions can be registered via the ValidMind Library to be used with the ValidMind Platform.\n", - "\n", - "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n", - "\n", - " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n", - " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n", - " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom test.\n", - " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom test. See this [example](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html) for more information.\n", - "\n", - "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a test, customize its behavior, or provide additional context.\n", - "\n", - "**Outputs**: Custom tests can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n", - "\n", - "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n", - "\n", - "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Install the ValidMind Library\n", - "\n", - "
Recommended Python versions\n", - "

\n", - "Python 3.8 <= x <= 3.11
\n", - "\n", - "To install the library:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -q validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## List all available tests\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "Start by importing the functions from the [validmind.tests](https://docs.validmind.ai/validmind/validmind/tests.html) module for listing tests, listing tasks, listing tags, and listing tasks and tags to access these functions in the rest of this notebook:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from validmind.tests import (\n", - " list_tests,\n", - " list_tasks,\n", - " list_tags,\n", - " list_tasks_and_tags,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use [list_tests()](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to retrieve all available ValidMind tests, which returns a DataFrame with the following columns:\n", - "\n", - "- **ID** – A unique identifier for each test.\n", - "- **Name** – The test’s name.\n", - "- **Description** – A short summary of what the test evaluates.\n", - "- **Tags** – Keywords that describe what the test does or applies to.\n", - "- **Tasks** – The type of modeling task the test supports." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
IDNameDescriptionHas FigureHas TableRequired InputsParamsTagsTasks
validmind.data_validation.ACFandPACFPlotAC Fand PACF PlotAnalyzes time series data using Autocorrelation Function (ACF) and Partial Autocorrelation Function (PACF) plots to...TrueFalse['dataset']{}['time_series_data', 'forecasting', 'statistical_test', 'visualization']['regression']
validmind.data_validation.ADFADFAssesses the stationarity of a time series dataset using the Augmented Dickey-Fuller (ADF) test....FalseTrue['dataset']{}['time_series_data', 'statsmodels', 'forecasting', 'statistical_test', 'stationarity']['regression']
validmind.data_validation.AutoARAuto ARAutomatically identifies the optimal Autoregressive (AR) order for a time series using BIC and AIC criteria....FalseTrue['dataset']{'max_ar_order': {'type': 'int', 'default': 3}}['time_series_data', 'statsmodels', 'forecasting', 'statistical_test']['regression']
validmind.data_validation.AutoMAAuto MAAutomatically selects the optimal Moving Average (MA) order for each variable in a time series dataset based on...FalseTrue['dataset']{'max_ma_order': {'type': 'int', 'default': 3}}['time_series_data', 'statsmodels', 'forecasting', 'statistical_test']['regression']
validmind.data_validation.AutoStationarityAuto StationarityAutomates Augmented Dickey-Fuller test to assess stationarity across multiple time series in a DataFrame....FalseTrue['dataset']{'max_order': {'type': 'int', 'default': 5}, 'threshold': {'type': 'float', 'default': 0.05}}['time_series_data', 'statsmodels', 'forecasting', 'statistical_test']['regression']
validmind.data_validation.BivariateScatterPlotsBivariate Scatter PlotsGenerates bivariate scatterplots to visually inspect relationships between pairs of numerical predictor variables...TrueFalse['dataset']{}['tabular_data', 'numerical_data', 'visualization']['classification']
validmind.data_validation.BoxPierceBox PierceDetects autocorrelation in time-series data through the Box-Pierce test to validate model performance....FalseTrue['dataset']{}['time_series_data', 'forecasting', 'statistical_test', 'statsmodels']['regression']
validmind.data_validation.ChiSquaredFeaturesTableChi Squared Features TableAssesses the statistical association between categorical features and a target variable using the Chi-Squared test....FalseTrue['dataset']{'p_threshold': {'type': '_empty', 'default': 0.05}}['tabular_data', 'categorical_data', 'statistical_test']['classification']
validmind.data_validation.ClassImbalanceClass ImbalanceEvaluates and quantifies class distribution imbalance in a dataset used by a machine learning model....TrueTrue['dataset']{'min_percent_threshold': {'type': 'int', 'default': 10}}['tabular_data', 'binary_classification', 'multiclass_classification', 'data_quality']['classification']
validmind.data_validation.DatasetDescriptionDataset DescriptionProvides comprehensive analysis and statistical summaries of each column in a machine learning model's dataset....FalseTrue['dataset']{}['tabular_data', 'time_series_data', 'text_data']['classification', 'regression', 'text_classification', 'text_summarization']
validmind.data_validation.DatasetSplitDataset SplitEvaluates and visualizes the distribution proportions among training, testing, and validation datasets of an ML...FalseTrue['datasets']{}['tabular_data', 'time_series_data', 'text_data']['classification', 'regression', 'text_classification', 'text_summarization']
validmind.data_validation.DescriptiveStatisticsDescriptive StatisticsPerforms a detailed descriptive statistical analysis of both numerical and categorical data within a model's...FalseTrue['dataset']{}['tabular_data', 'time_series_data', 'data_quality']['classification', 'regression']
validmind.data_validation.DickeyFullerGLSDickey Fuller GLSAssesses stationarity in time series data using the Dickey-Fuller GLS test to determine the order of integration....FalseTrue['dataset']{}['time_series_data', 'forecasting', 'unit_root_test']['regression']
validmind.data_validation.DuplicatesDuplicatesTests dataset for duplicate entries, ensuring model reliability via data quality verification....FalseTrue['dataset']{'min_threshold': {'type': '_empty', 'default': 1}}['tabular_data', 'data_quality', 'text_data']['classification', 'regression']
validmind.data_validation.EngleGrangerCointEngle Granger CointAssesses the degree of co-movement between pairs of time series data using the Engle-Granger cointegration test....FalseTrue['dataset']{'threshold': {'type': 'float', 'default': 0.05}}['time_series_data', 'statistical_test', 'forecasting']['regression']
validmind.data_validation.FeatureTargetCorrelationPlotFeature Target Correlation PlotVisualizes the correlation between input features and the model's target output in a color-coded horizontal bar...TrueFalse['dataset']{'fig_height': {'type': '_empty', 'default': 600}}['tabular_data', 'visualization', 'correlation']['classification', 'regression']
validmind.data_validation.HighCardinalityHigh CardinalityAssesses the number of unique values in categorical columns to detect high cardinality and potential overfitting....FalseTrue['dataset']{'num_threshold': {'type': 'int', 'default': 100}, 'percent_threshold': {'type': 'float', 'default': 0.1}, 'threshold_type': {'type': 'str', 'default': 'percent'}}['tabular_data', 'data_quality', 'categorical_data']['classification', 'regression']
validmind.data_validation.HighPearsonCorrelationHigh Pearson CorrelationIdentifies highly correlated feature pairs in a dataset suggesting feature redundancy or multicollinearity....FalseTrue['dataset']{'max_threshold': {'type': 'float', 'default': 0.3}, 'top_n_correlations': {'type': 'int', 'default': 10}, 'feature_columns': {'type': 'list', 'default': None}}['tabular_data', 'data_quality', 'correlation']['classification', 'regression']
validmind.data_validation.IQROutliersBarPlotIQR Outliers Bar PlotVisualizes outlier distribution across percentiles in numerical data using the Interquartile Range (IQR) method....TrueFalse['dataset']{'threshold': {'type': 'float', 'default': 1.5}, 'fig_width': {'type': 'int', 'default': 800}}['tabular_data', 'visualization', 'numerical_data']['classification', 'regression']
validmind.data_validation.IQROutliersTableIQR Outliers TableDetermines and summarizes outliers in numerical features using the Interquartile Range method....FalseTrue['dataset']{'threshold': {'type': 'float', 'default': 1.5}}['tabular_data', 'numerical_data']['classification', 'regression']
validmind.data_validation.IsolationForestOutliersIsolation Forest OutliersDetects outliers in a dataset using the Isolation Forest algorithm and visualizes results through scatter plots....TrueFalse['dataset']{'random_state': {'type': 'int', 'default': 0}, 'contamination': {'type': 'float', 'default': 0.1}, 'feature_columns': {'type': 'list', 'default': None}}['tabular_data', 'anomaly_detection']['classification']
validmind.data_validation.JarqueBeraJarque BeraAssesses normality of dataset features in an ML model using the Jarque-Bera test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.data_validation.KPSSKPSSAssesses the stationarity of time-series data in a machine learning model using the KPSS unit root test....FalseTrue['dataset']{}['time_series_data', 'stationarity', 'unit_root_test', 'statsmodels']['data_validation']
validmind.data_validation.LJungBoxL Jung BoxAssesses autocorrelations in dataset features by performing a Ljung-Box test on each feature....FalseTrue['dataset']{}['time_series_data', 'forecasting', 'statistical_test', 'statsmodels']['regression']
validmind.data_validation.LaggedCorrelationHeatmapLagged Correlation HeatmapAssesses and visualizes correlation between target variable and lagged independent variables in a time-series...TrueFalse['dataset']{'num_lags': {'type': 'int', 'default': 10}}['time_series_data', 'visualization']['regression']
validmind.data_validation.MissingValuesMissing ValuesEvaluates dataset quality by ensuring missing value ratio across all features does not exceed a set threshold....FalseTrue['dataset']{'min_threshold': {'type': 'int', 'default': 1}}['tabular_data', 'data_quality']['classification', 'regression']
validmind.data_validation.MissingValuesBarPlotMissing Values Bar PlotAssesses the percentage and distribution of missing values in the dataset via a bar plot, with emphasis on...TrueFalse['dataset']{'threshold': {'type': 'int', 'default': 80}, 'fig_height': {'type': 'int', 'default': 600}}['tabular_data', 'data_quality', 'visualization']['classification', 'regression']
validmind.data_validation.MutualInformationMutual InformationCalculates mutual information scores between features and target variable to evaluate feature relevance....TrueFalse['dataset']{'min_threshold': {'type': 'float', 'default': 0.01}, 'task': {'type': 'str', 'default': 'classification'}}['feature_selection', 'data_analysis']['classification', 'regression']
validmind.data_validation.PearsonCorrelationMatrixPearson Correlation MatrixEvaluates linear dependency between numerical variables in a dataset via a Pearson Correlation coefficient heat map....TrueFalse['dataset']{}['tabular_data', 'numerical_data', 'correlation']['classification', 'regression']
validmind.data_validation.PhillipsPerronArchPhillips Perron ArchAssesses the stationarity of time series data in each feature of the ML model using the Phillips-Perron test....FalseTrue['dataset']{}['time_series_data', 'forecasting', 'statistical_test', 'unit_root_test']['regression']
validmind.data_validation.ProtectedClassesDescriptionProtected Classes DescriptionVisualizes the distribution of protected classes in the dataset relative to the target variable...TrueTrue['dataset']{'protected_classes': {'type': '_empty', 'default': None}}['bias_and_fairness', 'descriptive_statistics']['classification', 'regression']
validmind.data_validation.RollingStatsPlotRolling Stats PlotEvaluates the stationarity of time series data by plotting its rolling mean and standard deviation over a specified...TrueFalse['dataset']{'window_size': {'type': 'int', 'default': 12}}['time_series_data', 'visualization', 'stationarity']['regression']
validmind.data_validation.RunsTestRuns TestExecutes Runs Test on ML model to detect non-random patterns in output data sequence....FalseTrue['dataset']{}['tabular_data', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.data_validation.ScatterPlotScatter PlotAssesses visual relationships, patterns, and outliers among features in a dataset through scatter plot matrices....TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.ScoreBandDefaultRatesScore Band Default RatesAnalyzes default rates and population distribution across credit score bands....FalseTrue['dataset', 'model']{'score_column': {'type': 'str', 'default': 'score'}, 'score_bands': {'type': 'list', 'default': None}}['visualization', 'credit_risk', 'scorecard']['classification']
validmind.data_validation.SeasonalDecomposeSeasonal DecomposeAssesses patterns and seasonality in a time series dataset by decomposing its features into foundational components....TrueFalse['dataset']{'seasonal_model': {'type': 'str', 'default': 'additive'}}['time_series_data', 'seasonality', 'statsmodels']['regression']
validmind.data_validation.ShapiroWilkShapiro WilkEvaluates feature-wise normality of training data using the Shapiro-Wilk test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test']['classification', 'regression']
validmind.data_validation.SkewnessSkewnessEvaluates the skewness of numerical data in a dataset to check against a defined threshold, aiming to ensure data...FalseTrue['dataset']{'max_threshold': {'type': '_empty', 'default': 1}}['data_quality', 'tabular_data']['classification', 'regression']
validmind.data_validation.SpreadPlotSpread PlotAssesses potential correlations between pairs of time series variables through visualization to enhance...TrueFalse['dataset']{}['time_series_data', 'visualization']['regression']
validmind.data_validation.TabularCategoricalBarPlotsTabular Categorical Bar PlotsGenerates and visualizes bar plots for each category in categorical features to evaluate the dataset's composition....TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.TabularDateTimeHistogramsTabular Date Time HistogramsGenerates histograms to provide graphical insight into the distribution of time intervals in a model's datetime...TrueFalse['dataset']{}['time_series_data', 'visualization']['classification', 'regression']
validmind.data_validation.TabularDescriptionTablesTabular Description TablesSummarizes key descriptive statistics for numerical, categorical, and datetime variables in a dataset....FalseTrue['dataset']{}['tabular_data']['classification', 'regression']
validmind.data_validation.TabularNumericalHistogramsTabular Numerical HistogramsGenerates histograms for each numerical feature in a dataset to provide visual insights into data distribution and...TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.TargetRateBarPlotsTarget Rate Bar PlotsGenerates bar plots visualizing the default rates of categorical features for a classification machine learning...TrueFalse['dataset']{}['tabular_data', 'visualization', 'categorical_data']['classification']
validmind.data_validation.TimeSeriesDescriptionTime Series DescriptionGenerates a detailed analysis for the provided time series dataset, summarizing key statistics to identify trends,...FalseTrue['dataset']{}['time_series_data', 'analysis']['regression']
validmind.data_validation.TimeSeriesDescriptiveStatisticsTime Series Descriptive StatisticsEvaluates the descriptive statistics of a time series dataset to identify trends, patterns, and data quality issues....FalseTrue['dataset']{}['time_series_data', 'analysis']['regression']
validmind.data_validation.TimeSeriesFrequencyTime Series FrequencyEvaluates consistency of time series data frequency and generates a frequency plot....TrueTrue['dataset']{}['time_series_data']['regression']
validmind.data_validation.TimeSeriesHistogramTime Series HistogramVisualizes distribution of time-series data using histograms and Kernel Density Estimation (KDE) lines....TrueFalse['dataset']{'nbins': {'type': '_empty', 'default': 30}}['data_validation', 'visualization', 'time_series_data']['regression', 'time_series_forecasting']
validmind.data_validation.TimeSeriesLinePlotTime Series Line PlotGenerates and analyses time-series data through line plots revealing trends, patterns, anomalies over time....TrueFalse['dataset']{}['time_series_data', 'visualization']['regression']
validmind.data_validation.TimeSeriesMissingValuesTime Series Missing ValuesValidates time-series data quality by confirming the count of missing values is below a certain threshold....TrueTrue['dataset']{'min_threshold': {'type': 'int', 'default': 1}}['time_series_data']['regression']
validmind.data_validation.TimeSeriesOutliersTime Series OutliersIdentifies and visualizes outliers in time-series data using the z-score method....FalseTrue['dataset']{'zscore_threshold': {'type': 'int', 'default': 3}}['time_series_data']['regression']
validmind.data_validation.TooManyZeroValuesToo Many Zero ValuesIdentifies numerical columns in a dataset that contain an excessive number of zero values, defined by a threshold...FalseTrue['dataset']{'max_percent_threshold': {'type': 'float', 'default': 0.03}}['tabular_data']['regression', 'classification']
validmind.data_validation.UniqueRowsUnique RowsVerifies the diversity of the dataset by ensuring that the count of unique rows exceeds a prescribed threshold....FalseTrue['dataset']{'min_percent_threshold': {'type': 'float', 'default': 1}}['tabular_data']['regression', 'classification']
validmind.data_validation.WOEBinPlotsWOE Bin PlotsGenerates visualizations of Weight of Evidence (WoE) and Information Value (IV) for understanding predictive power...TrueFalse['dataset']{'breaks_adj': {'type': 'list', 'default': None}, 'fig_height': {'type': 'int', 'default': 600}, 'fig_width': {'type': 'int', 'default': 500}}['tabular_data', 'visualization', 'categorical_data']['classification']
validmind.data_validation.WOEBinTableWOE Bin TableAssesses the Weight of Evidence (WoE) and Information Value (IV) of each feature to evaluate its predictive power...FalseTrue['dataset']{'breaks_adj': {'type': 'list', 'default': None}}['tabular_data', 'categorical_data']['classification']
validmind.data_validation.ZivotAndrewsArchZivot Andrews ArchEvaluates the order of integration and stationarity of time series data using the Zivot-Andrews unit root test....FalseTrue['dataset']{}['time_series_data', 'stationarity', 'unit_root_test']['regression']
validmind.data_validation.nlp.CommonWordsCommon WordsAssesses the most frequent non-stopwords in a text column for identifying prevalent language patterns....TrueFalse['dataset']{}['nlp', 'text_data', 'visualization', 'frequency_analysis']['text_classification', 'text_summarization']
validmind.data_validation.nlp.HashtagsHashtagsAssesses hashtag frequency in a text column, highlighting usage trends and potential dataset bias or spam....TrueFalse['dataset']{'top_hashtags': {'type': 'int', 'default': 25}}['nlp', 'text_data', 'visualization', 'frequency_analysis']['text_classification', 'text_summarization']
validmind.data_validation.nlp.LanguageDetectionLanguage DetectionAssesses the diversity of languages in a textual dataset by detecting and visualizing the distribution of languages....TrueFalse['dataset']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.data_validation.nlp.MentionsMentionsCalculates and visualizes frequencies of '@' prefixed mentions in a text-based dataset for NLP model analysis....TrueFalse['dataset']{'top_mentions': {'type': 'int', 'default': 25}}['nlp', 'text_data', 'visualization', 'frequency_analysis']['text_classification', 'text_summarization']
validmind.data_validation.nlp.PolarityAndSubjectivityPolarity And SubjectivityAnalyzes the polarity and subjectivity of text data within a given dataset to visualize the sentiment distribution....TrueTrue['dataset']{'threshold_subjectivity': {'type': '_empty', 'default': 0.5}, 'threshold_polarity': {'type': '_empty', 'default': 0}}['nlp', 'text_data', 'data_validation']['nlp']
validmind.data_validation.nlp.PunctuationsPunctuationsAnalyzes and visualizes the frequency distribution of punctuation usage in a given text dataset....TrueFalse['dataset']{'count_mode': {'type': '_empty', 'default': 'token'}}['nlp', 'text_data', 'visualization', 'frequency_analysis']['text_classification', 'text_summarization', 'nlp']
validmind.data_validation.nlp.SentimentSentimentAnalyzes the sentiment of text data within a dataset using the VADER sentiment analysis tool....TrueFalse['dataset']{}['nlp', 'text_data', 'data_validation']['nlp']
validmind.data_validation.nlp.StopWordsStop WordsEvaluates and visualizes the frequency of English stop words in a text dataset against a defined threshold....TrueTrue['dataset']{'min_percent_threshold': {'type': 'float', 'default': 0.5}, 'num_words': {'type': 'int', 'default': 25}}['nlp', 'text_data', 'frequency_analysis', 'visualization']['text_classification', 'text_summarization']
validmind.data_validation.nlp.TextDescriptionText DescriptionConducts comprehensive textual analysis on a dataset using NLTK to evaluate various parameters and generate...TrueFalse['dataset']{'unwanted_tokens': {'type': 'set', 'default': {'s', 'mrs', 'us', \"''\", ' ', 'ms', 'dr', 'dollar', '``', 'mr', \"'s\", \"s'\"}}, 'lang': {'type': 'str', 'default': 'english'}}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.data_validation.nlp.ToxicityToxicityAssesses the toxicity of text data within a dataset to visualize the distribution of toxicity scores....TrueFalse['dataset']{}['nlp', 'text_data', 'data_validation']['nlp']
validmind.model_validation.BertScoreBert ScoreAssesses the quality of machine-generated text using BERTScore metrics and visualizes results through histograms...TrueTrue['dataset', 'model']{'evaluation_model': {'type': '_empty', 'default': 'distilbert-base-uncased'}}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.BleuScoreBleu ScoreEvaluates the quality of machine-generated text using BLEU metrics and visualizes the results through histograms...TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.ClusterSizeDistributionCluster Size DistributionAssesses the performance of clustering models by comparing the distribution of cluster sizes in model predictions...TrueFalse['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.ContextualRecallContextual RecallEvaluates a Natural Language Generation model's ability to generate contextually relevant and factually correct...TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.FeaturesAUCFeatures AUCEvaluates the discriminatory power of each individual feature within a binary classification model by calculating...TrueFalse['dataset']{'fontsize': {'type': 'int', 'default': 12}, 'figure_height': {'type': 'int', 'default': 500}}['feature_importance', 'AUC', 'visualization']['classification']
validmind.model_validation.MeteorScoreMeteor ScoreAssesses the quality of machine-generated translations by comparing them to human-produced references using the...TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.ModelMetadataModel MetadataCompare metadata of different models and generate a summary table with the results....FalseTrue['model']{}['model_training', 'metadata']['regression', 'time_series_forecasting']
validmind.model_validation.ModelPredictionResidualsModel Prediction ResidualsAssesses normality and behavior of residuals in regression models through visualization and statistical tests....TrueTrue['dataset', 'model']{'nbins': {'type': 'int', 'default': 100}, 'p_value_threshold': {'type': 'float', 'default': 0.05}, 'start_date': {'type': None, 'default': None}, 'end_date': {'type': None, 'default': None}}['regression']['residual_analysis', 'visualization']
validmind.model_validation.RegardScoreRegard ScoreAssesses the sentiment and potential biases in text generated by NLP models by computing and visualizing regard...TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.RegressionResidualsPlotRegression Residuals PlotEvaluates regression model performance using residual distribution and actual vs. predicted plots....TrueFalse['model', 'dataset']{'bin_size': {'type': 'float', 'default': 0.1}}['model_performance', 'visualization']['regression']
validmind.model_validation.RougeScoreRouge ScoreAssesses the quality of machine-generated text using ROUGE metrics and visualizes the results to provide...TrueTrue['dataset', 'model']{'metric': {'type': 'str', 'default': 'rouge-1'}}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.TimeSeriesPredictionWithCITime Series Prediction With CIAssesses predictive accuracy and uncertainty in time series models, highlighting breaches beyond confidence...TrueTrue['dataset', 'model']{'confidence': {'type': 'float', 'default': 0.95}}['model_predictions', 'visualization']['regression', 'time_series_forecasting']
validmind.model_validation.TimeSeriesPredictionsPlotTime Series Predictions PlotPlot actual vs predicted values for time series data and generate a visual comparison for the model....TrueFalse['dataset', 'model']{}['model_predictions', 'visualization']['regression', 'time_series_forecasting']
validmind.model_validation.TimeSeriesR2SquareBySegmentsTime Series R2 Square By SegmentsEvaluates the R-Squared values of regression models over specified time segments in time series data to assess...TrueTrue['dataset', 'model']{'segments': {'type': None, 'default': None}}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.TokenDisparityToken DisparityEvaluates the token disparity between reference and generated texts, visualizing the results through histograms and...TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.ToxicityScoreToxicity ScoreAssesses the toxicity levels of texts generated by NLP models to identify and mitigate harmful or offensive content....TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.embeddings.ClusterDistributionCluster DistributionAssesses the distribution of text embeddings across clusters produced by a model using KMeans clustering....TrueFalse['model', 'dataset']{'num_clusters': {'type': 'int', 'default': 5}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.CosineSimilarityComparisonCosine Similarity ComparisonAssesses the similarity between embeddings generated by different models using Cosine Similarity, providing both...TrueTrue['dataset', 'models']{}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.embeddings.CosineSimilarityDistributionCosine Similarity DistributionAssesses the similarity between predicted text embeddings from a model using a Cosine Similarity distribution...TrueFalse['dataset', 'model']{}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.CosineSimilarityHeatmapCosine Similarity HeatmapGenerates an interactive heatmap to visualize the cosine similarities among embeddings derived from a given model....TrueFalse['dataset', 'model']{'title': {'type': '_empty', 'default': 'Cosine Similarity Matrix'}, 'color': {'type': '_empty', 'default': 'Cosine Similarity'}, 'xaxis_title': {'type': '_empty', 'default': 'Index'}, 'yaxis_title': {'type': '_empty', 'default': 'Index'}, 'color_scale': {'type': '_empty', 'default': 'Blues'}}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.embeddings.DescriptiveAnalyticsDescriptive AnalyticsEvaluates statistical properties of text embeddings in an ML model via mean, median, and standard deviation...TrueFalse['dataset', 'model']{}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.EmbeddingsVisualization2DEmbeddings Visualization2 DVisualizes 2D representation of text embeddings generated by a model using t-SNE technique....TrueFalse['dataset', 'model']{'cluster_column': {'type': None, 'default': None}, 'perplexity': {'type': 'int', 'default': 30}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.EuclideanDistanceComparisonEuclidean Distance ComparisonAssesses and visualizes the dissimilarity between model embeddings using Euclidean distance, providing insights...TrueTrue['dataset', 'models']{}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.embeddings.EuclideanDistanceHeatmapEuclidean Distance HeatmapGenerates an interactive heatmap to visualize the Euclidean distances among embeddings derived from a given model....TrueFalse['dataset', 'model']{'title': {'type': '_empty', 'default': 'Euclidean Distance Matrix'}, 'color': {'type': '_empty', 'default': 'Euclidean Distance'}, 'xaxis_title': {'type': '_empty', 'default': 'Index'}, 'yaxis_title': {'type': '_empty', 'default': 'Index'}, 'color_scale': {'type': '_empty', 'default': 'Blues'}}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.embeddings.PCAComponentsPairwisePlotsPCA Components Pairwise PlotsGenerates scatter plots for pairwise combinations of principal component analysis (PCA) components of model...TrueFalse['dataset', 'model']{'n_components': {'type': 'int', 'default': 3}}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.embeddings.StabilityAnalysisKeywordStability Analysis KeywordEvaluates robustness of embedding models to keyword swaps in the test dataset....TrueTrue['dataset', 'model']{'keyword_dict': {'type': None, 'default': None}, 'mean_similarity_threshold': {'type': 'float', 'default': 0.7}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.StabilityAnalysisRandomNoiseStability Analysis Random NoiseAssesses the robustness of text embeddings models to random noise introduced via text perturbations....TrueTrue['dataset', 'model']{'probability': {'type': 'float', 'default': 0.02}, 'mean_similarity_threshold': {'type': 'float', 'default': 0.7}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.StabilityAnalysisSynonymsStability Analysis SynonymsEvaluates the stability of text embeddings models when words in test data are replaced by their synonyms randomly....TrueTrue['dataset', 'model']{'probability': {'type': 'float', 'default': 0.02}, 'mean_similarity_threshold': {'type': 'float', 'default': 0.7}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.StabilityAnalysisTranslationStability Analysis TranslationEvaluates robustness of text embeddings models to noise introduced by translating the original text to another...TrueTrue['dataset', 'model']{'source_lang': {'type': 'str', 'default': 'en'}, 'target_lang': {'type': 'str', 'default': 'fr'}, 'mean_similarity_threshold': {'type': 'float', 'default': 0.7}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.TSNEComponentsPairwisePlotsTSNE Components Pairwise PlotsCreates scatter plots for pairwise combinations of t-SNE components to visualize embeddings and highlight potential...TrueFalse['dataset', 'model']{'n_components': {'type': 'int', 'default': 2}, 'perplexity': {'type': 'int', 'default': 30}, 'title': {'type': 'str', 'default': 't-SNE'}}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.AnswerCorrectnessAnswer CorrectnessEvaluates the correctness of answers in a dataset with respect to the provided ground...TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'response_column': {'type': 'str', 'default': 'response'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.AspectCriticAspect CriticEvaluates generations against the following aspects: harmfulness, maliciousness,...TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'response_column': {'type': 'str', 'default': 'response'}, 'retrieved_contexts_column': {'type': None, 'default': None}, 'aspects': {'type': None, 'default': ['coherence', 'conciseness', 'correctness', 'harmfulness', 'maliciousness']}, 'additional_aspects': {'type': None, 'default': None}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'qualitative']['text_summarization', 'text_generation', 'text_qa']
validmind.model_validation.ragas.ContextEntityRecallContext Entity RecallEvaluates the context entity recall for dataset entries and visualizes the results....TrueTrue['dataset']{'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'retrieval_performance']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.ContextPrecisionContext PrecisionContext Precision is a metric that evaluates whether all of the ground-truth...TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'retrieval_performance']['text_qa', 'text_generation', 'text_summarization', 'text_classification']
validmind.model_validation.ragas.ContextPrecisionWithoutReferenceContext Precision Without ReferenceContext Precision Without Reference is a metric used to evaluate the relevance of...TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'response_column': {'type': 'str', 'default': 'response'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'retrieval_performance']['text_qa', 'text_generation', 'text_summarization', 'text_classification']
validmind.model_validation.ragas.ContextRecallContext RecallContext recall measures the extent to which the retrieved context aligns with the...TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'retrieval_performance']['text_qa', 'text_generation', 'text_summarization', 'text_classification']
validmind.model_validation.ragas.FaithfulnessFaithfulnessEvaluates the faithfulness of the generated answers with respect to retrieved contexts....TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'response_column': {'type': 'str', 'default': 'response'}, 'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'rag_performance']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.NoiseSensitivityNoise SensitivityAssesses the sensitivity of a Large Language Model (LLM) to noise in retrieved context by measuring how often it...TrueTrue['dataset']{'response_column': {'type': 'str', 'default': 'response'}, 'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'focus': {'type': 'str', 'default': 'relevant'}, 'user_input_column': {'type': 'str', 'default': 'user_input'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'rag_performance']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.ResponseRelevancyResponse RelevancyAssesses how pertinent the generated answer is to the given prompt....TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'retrieved_contexts_column': {'type': 'str', 'default': None}, 'response_column': {'type': 'str', 'default': 'response'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'rag_performance']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.SemanticSimilaritySemantic SimilarityCalculates the semantic similarity between generated responses and ground truths...TrueTrue['dataset']{'response_column': {'type': 'str', 'default': 'response'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.sklearn.AdjustedMutualInformationAdjusted Mutual InformationEvaluates clustering model performance by measuring mutual information between true and predicted labels, adjusting...FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.AdjustedRandIndexAdjusted Rand IndexMeasures the similarity between two data clusters using the Adjusted Rand Index (ARI) metric in clustering machine...FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.CalibrationCurveCalibration CurveEvaluates the calibration of probability estimates by comparing predicted probabilities against observed...TrueFalse['model', 'dataset']{'n_bins': {'type': 'int', 'default': 10}}['sklearn', 'model_performance', 'classification']['classification']
validmind.model_validation.sklearn.ClassifierPerformanceClassifier PerformanceEvaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy,...FalseTrue['dataset', 'model']{'average': {'type': 'str', 'default': 'macro'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ClassifierThresholdOptimizationClassifier Threshold OptimizationAnalyzes and visualizes different threshold optimization methods for binary classification models....FalseTrue['dataset', 'model']{'methods': {'type': None, 'default': None}, 'target_recall': {'type': None, 'default': None}}['model_validation', 'threshold_optimization', 'classification_metrics']['classification']
validmind.model_validation.sklearn.ClusterCosineSimilarityCluster Cosine SimilarityMeasures the intra-cluster similarity of a clustering model using cosine similarity....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.ClusterPerformanceMetricsCluster Performance MetricsEvaluates the performance of clustering machine learning models using multiple established metrics....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.CompletenessScoreCompleteness ScoreEvaluates a clustering model's capacity to categorize instances from a single class into the same cluster....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.ConfusionMatrixConfusion MatrixEvaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix...TrueFalse['dataset', 'model']{'threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.FeatureImportanceFeature ImportanceCompute feature importance scores for a given model and generate a summary table...FalseTrue['dataset', 'model']{'num_features': {'type': 'int', 'default': 3}}['model_explainability', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.FowlkesMallowsScoreFowlkes Mallows ScoreEvaluates the similarity between predicted and actual cluster assignments in a model using the Fowlkes-Mallows...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.HomogeneityScoreHomogeneity ScoreAssesses clustering homogeneity by comparing true and predicted labels, scoring from 0 (heterogeneous) to 1...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.HyperParametersTuningHyper Parameters TuningPerforms exhaustive grid search over specified parameter ranges to find optimal model configurations...FalseTrue['model', 'dataset']{'param_grid': {'type': 'dict', 'default': None}, 'scoring': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}, 'fit_params': {'type': 'dict', 'default': None}}['sklearn', 'model_performance']['clustering', 'classification']
validmind.model_validation.sklearn.KMeansClustersOptimizationK Means Clusters OptimizationOptimizes the number of clusters in K-means models using Elbow and Silhouette methods....TrueFalse['model', 'dataset']{'n_clusters': {'type': None, 'default': None}}['sklearn', 'model_performance', 'kmeans']['clustering']
validmind.model_validation.sklearn.MinimumAccuracyMinimum AccuracyChecks if the model's prediction accuracy meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.7}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumF1ScoreMinimum F1 ScoreAssesses if the model's F1 score on the validation set meets a predefined minimum threshold, ensuring balanced...FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumROCAUCScoreMinimum ROCAUC ScoreValidates model by checking if the ROC AUC score meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ModelParametersModel ParametersExtracts and displays model parameters in a structured format for transparency and reproducibility....FalseTrue['model']{'model_params': {'type': None, 'default': None}}['model_training', 'metadata']['classification', 'regression']
validmind.model_validation.sklearn.ModelsPerformanceComparisonModels Performance ComparisonEvaluates and compares the performance of multiple Machine Learning models using various metrics like accuracy,...FalseTrue['dataset', 'models']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'model_comparison']['classification', 'text_classification']
validmind.model_validation.sklearn.OverfitDiagnosisOverfit DiagnosisAssesses potential overfitting in a model's predictions, identifying regions where performance between training and...TrueTrue['model', 'datasets']{'metric': {'type': 'str', 'default': None}, 'cut_off_threshold': {'type': 'float', 'default': 0.04}}['sklearn', 'binary_classification', 'multiclass_classification', 'linear_regression', 'model_diagnosis']['classification', 'regression']
validmind.model_validation.sklearn.PermutationFeatureImportancePermutation Feature ImportanceAssesses the significance of each feature in a model by evaluating the impact on model performance when feature...TrueFalse['model', 'dataset']{'fontsize': {'type': None, 'default': None}, 'figure_height': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.PopulationStabilityIndexPopulation Stability IndexAssesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across...TrueTrue['datasets', 'model']{'num_bins': {'type': 'int', 'default': 10}, 'mode': {'type': 'str', 'default': 'fixed'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.PrecisionRecallCurvePrecision Recall CurveEvaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve....TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ROCCurveROC CurveEvaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic...TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.RegressionErrorsRegression ErrorsAssesses the performance and error distribution of a regression model using various error metrics....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance']['regression', 'classification']
validmind.model_validation.sklearn.RegressionErrorsComparisonRegression Errors ComparisonAssesses multiple regression error metrics to compare model performance across different datasets, emphasizing...FalseTrue['datasets', 'models']{}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.RegressionPerformanceRegression PerformanceEvaluates the performance of a regression model using five different metrics: MAE, MSE, RMSE, MAPE, and MBD....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance']['regression']
validmind.model_validation.sklearn.RegressionR2SquareRegression R2 SquareAssesses the overall goodness-of-fit of a regression model by evaluating R-squared (R2) and Adjusted R-squared (Adj...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['regression']
validmind.model_validation.sklearn.RegressionR2SquareComparisonRegression R2 Square ComparisonCompares R-Squared and Adjusted R-Squared values for different regression models across multiple datasets to assess...FalseTrue['datasets', 'models']{}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.RobustnessDiagnosisRobustness DiagnosisAssesses the robustness of a machine learning model by evaluating performance decay under noisy conditions....TrueTrue['datasets', 'model']{'metric': {'type': 'str', 'default': None}, 'scaling_factor_std_dev_list': {'type': None, 'default': [0.1, 0.2, 0.3, 0.4, 0.5]}, 'performance_decay_threshold': {'type': 'float', 'default': 0.05}}['sklearn', 'model_diagnosis', 'visualization']['classification', 'regression']
validmind.model_validation.sklearn.SHAPGlobalImportanceSHAP Global ImportanceEvaluates and visualizes global feature importance using SHAP values for model explanation and risk identification....FalseTrue['model', 'dataset']{'kernel_explainer_samples': {'type': 'int', 'default': 10}, 'tree_or_linear_explainer_samples': {'type': 'int', 'default': 200}, 'class_of_interest': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ScoreProbabilityAlignmentScore Probability AlignmentAnalyzes the alignment between credit scores and predicted probabilities....TrueTrue['model', 'dataset']{'score_column': {'type': 'str', 'default': 'score'}, 'n_bins': {'type': 'int', 'default': 10}}['visualization', 'credit_risk', 'calibration']['classification']
validmind.model_validation.sklearn.SilhouettePlotSilhouette PlotCalculates and visualizes Silhouette Score, assessing the degree of data point suitability to its cluster in ML...TrueTrue['model', 'dataset']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.TrainingTestDegradationTraining Test DegradationTests if model performance degradation between training and test datasets exceeds a predefined threshold....FalseTrue['datasets', 'model']{'max_threshold': {'type': 'float', 'default': 0.1}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.VMeasureV MeasureEvaluates homogeneity and completeness of a clustering model using the V Measure Score....FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.WeakspotsDiagnosisWeakspots DiagnosisIdentifies and visualizes weak spots in a machine learning model's performance across various sections of the...TrueTrue['datasets', 'model']{'features_columns': {'type': None, 'default': None}, 'metrics': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_diagnosis', 'visualization']['classification', 'text_classification']
validmind.model_validation.statsmodels.AutoARIMAAuto ARIMAEvaluates ARIMA models for time-series forecasting, ranking them using Bayesian and Akaike Information Criteria....FalseTrue['model', 'dataset']{}['time_series_data', 'forecasting', 'model_selection', 'statsmodels']['regression']
validmind.model_validation.statsmodels.CumulativePredictionProbabilitiesCumulative Prediction ProbabilitiesVisualizes cumulative probabilities of positive and negative classes for both training and testing in classification models....TrueFalse['dataset', 'model']{'title': {'type': 'str', 'default': 'Cumulative Probabilities'}}['visualization', 'credit_risk']['classification']
validmind.model_validation.statsmodels.DurbinWatsonTestDurbin Watson TestAssesses autocorrelation in time series data features using the Durbin-Watson statistic....FalseTrue['dataset', 'model']{'threshold': {'type': None, 'default': [1.5, 2.5]}}['time_series_data', 'forecasting', 'statistical_test', 'statsmodels']['regression']
validmind.model_validation.statsmodels.GINITableGINI TableEvaluates classification model performance using AUC, GINI, and KS metrics for training and test datasets....FalseTrue['dataset', 'model']{}['model_performance']['classification']
validmind.model_validation.statsmodels.KolmogorovSmirnovKolmogorov SmirnovAssesses whether each feature in the dataset aligns with a normal distribution using the Kolmogorov-Smirnov test....FalseTrue['model', 'dataset']{'dist': {'type': 'str', 'default': 'norm'}}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.model_validation.statsmodels.LillieforsLillieforsAssesses the normality of feature distributions in an ML model's training dataset using the Lilliefors test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.model_validation.statsmodels.PredictionProbabilitiesHistogramPrediction Probabilities HistogramAssesses the predictive probability distribution for binary classification to evaluate model performance and...TrueFalse['dataset', 'model']{'title': {'type': 'str', 'default': 'Histogram of Predictive Probabilities'}}['visualization', 'credit_risk']['classification']
validmind.model_validation.statsmodels.RegressionCoeffsRegression CoeffsAssesses the significance and uncertainty of predictor variables in a regression model through visualization of...TrueTrue['model']{}['tabular_data', 'visualization', 'model_training']['regression']
validmind.model_validation.statsmodels.RegressionFeatureSignificanceRegression Feature SignificanceAssesses and visualizes the statistical significance of features in a regression model....TrueFalse['model']{'fontsize': {'type': 'int', 'default': 10}, 'p_threshold': {'type': 'float', 'default': 0.05}}['statistical_test', 'model_interpretation', 'visualization', 'feature_importance']['regression']
validmind.model_validation.statsmodels.RegressionModelForecastPlotRegression Model Forecast PlotGenerates plots to visually compare the forecasted outcomes of a regression model against actual observed values over...TrueFalse['model', 'dataset']{'start_date': {'type': None, 'default': None}, 'end_date': {'type': None, 'default': None}}['time_series_data', 'forecasting', 'visualization']['regression']
validmind.model_validation.statsmodels.RegressionModelForecastPlotLevelsRegression Model Forecast Plot LevelsAssesses the alignment between forecasted and observed values in regression models through visual plots...TrueFalse['model', 'dataset']{}['time_series_data', 'forecasting', 'visualization']['regression']
validmind.model_validation.statsmodels.RegressionModelSensitivityPlotRegression Model Sensitivity PlotAssesses the sensitivity of a regression model to changes in independent variables by applying shocks and...TrueFalse['dataset', 'model']{'shocks': {'type': None, 'default': [0.1]}, 'transformation': {'type': None, 'default': None}}['senstivity_analysis', 'visualization']['regression']
validmind.model_validation.statsmodels.RegressionModelSummaryRegression Model SummaryEvaluates regression model performance using metrics including R-Squared, Adjusted R-Squared, MSE, and RMSE....FalseTrue['dataset', 'model']{}['model_performance', 'regression']['regression']
validmind.model_validation.statsmodels.RegressionPermutationFeatureImportanceRegression Permutation Feature ImportanceAssesses the significance of each feature in a model by evaluating the impact on model performance when feature...TrueFalse['dataset', 'model']{'fontsize': {'type': 'int', 'default': 12}, 'figure_height': {'type': 'int', 'default': 500}}['statsmodels', 'feature_importance', 'visualization']['regression']
validmind.model_validation.statsmodels.ScorecardHistogramScorecard HistogramThe Scorecard Histogram test evaluates the distribution of credit scores between default and non-default instances,...TrueFalse['dataset']{'title': {'type': 'str', 'default': 'Histogram of Scores'}, 'score_column': {'type': 'str', 'default': 'score'}}['visualization', 'credit_risk', 'logistic_regression']['classification']
validmind.ongoing_monitoring.CalibrationCurveDriftCalibration Curve DriftEvaluates changes in probability calibration between reference and monitoring datasets....TrueTrue['datasets', 'model']{'n_bins': {'type': 'int', 'default': 10}, 'drift_pct_threshold': {'type': 'float', 'default': 20}}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassDiscriminationDriftClass Discrimination DriftCompares classification discrimination metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassImbalanceDriftClass Imbalance DriftEvaluates drift in class distribution between reference and monitoring datasets....TrueTrue['datasets']{'drift_pct_threshold': {'type': 'float', 'default': 5.0}, 'title': {'type': 'str', 'default': 'Class Distribution Drift'}}['tabular_data', 'binary_classification', 'multiclass_classification']['classification']
validmind.ongoing_monitoring.ClassificationAccuracyDriftClassification Accuracy DriftCompares classification accuracy metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ConfusionMatrixDriftConfusion Matrix DriftCompares confusion matrix metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.CumulativePredictionProbabilitiesDriftCumulative Prediction Probabilities DriftCompares cumulative prediction probability distributions between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['visualization', 'credit_risk']['classification']
validmind.ongoing_monitoring.FeatureDriftFeature DriftEvaluates changes in feature distribution over time to identify potential model drift....TrueTrue['datasets']{'bins': {'type': '_empty', 'default': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]}, 'feature_columns': {'type': '_empty', 'default': None}, 'psi_threshold': {'type': '_empty', 'default': 0.2}}['visualization']['monitoring']
validmind.ongoing_monitoring.PredictionAcrossEachFeaturePrediction Across Each FeatureAssesses differences in model predictions across individual features between reference and monitoring datasets...TrueFalse['datasets', 'model']{}['visualization']['monitoring']
validmind.ongoing_monitoring.PredictionCorrelationPrediction CorrelationAssesses correlation changes between model predictions from reference and monitoring datasets to detect potential...TrueTrue['datasets', 'model']{'drift_pct_threshold': {'type': 'float', 'default': 20}}['visualization']['monitoring']
validmind.ongoing_monitoring.PredictionProbabilitiesHistogramDriftPrediction Probabilities Histogram DriftCompares prediction probability distributions between reference and monitoring datasets....TrueTrue['datasets', 'model']{'title': {'type': '_empty', 'default': 'Prediction Probabilities Histogram Drift'}, 'drift_pct_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk']['classification']
validmind.ongoing_monitoring.PredictionQuantilesAcrossFeaturesPrediction Quantiles Across FeaturesAssesses differences in model prediction distributions across individual features between reference...TrueFalse['datasets', 'model']{}['visualization']['monitoring']
validmind.ongoing_monitoring.ROCCurveDriftROC Curve DriftCompares ROC curves between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ScoreBandsDriftScore Bands DriftAnalyzes drift in population distribution and default rates across score bands....FalseTrue['datasets', 'model']{'score_column': {'type': 'str', 'default': 'score'}, 'score_bands': {'type': 'list', 'default': None}, 'drift_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk', 'scorecard']['classification']
validmind.ongoing_monitoring.ScorecardHistogramDriftScorecard Histogram DriftCompares score distributions between reference and monitoring datasets for each class....TrueTrue['datasets']{'score_column': {'type': 'str', 'default': 'score'}, 'title': {'type': 'str', 'default': 'Scorecard Histogram Drift'}, 'drift_pct_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk', 'logistic_regression']['classification']
validmind.ongoing_monitoring.TargetPredictionDistributionPlotTarget Prediction Distribution PlotAssesses differences in prediction distributions between a reference dataset and a monitoring dataset to identify...TrueTrue['datasets', 'model']{'drift_pct_threshold': {'type': 'float', 'default': 20}}['visualization']['monitoring']
validmind.prompt_validation.BiasBiasAssesses potential bias in a Large Language Model by analyzing the distribution and order of exemplars in the...FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.ClarityClarityEvaluates and scores the clarity of prompts in a Large Language Model based on specified guidelines....FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.ConcisenessConcisenessAnalyzes and grades the conciseness of prompts provided to a Large Language Model....FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.DelimitationDelimitationEvaluates the proper use of delimiters in prompts provided to Large Language Models....FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.NegativeInstructionNegative InstructionEvaluates and grades the use of affirmative, proactive language over negative instructions in LLM prompts....FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.RobustnessRobustnessAssesses the robustness of prompts provided to a Large Language Model under varying conditions and contexts. This test...FalseTrue['model', 'dataset']{'num_tests': {'type': '_empty', 'default': 10}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.SpecificitySpecificityEvaluates and scores the specificity of prompts provided to a Large Language Model (LLM), based on clarity, detail,...FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.unit_metrics.classification.AccuracyAccuracyCalculates the accuracy of a modelFalseFalse['dataset', 'model']{}['classification']['classification']
validmind.unit_metrics.classification.F1F1Calculates the F1 score for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.PrecisionPrecisionCalculates the precision for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.ROC_AUCROC AUCCalculates the ROC AUC for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.RecallRecallCalculates the recall for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.regression.AdjustedRSquaredScoreAdjusted R Squared ScoreCalculates the adjusted R-squared score for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.GiniCoefficientGini CoefficientCalculates the Gini coefficient for a regression model.FalseFalse['dataset', 'model']{}['regression']['regression']
validmind.unit_metrics.regression.HuberLossHuber LossCalculates the Huber loss for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.KolmogorovSmirnovStatisticKolmogorov Smirnov StatisticCalculates the Kolmogorov-Smirnov statistic for a regression model.FalseFalse['dataset', 'model']{}['regression']['regression']
validmind.unit_metrics.regression.MeanAbsoluteErrorMean Absolute ErrorCalculates the mean absolute error for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.MeanAbsolutePercentageErrorMean Absolute Percentage ErrorCalculates the mean absolute percentage error for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.MeanBiasDeviationMean Bias DeviationCalculates the mean bias deviation for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.MeanSquaredErrorMean Squared ErrorCalculates the mean squared error for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.QuantileLossQuantile LossCalculates the quantile loss for a regression model.FalseFalse['model', 'dataset']{'quantile': {'type': '_empty', 'default': 0.5}}['regression']['regression']
validmind.unit_metrics.regression.RSquaredScoreR Squared ScoreCalculates the R-squared score for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.RootMeanSquaredErrorRoot Mean Squared ErrorCalculates the root mean squared error for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list_tests()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Understand tags and task types\n", - "\n", - "Use [list_tasks()](https://docs.validmind.ai/validmind/validmind/tests.html#list_tasks) to view all unique task types used to classify tests in the ValidMind Library.\n", - "\n", - "Understanding `task` types helps you filter tests that match your model’s objective. For example:\n", - "\n", - "- **classification:** Works with Classification Models and Datasets.\n", - "- **regression:** Works with Regression Models and Datasets.\n", - "- **text classification:** Works with Text Classification Models and Datasets.\n", - "- **text summarization:** Works with Text Summarization Models and Datasets." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['text_qa',\n", - " 'classification',\n", - " 'data_validation',\n", - " 'text_classification',\n", - " 'feature_extraction',\n", - " 'regression',\n", - " 'visualization',\n", - " 'clustering',\n", - " 'time_series_forecasting',\n", - " 'text_summarization',\n", - " 'nlp',\n", - " 'residual_analysis',\n", - " 'monitoring',\n", - " 'text_generation']" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list_tasks()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use [list_tags()](https://docs.validmind.ai/validmind/validmind/tests.html#list_tags) to view all unique tags used to describe tests in the ValidMind Library.\n", - "\n", - "`Tags` describe what a test applies to and help you filter tests for your use case. Examples include:\n", - "\n", - "- **llm:** Tests that work with Large Language Models.\n", - "- **nlp:** Tests relevant for natural language processing.\n", - "- **binary_classification:** Tests for binary classification tasks.\n", - "- **forecasting:** Tests for forecasting and time-series analysis.\n", - "- **tabular_data:** Tests for tabular data like CSVs and Excel spreadsheets.\n", - "\n" - ] - }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Explore tests\n", + "\n", + "Explore the individual out-the-box tests available in the ValidMind Library, and identify which tests to run to evaluate different aspects of your model. Browse available tests, view their descriptions, and filter by tags or task type to find tests relevant to your use case.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [About ValidMind](#toc1_) \n", + " - [Before you begin](#toc1_1_) \n", + " - [New to ValidMind?](#toc1_2_) \n", + " - [Key concepts](#toc1_3_) \n", + "- [Install the ValidMind Library](#toc2_) \n", + "- [List all available tests](#toc3_) \n", + "- [Understand tags and task types](#toc4_) \n", + "- [Filter tests by tags and task types](#toc5_)\n", + "- [Store test sets for use](#toc6_) \n", + "- [Next steps](#toc7_) \n", + " - [Discover more learning resources](#toc7_1_) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## About ValidMind\n", + "\n", + "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n", + "\n", + "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n", + "\n", + "\n", + "\n", + "### Before you begin\n", + "\n", + "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n", + "\n", + "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n", + "\n", + "\n", + "\n", + "\n", + "### New to ValidMind?\n", + "\n", + "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n", + "\n", + "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", + "

\n", + "Register with ValidMind
\n", + "\n", + "\n", + "\n", + "### Key concepts\n", + "\n", + "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n", + "\n", + "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", + "\n", + "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n", + "\n", + "**Custom tests**: Custom tests are functions that you define to evaluate your model or dataset. These functions can be registered via the ValidMind Library to be used with the ValidMind Platform.\n", + "\n", + "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n", + "\n", + " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n", + " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n", + " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom test.\n", + " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom test. See this [example](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html) for more information.\n", + "\n", + "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a test, customize its behavior, or provide additional context.\n", + "\n", + "**Outputs**: Custom tests can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n", + "\n", + "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n", + "\n", + "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Install the ValidMind Library\n", + "\n", + "
Recommended Python versions\n", + "

\n", + "Python 3.8 <= x <= 3.11
\n", + "\n", + "To install the library:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q validmind" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## List all available tests\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Start by importing the functions from the [validmind.tests](https://docs.validmind.ai/validmind/validmind/tests.html) module for listing tests, listing tasks, listing tags, and listing tasks and tags to access these functions in the rest of this notebook:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.tests import (\n", + " list_tests,\n", + " list_tasks,\n", + " list_tags,\n", + " list_tasks_and_tags,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use [list_tests()](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to retrieve all available ValidMind tests, which returns a DataFrame with the following columns:\n", + "\n", + "- **ID** – A unique identifier for each test.\n", + "- **Name** – The test’s name.\n", + "- **Description** – A short summary of what the test evaluates.\n", + "- **Tags** – Keywords that describe what the test does or applies to.\n", + "- **Tasks** – The type of modeling task the test supports." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['senstivity_analysis',\n", - " 'calibration',\n", - " 'clustering',\n", - " 'anomaly_detection',\n", - " 'nlp',\n", - " 'classification_metrics',\n", - " 'dimensionality_reduction',\n", - " 'tabular_data',\n", - " 'time_series_data',\n", - " 'model_predictions',\n", - " 'feature_selection',\n", - " 'correlation',\n", - " 'frequency_analysis',\n", - " 'embeddings',\n", - " 'regression',\n", - " 'llm',\n", - " 'statsmodels',\n", - " 'ragas',\n", - " 'model_performance',\n", - " 'model_validation',\n", - " 'rag_performance',\n", - " 'model_training',\n", - " 'qualitative',\n", - " 'classification',\n", - " 'kmeans',\n", - " 'multiclass_classification',\n", - " 'linear_regression',\n", - " 'data_quality',\n", - " 'text_data',\n", - " 'binary_classification',\n", - " 'threshold_optimization',\n", - " 'stationarity',\n", - " 'bias_and_fairness',\n", - " 'scorecard',\n", - " 'model_explainability',\n", - " 'model_comparison',\n", - " 'numerical_data',\n", - " 'sklearn',\n", - " 'model_selection',\n", - " 'retrieval_performance',\n", - " 'zero_shot',\n", - " 'statistical_test',\n", - " 'descriptive_statistics',\n", - " 'seasonality',\n", - " 'analysis',\n", - " 'data_validation',\n", - " 'data_distribution',\n", - " 'feature_importance',\n", - " 'metadata',\n", - " 'few_shot',\n", - " 'visualization',\n", - " 'credit_risk',\n", - " 'forecasting',\n", - " 'AUC',\n", - " 'logistic_regression',\n", - " 'model_diagnosis',\n", - " 'model_interpretation',\n", - " 'unit_root_test',\n", - " 'categorical_data',\n", - " 'data_analysis']" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDNameDescriptionHas FigureHas TableRequired InputsParamsTagsTasks
validmind.data_validation.ACFandPACFPlotAC Fand PACF PlotAnalyzes time series data using Autocorrelation Function (ACF) and Partial Autocorrelation Function (PACF) plots to...TrueFalse['dataset']{}['time_series_data', 'forecasting', 'statistical_test', 'visualization']['regression']
validmind.data_validation.ADFADFAssesses the stationarity of a time series dataset using the Augmented Dickey-Fuller (ADF) test....FalseTrue['dataset']{}['time_series_data', 'statsmodels', 'forecasting', 'statistical_test', 'stationarity']['regression']
validmind.data_validation.AutoARAuto ARAutomatically identifies the optimal Autoregressive (AR) order for a time series using BIC and AIC criteria....FalseTrue['dataset']{'max_ar_order': {'type': 'int', 'default': 3}}['time_series_data', 'statsmodels', 'forecasting', 'statistical_test']['regression']
validmind.data_validation.AutoMAAuto MAAutomatically selects the optimal Moving Average (MA) order for each variable in a time series dataset based on...FalseTrue['dataset']{'max_ma_order': {'type': 'int', 'default': 3}}['time_series_data', 'statsmodels', 'forecasting', 'statistical_test']['regression']
validmind.data_validation.AutoStationarityAuto StationarityAutomates Augmented Dickey-Fuller test to assess stationarity across multiple time series in a DataFrame....FalseTrue['dataset']{'max_order': {'type': 'int', 'default': 5}, 'threshold': {'type': 'float', 'default': 0.05}}['time_series_data', 'statsmodels', 'forecasting', 'statistical_test']['regression']
validmind.data_validation.BivariateScatterPlotsBivariate Scatter PlotsGenerates bivariate scatterplots to visually inspect relationships between pairs of numerical predictor variables...TrueFalse['dataset']{}['tabular_data', 'numerical_data', 'visualization']['classification']
validmind.data_validation.BoxPierceBox PierceDetects autocorrelation in time-series data through the Box-Pierce test to validate model performance....FalseTrue['dataset']{}['time_series_data', 'forecasting', 'statistical_test', 'statsmodels']['regression']
validmind.data_validation.ChiSquaredFeaturesTableChi Squared Features TableAssesses the statistical association between categorical features and a target variable using the Chi-Squared test....FalseTrue['dataset']{'p_threshold': {'type': '_empty', 'default': 0.05}}['tabular_data', 'categorical_data', 'statistical_test']['classification']
validmind.data_validation.ClassImbalanceClass ImbalanceEvaluates and quantifies class distribution imbalance in a dataset used by a machine learning model....TrueTrue['dataset']{'min_percent_threshold': {'type': 'int', 'default': 10}}['tabular_data', 'binary_classification', 'multiclass_classification', 'data_quality']['classification']
validmind.data_validation.DatasetDescriptionDataset DescriptionProvides comprehensive analysis and statistical summaries of each column in a machine learning model's dataset....FalseTrue['dataset']{}['tabular_data', 'time_series_data', 'text_data']['classification', 'regression', 'text_classification', 'text_summarization']
validmind.data_validation.DatasetSplitDataset SplitEvaluates and visualizes the distribution proportions among training, testing, and validation datasets of an ML...FalseTrue['datasets']{}['tabular_data', 'time_series_data', 'text_data']['classification', 'regression', 'text_classification', 'text_summarization']
validmind.data_validation.DescriptiveStatisticsDescriptive StatisticsPerforms a detailed descriptive statistical analysis of both numerical and categorical data within a model's...FalseTrue['dataset']{}['tabular_data', 'time_series_data', 'data_quality']['classification', 'regression']
validmind.data_validation.DickeyFullerGLSDickey Fuller GLSAssesses stationarity in time series data using the Dickey-Fuller GLS test to determine the order of integration....FalseTrue['dataset']{}['time_series_data', 'forecasting', 'unit_root_test']['regression']
validmind.data_validation.DuplicatesDuplicatesTests dataset for duplicate entries, ensuring model reliability via data quality verification....FalseTrue['dataset']{'min_threshold': {'type': '_empty', 'default': 1}}['tabular_data', 'data_quality', 'text_data']['classification', 'regression']
validmind.data_validation.EngleGrangerCointEngle Granger CointAssesses the degree of co-movement between pairs of time series data using the Engle-Granger cointegration test....FalseTrue['dataset']{'threshold': {'type': 'float', 'default': 0.05}}['time_series_data', 'statistical_test', 'forecasting']['regression']
validmind.data_validation.FeatureTargetCorrelationPlotFeature Target Correlation PlotVisualizes the correlation between input features and the model's target output in a color-coded horizontal bar...TrueFalse['dataset']{'fig_height': {'type': '_empty', 'default': 600}}['tabular_data', 'visualization', 'correlation']['classification', 'regression']
validmind.data_validation.HighCardinalityHigh CardinalityAssesses the number of unique values in categorical columns to detect high cardinality and potential overfitting....FalseTrue['dataset']{'num_threshold': {'type': 'int', 'default': 100}, 'percent_threshold': {'type': 'float', 'default': 0.1}, 'threshold_type': {'type': 'str', 'default': 'percent'}}['tabular_data', 'data_quality', 'categorical_data']['classification', 'regression']
validmind.data_validation.HighPearsonCorrelationHigh Pearson CorrelationIdentifies highly correlated feature pairs in a dataset suggesting feature redundancy or multicollinearity....FalseTrue['dataset']{'max_threshold': {'type': 'float', 'default': 0.3}, 'top_n_correlations': {'type': 'int', 'default': 10}, 'feature_columns': {'type': 'list', 'default': None}}['tabular_data', 'data_quality', 'correlation']['classification', 'regression']
validmind.data_validation.IQROutliersBarPlotIQR Outliers Bar PlotVisualizes outlier distribution across percentiles in numerical data using the Interquartile Range (IQR) method....TrueFalse['dataset']{'threshold': {'type': 'float', 'default': 1.5}, 'fig_width': {'type': 'int', 'default': 800}}['tabular_data', 'visualization', 'numerical_data']['classification', 'regression']
validmind.data_validation.IQROutliersTableIQR Outliers TableDetermines and summarizes outliers in numerical features using the Interquartile Range method....FalseTrue['dataset']{'threshold': {'type': 'float', 'default': 1.5}}['tabular_data', 'numerical_data']['classification', 'regression']
validmind.data_validation.IsolationForestOutliersIsolation Forest OutliersDetects outliers in a dataset using the Isolation Forest algorithm and visualizes results through scatter plots....TrueFalse['dataset']{'random_state': {'type': 'int', 'default': 0}, 'contamination': {'type': 'float', 'default': 0.1}, 'feature_columns': {'type': 'list', 'default': None}}['tabular_data', 'anomaly_detection']['classification']
validmind.data_validation.JarqueBeraJarque BeraAssesses normality of dataset features in an ML model using the Jarque-Bera test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.data_validation.KPSSKPSSAssesses the stationarity of time-series data in a machine learning model using the KPSS unit root test....FalseTrue['dataset']{}['time_series_data', 'stationarity', 'unit_root_test', 'statsmodels']['data_validation']
validmind.data_validation.LJungBoxL Jung BoxAssesses autocorrelations in dataset features by performing a Ljung-Box test on each feature....FalseTrue['dataset']{}['time_series_data', 'forecasting', 'statistical_test', 'statsmodels']['regression']
validmind.data_validation.LaggedCorrelationHeatmapLagged Correlation HeatmapAssesses and visualizes correlation between target variable and lagged independent variables in a time-series...TrueFalse['dataset']{'num_lags': {'type': 'int', 'default': 10}}['time_series_data', 'visualization']['regression']
validmind.data_validation.MissingValuesMissing ValuesEvaluates dataset quality by ensuring missing value ratio across all features does not exceed a set threshold....FalseTrue['dataset']{'min_threshold': {'type': 'int', 'default': 1}}['tabular_data', 'data_quality']['classification', 'regression']
validmind.data_validation.MissingValuesBarPlotMissing Values Bar PlotAssesses the percentage and distribution of missing values in the dataset via a bar plot, with emphasis on...TrueFalse['dataset']{'threshold': {'type': 'int', 'default': 80}, 'fig_height': {'type': 'int', 'default': 600}}['tabular_data', 'data_quality', 'visualization']['classification', 'regression']
validmind.data_validation.MutualInformationMutual InformationCalculates mutual information scores between features and target variable to evaluate feature relevance....TrueFalse['dataset']{'min_threshold': {'type': 'float', 'default': 0.01}, 'task': {'type': 'str', 'default': 'classification'}}['feature_selection', 'data_analysis']['classification', 'regression']
validmind.data_validation.PearsonCorrelationMatrixPearson Correlation MatrixEvaluates linear dependency between numerical variables in a dataset via a Pearson Correlation coefficient heat map....TrueFalse['dataset']{}['tabular_data', 'numerical_data', 'correlation']['classification', 'regression']
validmind.data_validation.PhillipsPerronArchPhillips Perron ArchAssesses the stationarity of time series data in each feature of the ML model using the Phillips-Perron test....FalseTrue['dataset']{}['time_series_data', 'forecasting', 'statistical_test', 'unit_root_test']['regression']
validmind.data_validation.ProtectedClassesDescriptionProtected Classes DescriptionVisualizes the distribution of protected classes in the dataset relative to the target variable...TrueTrue['dataset']{'protected_classes': {'type': '_empty', 'default': None}}['bias_and_fairness', 'descriptive_statistics']['classification', 'regression']
validmind.data_validation.RollingStatsPlotRolling Stats PlotEvaluates the stationarity of time series data by plotting its rolling mean and standard deviation over a specified...TrueFalse['dataset']{'window_size': {'type': 'int', 'default': 12}}['time_series_data', 'visualization', 'stationarity']['regression']
validmind.data_validation.RunsTestRuns TestExecutes Runs Test on ML model to detect non-random patterns in output data sequence....FalseTrue['dataset']{}['tabular_data', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.data_validation.ScatterPlotScatter PlotAssesses visual relationships, patterns, and outliers among features in a dataset through scatter plot matrices....TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.ScoreBandDefaultRatesScore Band Default RatesAnalyzes default rates and population distribution across credit score bands....FalseTrue['dataset', 'model']{'score_column': {'type': 'str', 'default': 'score'}, 'score_bands': {'type': 'list', 'default': None}}['visualization', 'credit_risk', 'scorecard']['classification']
validmind.data_validation.SeasonalDecomposeSeasonal DecomposeAssesses patterns and seasonality in a time series dataset by decomposing its features into foundational components....TrueFalse['dataset']{'seasonal_model': {'type': 'str', 'default': 'additive'}}['time_series_data', 'seasonality', 'statsmodels']['regression']
validmind.data_validation.ShapiroWilkShapiro WilkEvaluates feature-wise normality of training data using the Shapiro-Wilk test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test']['classification', 'regression']
validmind.data_validation.SkewnessSkewnessEvaluates the skewness of numerical data in a dataset to check against a defined threshold, aiming to ensure data...FalseTrue['dataset']{'max_threshold': {'type': '_empty', 'default': 1}}['data_quality', 'tabular_data']['classification', 'regression']
validmind.data_validation.SpreadPlotSpread PlotAssesses potential correlations between pairs of time series variables through visualization to enhance...TrueFalse['dataset']{}['time_series_data', 'visualization']['regression']
validmind.data_validation.TabularCategoricalBarPlotsTabular Categorical Bar PlotsGenerates and visualizes bar plots for each category in categorical features to evaluate the dataset's composition....TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.TabularDateTimeHistogramsTabular Date Time HistogramsGenerates histograms to provide graphical insight into the distribution of time intervals in a model's datetime...TrueFalse['dataset']{}['time_series_data', 'visualization']['classification', 'regression']
validmind.data_validation.TabularDescriptionTablesTabular Description TablesSummarizes key descriptive statistics for numerical, categorical, and datetime variables in a dataset....FalseTrue['dataset']{}['tabular_data']['classification', 'regression']
validmind.data_validation.TabularNumericalHistogramsTabular Numerical HistogramsGenerates histograms for each numerical feature in a dataset to provide visual insights into data distribution and...TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.TargetRateBarPlotsTarget Rate Bar PlotsGenerates bar plots visualizing the default rates of categorical features for a classification machine learning...TrueFalse['dataset']{}['tabular_data', 'visualization', 'categorical_data']['classification']
validmind.data_validation.TimeSeriesDescriptionTime Series DescriptionGenerates a detailed analysis for the provided time series dataset, summarizing key statistics to identify trends,...FalseTrue['dataset']{}['time_series_data', 'analysis']['regression']
validmind.data_validation.TimeSeriesDescriptiveStatisticsTime Series Descriptive StatisticsEvaluates the descriptive statistics of a time series dataset to identify trends, patterns, and data quality issues....FalseTrue['dataset']{}['time_series_data', 'analysis']['regression']
validmind.data_validation.TimeSeriesFrequencyTime Series FrequencyEvaluates consistency of time series data frequency and generates a frequency plot....TrueTrue['dataset']{}['time_series_data']['regression']
validmind.data_validation.TimeSeriesHistogramTime Series HistogramVisualizes distribution of time-series data using histograms and Kernel Density Estimation (KDE) lines....TrueFalse['dataset']{'nbins': {'type': '_empty', 'default': 30}}['data_validation', 'visualization', 'time_series_data']['regression', 'time_series_forecasting']
validmind.data_validation.TimeSeriesLinePlotTime Series Line PlotGenerates and analyses time-series data through line plots revealing trends, patterns, anomalies over time....TrueFalse['dataset']{}['time_series_data', 'visualization']['regression']
validmind.data_validation.TimeSeriesMissingValuesTime Series Missing ValuesValidates time-series data quality by confirming the count of missing values is below a certain threshold....TrueTrue['dataset']{'min_threshold': {'type': 'int', 'default': 1}}['time_series_data']['regression']
validmind.data_validation.TimeSeriesOutliersTime Series OutliersIdentifies and visualizes outliers in time-series data using the z-score method....FalseTrue['dataset']{'zscore_threshold': {'type': 'int', 'default': 3}}['time_series_data']['regression']
validmind.data_validation.TooManyZeroValuesToo Many Zero ValuesIdentifies numerical columns in a dataset that contain an excessive number of zero values, defined by a threshold...FalseTrue['dataset']{'max_percent_threshold': {'type': 'float', 'default': 0.03}}['tabular_data']['regression', 'classification']
validmind.data_validation.UniqueRowsUnique RowsVerifies the diversity of the dataset by ensuring that the count of unique rows exceeds a prescribed threshold....FalseTrue['dataset']{'min_percent_threshold': {'type': 'float', 'default': 1}}['tabular_data']['regression', 'classification']
validmind.data_validation.WOEBinPlotsWOE Bin PlotsGenerates visualizations of Weight of Evidence (WoE) and Information Value (IV) for understanding predictive power...TrueFalse['dataset']{'breaks_adj': {'type': 'list', 'default': None}, 'fig_height': {'type': 'int', 'default': 600}, 'fig_width': {'type': 'int', 'default': 500}}['tabular_data', 'visualization', 'categorical_data']['classification']
validmind.data_validation.WOEBinTableWOE Bin TableAssesses the Weight of Evidence (WoE) and Information Value (IV) of each feature to evaluate its predictive power...FalseTrue['dataset']{'breaks_adj': {'type': 'list', 'default': None}}['tabular_data', 'categorical_data']['classification']
validmind.data_validation.ZivotAndrewsArchZivot Andrews ArchEvaluates the order of integration and stationarity of time series data using the Zivot-Andrews unit root test....FalseTrue['dataset']{}['time_series_data', 'stationarity', 'unit_root_test']['regression']
validmind.data_validation.nlp.CommonWordsCommon WordsAssesses the most frequent non-stopwords in a text column for identifying prevalent language patterns....TrueFalse['dataset']{}['nlp', 'text_data', 'visualization', 'frequency_analysis']['text_classification', 'text_summarization']
validmind.data_validation.nlp.HashtagsHashtagsAssesses hashtag frequency in a text column, highlighting usage trends and potential dataset bias or spam....TrueFalse['dataset']{'top_hashtags': {'type': 'int', 'default': 25}}['nlp', 'text_data', 'visualization', 'frequency_analysis']['text_classification', 'text_summarization']
validmind.data_validation.nlp.LanguageDetectionLanguage DetectionAssesses the diversity of languages in a textual dataset by detecting and visualizing the distribution of languages....TrueFalse['dataset']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.data_validation.nlp.MentionsMentionsCalculates and visualizes frequencies of '@' prefixed mentions in a text-based dataset for NLP model analysis....TrueFalse['dataset']{'top_mentions': {'type': 'int', 'default': 25}}['nlp', 'text_data', 'visualization', 'frequency_analysis']['text_classification', 'text_summarization']
validmind.data_validation.nlp.PolarityAndSubjectivityPolarity And SubjectivityAnalyzes the polarity and subjectivity of text data within a given dataset to visualize the sentiment distribution....TrueTrue['dataset']{'threshold_subjectivity': {'type': '_empty', 'default': 0.5}, 'threshold_polarity': {'type': '_empty', 'default': 0}}['nlp', 'text_data', 'data_validation']['nlp']
validmind.data_validation.nlp.PunctuationsPunctuationsAnalyzes and visualizes the frequency distribution of punctuation usage in a given text dataset....TrueFalse['dataset']{'count_mode': {'type': '_empty', 'default': 'token'}}['nlp', 'text_data', 'visualization', 'frequency_analysis']['text_classification', 'text_summarization', 'nlp']
validmind.data_validation.nlp.SentimentSentimentAnalyzes the sentiment of text data within a dataset using the VADER sentiment analysis tool....TrueFalse['dataset']{}['nlp', 'text_data', 'data_validation']['nlp']
validmind.data_validation.nlp.StopWordsStop WordsEvaluates and visualizes the frequency of English stop words in a text dataset against a defined threshold....TrueTrue['dataset']{'min_percent_threshold': {'type': 'float', 'default': 0.5}, 'num_words': {'type': 'int', 'default': 25}}['nlp', 'text_data', 'frequency_analysis', 'visualization']['text_classification', 'text_summarization']
validmind.data_validation.nlp.TextDescriptionText DescriptionConducts comprehensive textual analysis on a dataset using NLTK to evaluate various parameters and generate...TrueFalse['dataset']{'unwanted_tokens': {'type': 'set', 'default': {'s', 'mrs', 'us', \"''\", ' ', 'ms', 'dr', 'dollar', '``', 'mr', \"'s\", \"s'\"}}, 'lang': {'type': 'str', 'default': 'english'}}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.data_validation.nlp.ToxicityToxicityAssesses the toxicity of text data within a dataset to visualize the distribution of toxicity scores....TrueFalse['dataset']{}['nlp', 'text_data', 'data_validation']['nlp']
validmind.model_validation.BertScoreBert ScoreAssesses the quality of machine-generated text using BERTScore metrics and visualizes results through histograms...TrueTrue['dataset', 'model']{'evaluation_model': {'type': '_empty', 'default': 'distilbert-base-uncased'}}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.BleuScoreBleu ScoreEvaluates the quality of machine-generated text using BLEU metrics and visualizes the results through histograms...TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.ClusterSizeDistributionCluster Size DistributionAssesses the performance of clustering models by comparing the distribution of cluster sizes in model predictions...TrueFalse['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.ContextualRecallContextual RecallEvaluates a Natural Language Generation model's ability to generate contextually relevant and factually correct...TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.FeaturesAUCFeatures AUCEvaluates the discriminatory power of each individual feature within a binary classification model by calculating...TrueFalse['dataset']{'fontsize': {'type': 'int', 'default': 12}, 'figure_height': {'type': 'int', 'default': 500}}['feature_importance', 'AUC', 'visualization']['classification']
validmind.model_validation.MeteorScoreMeteor ScoreAssesses the quality of machine-generated translations by comparing them to human-produced references using the...TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.ModelMetadataModel MetadataCompare metadata of different models and generate a summary table with the results....FalseTrue['model']{}['model_training', 'metadata']['regression', 'time_series_forecasting']
validmind.model_validation.ModelPredictionResidualsModel Prediction ResidualsAssesses normality and behavior of residuals in regression models through visualization and statistical tests....TrueTrue['dataset', 'model']{'nbins': {'type': 'int', 'default': 100}, 'p_value_threshold': {'type': 'float', 'default': 0.05}, 'start_date': {'type': None, 'default': None}, 'end_date': {'type': None, 'default': None}}['regression']['residual_analysis', 'visualization']
validmind.model_validation.RegardScoreRegard ScoreAssesses the sentiment and potential biases in text generated by NLP models by computing and visualizing regard...TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.RegressionResidualsPlotRegression Residuals PlotEvaluates regression model performance using residual distribution and actual vs. predicted plots....TrueFalse['model', 'dataset']{'bin_size': {'type': 'float', 'default': 0.1}}['model_performance', 'visualization']['regression']
validmind.model_validation.RougeScoreRouge ScoreAssesses the quality of machine-generated text using ROUGE metrics and visualizes the results to provide...TrueTrue['dataset', 'model']{'metric': {'type': 'str', 'default': 'rouge-1'}}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.TimeSeriesPredictionWithCITime Series Prediction With CIAssesses predictive accuracy and uncertainty in time series models, highlighting breaches beyond confidence...TrueTrue['dataset', 'model']{'confidence': {'type': 'float', 'default': 0.95}}['model_predictions', 'visualization']['regression', 'time_series_forecasting']
validmind.model_validation.TimeSeriesPredictionsPlotTime Series Predictions PlotPlot actual vs predicted values for time series data and generate a visual comparison for the model....TrueFalse['dataset', 'model']{}['model_predictions', 'visualization']['regression', 'time_series_forecasting']
validmind.model_validation.TimeSeriesR2SquareBySegmentsTime Series R2 Square By SegmentsEvaluates the R-Squared values of regression models over specified time segments in time series data to assess...TrueTrue['dataset', 'model']{'segments': {'type': None, 'default': None}}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.TokenDisparityToken DisparityEvaluates the token disparity between reference and generated texts, visualizing the results through histograms and...TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.ToxicityScoreToxicity ScoreAssesses the toxicity levels of texts generated by NLP models to identify and mitigate harmful or offensive content....TrueTrue['dataset', 'model']{}['nlp', 'text_data', 'visualization']['text_classification', 'text_summarization']
validmind.model_validation.embeddings.ClusterDistributionCluster DistributionAssesses the distribution of text embeddings across clusters produced by a model using KMeans clustering....TrueFalse['model', 'dataset']{'num_clusters': {'type': 'int', 'default': 5}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.CosineSimilarityComparisonCosine Similarity ComparisonAssesses the similarity between embeddings generated by different models using Cosine Similarity, providing both...TrueTrue['dataset', 'models']{}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.embeddings.CosineSimilarityDistributionCosine Similarity DistributionAssesses the similarity between predicted text embeddings from a model using a Cosine Similarity distribution...TrueFalse['dataset', 'model']{}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.CosineSimilarityHeatmapCosine Similarity HeatmapGenerates an interactive heatmap to visualize the cosine similarities among embeddings derived from a given model....TrueFalse['dataset', 'model']{'title': {'type': '_empty', 'default': 'Cosine Similarity Matrix'}, 'color': {'type': '_empty', 'default': 'Cosine Similarity'}, 'xaxis_title': {'type': '_empty', 'default': 'Index'}, 'yaxis_title': {'type': '_empty', 'default': 'Index'}, 'color_scale': {'type': '_empty', 'default': 'Blues'}}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.embeddings.DescriptiveAnalyticsDescriptive AnalyticsEvaluates statistical properties of text embeddings in an ML model via mean, median, and standard deviation...TrueFalse['dataset', 'model']{}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.EmbeddingsVisualization2DEmbeddings Visualization2 DVisualizes 2D representation of text embeddings generated by a model using t-SNE technique....TrueFalse['dataset', 'model']{'cluster_column': {'type': None, 'default': None}, 'perplexity': {'type': 'int', 'default': 30}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.EuclideanDistanceComparisonEuclidean Distance ComparisonAssesses and visualizes the dissimilarity between model embeddings using Euclidean distance, providing insights...TrueTrue['dataset', 'models']{}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.embeddings.EuclideanDistanceHeatmapEuclidean Distance HeatmapGenerates an interactive heatmap to visualize the Euclidean distances among embeddings derived from a given model....TrueFalse['dataset', 'model']{'title': {'type': '_empty', 'default': 'Euclidean Distance Matrix'}, 'color': {'type': '_empty', 'default': 'Euclidean Distance'}, 'xaxis_title': {'type': '_empty', 'default': 'Index'}, 'yaxis_title': {'type': '_empty', 'default': 'Index'}, 'color_scale': {'type': '_empty', 'default': 'Blues'}}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.embeddings.PCAComponentsPairwisePlotsPCA Components Pairwise PlotsGenerates scatter plots for pairwise combinations of principal component analysis (PCA) components of model...TrueFalse['dataset', 'model']{'n_components': {'type': 'int', 'default': 3}}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.embeddings.StabilityAnalysisKeywordStability Analysis KeywordEvaluates robustness of embedding models to keyword swaps in the test dataset....TrueTrue['dataset', 'model']{'keyword_dict': {'type': None, 'default': None}, 'mean_similarity_threshold': {'type': 'float', 'default': 0.7}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.StabilityAnalysisRandomNoiseStability Analysis Random NoiseAssesses the robustness of text embeddings models to random noise introduced via text perturbations....TrueTrue['dataset', 'model']{'probability': {'type': 'float', 'default': 0.02}, 'mean_similarity_threshold': {'type': 'float', 'default': 0.7}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.StabilityAnalysisSynonymsStability Analysis SynonymsEvaluates the stability of text embeddings models when words in test data are replaced by their synonyms randomly....TrueTrue['dataset', 'model']{'probability': {'type': 'float', 'default': 0.02}, 'mean_similarity_threshold': {'type': 'float', 'default': 0.7}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.StabilityAnalysisTranslationStability Analysis TranslationEvaluates robustness of text embeddings models to noise introduced by translating the original text to another...TrueTrue['dataset', 'model']{'source_lang': {'type': 'str', 'default': 'en'}, 'target_lang': {'type': 'str', 'default': 'fr'}, 'mean_similarity_threshold': {'type': 'float', 'default': 0.7}}['llm', 'text_data', 'embeddings', 'visualization']['feature_extraction']
validmind.model_validation.embeddings.TSNEComponentsPairwisePlotsTSNE Components Pairwise PlotsCreates scatter plots for pairwise combinations of t-SNE components to visualize embeddings and highlight potential...TrueFalse['dataset', 'model']{'n_components': {'type': 'int', 'default': 2}, 'perplexity': {'type': 'int', 'default': 30}, 'title': {'type': 'str', 'default': 't-SNE'}}['visualization', 'dimensionality_reduction', 'embeddings']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.AnswerCorrectnessAnswer CorrectnessEvaluates the correctness of answers in a dataset with respect to the provided ground...TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'response_column': {'type': 'str', 'default': 'response'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.AspectCriticAspect CriticEvaluates generations against the following aspects: harmfulness, maliciousness,...TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'response_column': {'type': 'str', 'default': 'response'}, 'retrieved_contexts_column': {'type': None, 'default': None}, 'aspects': {'type': None, 'default': ['coherence', 'conciseness', 'correctness', 'harmfulness', 'maliciousness']}, 'additional_aspects': {'type': None, 'default': None}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'qualitative']['text_summarization', 'text_generation', 'text_qa']
validmind.model_validation.ragas.ContextEntityRecallContext Entity RecallEvaluates the context entity recall for dataset entries and visualizes the results....TrueTrue['dataset']{'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'retrieval_performance']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.ContextPrecisionContext PrecisionContext Precision is a metric that evaluates whether all of the ground-truth...TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'retrieval_performance']['text_qa', 'text_generation', 'text_summarization', 'text_classification']
validmind.model_validation.ragas.ContextPrecisionWithoutReferenceContext Precision Without ReferenceContext Precision Without Reference is a metric used to evaluate the relevance of...TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'response_column': {'type': 'str', 'default': 'response'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'retrieval_performance']['text_qa', 'text_generation', 'text_summarization', 'text_classification']
validmind.model_validation.ragas.ContextRecallContext RecallContext recall measures the extent to which the retrieved context aligns with the...TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'retrieval_performance']['text_qa', 'text_generation', 'text_summarization', 'text_classification']
validmind.model_validation.ragas.FaithfulnessFaithfulnessEvaluates the faithfulness of the generated answers with respect to retrieved contexts....TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'response_column': {'type': 'str', 'default': 'response'}, 'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'rag_performance']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.NoiseSensitivityNoise SensitivityAssesses the sensitivity of a Large Language Model (LLM) to noise in retrieved context by measuring how often it...TrueTrue['dataset']{'response_column': {'type': 'str', 'default': 'response'}, 'retrieved_contexts_column': {'type': 'str', 'default': 'retrieved_contexts'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'focus': {'type': 'str', 'default': 'relevant'}, 'user_input_column': {'type': 'str', 'default': 'user_input'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'rag_performance']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.ResponseRelevancyResponse RelevancyAssesses how pertinent the generated answer is to the given prompt....TrueTrue['dataset']{'user_input_column': {'type': 'str', 'default': 'user_input'}, 'retrieved_contexts_column': {'type': 'str', 'default': None}, 'response_column': {'type': 'str', 'default': 'response'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm', 'rag_performance']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.ragas.SemanticSimilaritySemantic SimilarityCalculates the semantic similarity between generated responses and ground truths...TrueTrue['dataset']{'response_column': {'type': 'str', 'default': 'response'}, 'reference_column': {'type': 'str', 'default': 'reference'}, 'judge_llm': {'type': '_empty', 'default': None}, 'judge_embeddings': {'type': '_empty', 'default': None}}['ragas', 'llm']['text_qa', 'text_generation', 'text_summarization']
validmind.model_validation.sklearn.AdjustedMutualInformationAdjusted Mutual InformationEvaluates clustering model performance by measuring mutual information between true and predicted labels, adjusting...FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.AdjustedRandIndexAdjusted Rand IndexMeasures the similarity between two data clusters using the Adjusted Rand Index (ARI) metric in clustering machine...FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.CalibrationCurveCalibration CurveEvaluates the calibration of probability estimates by comparing predicted probabilities against observed...TrueFalse['model', 'dataset']{'n_bins': {'type': 'int', 'default': 10}}['sklearn', 'model_performance', 'classification']['classification']
validmind.model_validation.sklearn.ClassifierPerformanceClassifier PerformanceEvaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy,...FalseTrue['dataset', 'model']{'average': {'type': 'str', 'default': 'macro'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ClassifierThresholdOptimizationClassifier Threshold OptimizationAnalyzes and visualizes different threshold optimization methods for binary classification models....FalseTrue['dataset', 'model']{'methods': {'type': None, 'default': None}, 'target_recall': {'type': None, 'default': None}}['model_validation', 'threshold_optimization', 'classification_metrics']['classification']
validmind.model_validation.sklearn.ClusterCosineSimilarityCluster Cosine SimilarityMeasures the intra-cluster similarity of a clustering model using cosine similarity....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.ClusterPerformanceMetricsCluster Performance MetricsEvaluates the performance of clustering machine learning models using multiple established metrics....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.CompletenessScoreCompleteness ScoreEvaluates a clustering model's capacity to categorize instances from a single class into the same cluster....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.ConfusionMatrixConfusion MatrixEvaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix...TrueFalse['dataset', 'model']{'threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.FeatureImportanceFeature ImportanceCompute feature importance scores for a given model and generate a summary table...FalseTrue['dataset', 'model']{'num_features': {'type': 'int', 'default': 3}}['model_explainability', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.FowlkesMallowsScoreFowlkes Mallows ScoreEvaluates the similarity between predicted and actual cluster assignments in a model using the Fowlkes-Mallows...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.HomogeneityScoreHomogeneity ScoreAssesses clustering homogeneity by comparing true and predicted labels, scoring from 0 (heterogeneous) to 1...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.HyperParametersTuningHyper Parameters TuningPerforms exhaustive grid search over specified parameter ranges to find optimal model configurations...FalseTrue['model', 'dataset']{'param_grid': {'type': 'dict', 'default': None}, 'scoring': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}, 'fit_params': {'type': 'dict', 'default': None}}['sklearn', 'model_performance']['clustering', 'classification']
validmind.model_validation.sklearn.KMeansClustersOptimizationK Means Clusters OptimizationOptimizes the number of clusters in K-means models using Elbow and Silhouette methods....TrueFalse['model', 'dataset']{'n_clusters': {'type': None, 'default': None}}['sklearn', 'model_performance', 'kmeans']['clustering']
validmind.model_validation.sklearn.MinimumAccuracyMinimum AccuracyChecks if the model's prediction accuracy meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.7}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumF1ScoreMinimum F1 ScoreAssesses if the model's F1 score on the validation set meets a predefined minimum threshold, ensuring balanced...FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumROCAUCScoreMinimum ROCAUC ScoreValidates model by checking if the ROC AUC score meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ModelParametersModel ParametersExtracts and displays model parameters in a structured format for transparency and reproducibility....FalseTrue['model']{'model_params': {'type': None, 'default': None}}['model_training', 'metadata']['classification', 'regression']
validmind.model_validation.sklearn.ModelsPerformanceComparisonModels Performance ComparisonEvaluates and compares the performance of multiple Machine Learning models using various metrics like accuracy,...FalseTrue['dataset', 'models']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'model_comparison']['classification', 'text_classification']
validmind.model_validation.sklearn.OverfitDiagnosisOverfit DiagnosisAssesses potential overfitting in a model's predictions, identifying regions where performance between training and...TrueTrue['model', 'datasets']{'metric': {'type': 'str', 'default': None}, 'cut_off_threshold': {'type': 'float', 'default': 0.04}}['sklearn', 'binary_classification', 'multiclass_classification', 'linear_regression', 'model_diagnosis']['classification', 'regression']
validmind.model_validation.sklearn.PermutationFeatureImportancePermutation Feature ImportanceAssesses the significance of each feature in a model by evaluating the impact on model performance when feature...TrueFalse['model', 'dataset']{'fontsize': {'type': None, 'default': None}, 'figure_height': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.PopulationStabilityIndexPopulation Stability IndexAssesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across...TrueTrue['datasets', 'model']{'num_bins': {'type': 'int', 'default': 10}, 'mode': {'type': 'str', 'default': 'fixed'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.PrecisionRecallCurvePrecision Recall CurveEvaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve....TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ROCCurveROC CurveEvaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic...TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.RegressionErrorsRegression ErrorsAssesses the performance and error distribution of a regression model using various error metrics....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance']['regression', 'classification']
validmind.model_validation.sklearn.RegressionErrorsComparisonRegression Errors ComparisonAssesses multiple regression error metrics to compare model performance across different datasets, emphasizing...FalseTrue['datasets', 'models']{}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.RegressionPerformanceRegression PerformanceEvaluates the performance of a regression model using five different metrics: MAE, MSE, RMSE, MAPE, and MBD....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance']['regression']
validmind.model_validation.sklearn.RegressionR2SquareRegression R2 SquareAssesses the overall goodness-of-fit of a regression model by evaluating R-squared (R2) and Adjusted R-squared (Adj...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['regression']
validmind.model_validation.sklearn.RegressionR2SquareComparisonRegression R2 Square ComparisonCompares R-Squared and Adjusted R-Squared values for different regression models across multiple datasets to assess...FalseTrue['datasets', 'models']{}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.RobustnessDiagnosisRobustness DiagnosisAssesses the robustness of a machine learning model by evaluating performance decay under noisy conditions....TrueTrue['datasets', 'model']{'metric': {'type': 'str', 'default': None}, 'scaling_factor_std_dev_list': {'type': None, 'default': [0.1, 0.2, 0.3, 0.4, 0.5]}, 'performance_decay_threshold': {'type': 'float', 'default': 0.05}}['sklearn', 'model_diagnosis', 'visualization']['classification', 'regression']
validmind.model_validation.sklearn.SHAPGlobalImportanceSHAP Global ImportanceEvaluates and visualizes global feature importance using SHAP values for model explanation and risk identification....FalseTrue['model', 'dataset']{'kernel_explainer_samples': {'type': 'int', 'default': 10}, 'tree_or_linear_explainer_samples': {'type': 'int', 'default': 200}, 'class_of_interest': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ScoreProbabilityAlignmentScore Probability AlignmentAnalyzes the alignment between credit scores and predicted probabilities....TrueTrue['model', 'dataset']{'score_column': {'type': 'str', 'default': 'score'}, 'n_bins': {'type': 'int', 'default': 10}}['visualization', 'credit_risk', 'calibration']['classification']
validmind.model_validation.sklearn.SilhouettePlotSilhouette PlotCalculates and visualizes Silhouette Score, assessing the degree of data point suitability to its cluster in ML...TrueTrue['model', 'dataset']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.TrainingTestDegradationTraining Test DegradationTests if model performance degradation between training and test datasets exceeds a predefined threshold....FalseTrue['datasets', 'model']{'max_threshold': {'type': 'float', 'default': 0.1}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.VMeasureV MeasureEvaluates homogeneity and completeness of a clustering model using the V Measure Score....FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.WeakspotsDiagnosisWeakspots DiagnosisIdentifies and visualizes weak spots in a machine learning model's performance across various sections of the...TrueTrue['datasets', 'model']{'features_columns': {'type': None, 'default': None}, 'metrics': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_diagnosis', 'visualization']['classification', 'text_classification']
validmind.model_validation.statsmodels.AutoARIMAAuto ARIMAEvaluates ARIMA models for time-series forecasting, ranking them using Bayesian and Akaike Information Criteria....FalseTrue['model', 'dataset']{}['time_series_data', 'forecasting', 'model_selection', 'statsmodels']['regression']
validmind.model_validation.statsmodels.CumulativePredictionProbabilitiesCumulative Prediction ProbabilitiesVisualizes cumulative probabilities of positive and negative classes for both training and testing in classification models....TrueFalse['dataset', 'model']{'title': {'type': 'str', 'default': 'Cumulative Probabilities'}}['visualization', 'credit_risk']['classification']
validmind.model_validation.statsmodels.DurbinWatsonTestDurbin Watson TestAssesses autocorrelation in time series data features using the Durbin-Watson statistic....FalseTrue['dataset', 'model']{'threshold': {'type': None, 'default': [1.5, 2.5]}}['time_series_data', 'forecasting', 'statistical_test', 'statsmodels']['regression']
validmind.model_validation.statsmodels.GINITableGINI TableEvaluates classification model performance using AUC, GINI, and KS metrics for training and test datasets....FalseTrue['dataset', 'model']{}['model_performance']['classification']
validmind.model_validation.statsmodels.KolmogorovSmirnovKolmogorov SmirnovAssesses whether each feature in the dataset aligns with a normal distribution using the Kolmogorov-Smirnov test....FalseTrue['model', 'dataset']{'dist': {'type': 'str', 'default': 'norm'}}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.model_validation.statsmodels.LillieforsLillieforsAssesses the normality of feature distributions in an ML model's training dataset using the Lilliefors test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.model_validation.statsmodels.PredictionProbabilitiesHistogramPrediction Probabilities HistogramAssesses the predictive probability distribution for binary classification to evaluate model performance and...TrueFalse['dataset', 'model']{'title': {'type': 'str', 'default': 'Histogram of Predictive Probabilities'}}['visualization', 'credit_risk']['classification']
validmind.model_validation.statsmodels.RegressionCoeffsRegression CoeffsAssesses the significance and uncertainty of predictor variables in a regression model through visualization of...TrueTrue['model']{}['tabular_data', 'visualization', 'model_training']['regression']
validmind.model_validation.statsmodels.RegressionFeatureSignificanceRegression Feature SignificanceAssesses and visualizes the statistical significance of features in a regression model....TrueFalse['model']{'fontsize': {'type': 'int', 'default': 10}, 'p_threshold': {'type': 'float', 'default': 0.05}}['statistical_test', 'model_interpretation', 'visualization', 'feature_importance']['regression']
validmind.model_validation.statsmodels.RegressionModelForecastPlotRegression Model Forecast PlotGenerates plots to visually compare the forecasted outcomes of a regression model against actual observed values over...TrueFalse['model', 'dataset']{'start_date': {'type': None, 'default': None}, 'end_date': {'type': None, 'default': None}}['time_series_data', 'forecasting', 'visualization']['regression']
validmind.model_validation.statsmodels.RegressionModelForecastPlotLevelsRegression Model Forecast Plot LevelsAssesses the alignment between forecasted and observed values in regression models through visual plots...TrueFalse['model', 'dataset']{}['time_series_data', 'forecasting', 'visualization']['regression']
validmind.model_validation.statsmodels.RegressionModelSensitivityPlotRegression Model Sensitivity PlotAssesses the sensitivity of a regression model to changes in independent variables by applying shocks and...TrueFalse['dataset', 'model']{'shocks': {'type': None, 'default': [0.1]}, 'transformation': {'type': None, 'default': None}}['senstivity_analysis', 'visualization']['regression']
validmind.model_validation.statsmodels.RegressionModelSummaryRegression Model SummaryEvaluates regression model performance using metrics including R-Squared, Adjusted R-Squared, MSE, and RMSE....FalseTrue['dataset', 'model']{}['model_performance', 'regression']['regression']
validmind.model_validation.statsmodels.RegressionPermutationFeatureImportanceRegression Permutation Feature ImportanceAssesses the significance of each feature in a model by evaluating the impact on model performance when feature...TrueFalse['dataset', 'model']{'fontsize': {'type': 'int', 'default': 12}, 'figure_height': {'type': 'int', 'default': 500}}['statsmodels', 'feature_importance', 'visualization']['regression']
validmind.model_validation.statsmodels.ScorecardHistogramScorecard HistogramThe Scorecard Histogram test evaluates the distribution of credit scores between default and non-default instances,...TrueFalse['dataset']{'title': {'type': 'str', 'default': 'Histogram of Scores'}, 'score_column': {'type': 'str', 'default': 'score'}}['visualization', 'credit_risk', 'logistic_regression']['classification']
validmind.ongoing_monitoring.CalibrationCurveDriftCalibration Curve DriftEvaluates changes in probability calibration between reference and monitoring datasets....TrueTrue['datasets', 'model']{'n_bins': {'type': 'int', 'default': 10}, 'drift_pct_threshold': {'type': 'float', 'default': 20}}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassDiscriminationDriftClass Discrimination DriftCompares classification discrimination metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassImbalanceDriftClass Imbalance DriftEvaluates drift in class distribution between reference and monitoring datasets....TrueTrue['datasets']{'drift_pct_threshold': {'type': 'float', 'default': 5.0}, 'title': {'type': 'str', 'default': 'Class Distribution Drift'}}['tabular_data', 'binary_classification', 'multiclass_classification']['classification']
validmind.ongoing_monitoring.ClassificationAccuracyDriftClassification Accuracy DriftCompares classification accuracy metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ConfusionMatrixDriftConfusion Matrix DriftCompares confusion matrix metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.CumulativePredictionProbabilitiesDriftCumulative Prediction Probabilities DriftCompares cumulative prediction probability distributions between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['visualization', 'credit_risk']['classification']
validmind.ongoing_monitoring.FeatureDriftFeature DriftEvaluates changes in feature distribution over time to identify potential model drift....TrueTrue['datasets']{'bins': {'type': '_empty', 'default': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]}, 'feature_columns': {'type': '_empty', 'default': None}, 'psi_threshold': {'type': '_empty', 'default': 0.2}}['visualization']['monitoring']
validmind.ongoing_monitoring.PredictionAcrossEachFeaturePrediction Across Each FeatureAssesses differences in model predictions across individual features between reference and monitoring datasets...TrueFalse['datasets', 'model']{}['visualization']['monitoring']
validmind.ongoing_monitoring.PredictionCorrelationPrediction CorrelationAssesses correlation changes between model predictions from reference and monitoring datasets to detect potential...TrueTrue['datasets', 'model']{'drift_pct_threshold': {'type': 'float', 'default': 20}}['visualization']['monitoring']
validmind.ongoing_monitoring.PredictionProbabilitiesHistogramDriftPrediction Probabilities Histogram DriftCompares prediction probability distributions between reference and monitoring datasets....TrueTrue['datasets', 'model']{'title': {'type': '_empty', 'default': 'Prediction Probabilities Histogram Drift'}, 'drift_pct_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk']['classification']
validmind.ongoing_monitoring.PredictionQuantilesAcrossFeaturesPrediction Quantiles Across FeaturesAssesses differences in model prediction distributions across individual features between reference...TrueFalse['datasets', 'model']{}['visualization']['monitoring']
validmind.ongoing_monitoring.ROCCurveDriftROC Curve DriftCompares ROC curves between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ScoreBandsDriftScore Bands DriftAnalyzes drift in population distribution and default rates across score bands....FalseTrue['datasets', 'model']{'score_column': {'type': 'str', 'default': 'score'}, 'score_bands': {'type': 'list', 'default': None}, 'drift_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk', 'scorecard']['classification']
validmind.ongoing_monitoring.ScorecardHistogramDriftScorecard Histogram DriftCompares score distributions between reference and monitoring datasets for each class....TrueTrue['datasets']{'score_column': {'type': 'str', 'default': 'score'}, 'title': {'type': 'str', 'default': 'Scorecard Histogram Drift'}, 'drift_pct_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk', 'logistic_regression']['classification']
validmind.ongoing_monitoring.TargetPredictionDistributionPlotTarget Prediction Distribution PlotAssesses differences in prediction distributions between a reference dataset and a monitoring dataset to identify...TrueTrue['datasets', 'model']{'drift_pct_threshold': {'type': 'float', 'default': 20}}['visualization']['monitoring']
validmind.prompt_validation.BiasBiasAssesses potential bias in a Large Language Model by analyzing the distribution and order of exemplars in the...FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.ClarityClarityEvaluates and scores the clarity of prompts in a Large Language Model based on specified guidelines....FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.ConcisenessConcisenessAnalyzes and grades the conciseness of prompts provided to a Large Language Model....FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.DelimitationDelimitationEvaluates the proper use of delimiters in prompts provided to Large Language Models....FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.NegativeInstructionNegative InstructionEvaluates and grades the use of affirmative, proactive language over negative instructions in LLM prompts....FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.RobustnessRobustnessAssesses the robustness of prompts provided to a Large Language Model under varying conditions and contexts. This test...FalseTrue['model', 'dataset']{'num_tests': {'type': '_empty', 'default': 10}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.prompt_validation.SpecificitySpecificityEvaluates and scores the specificity of prompts provided to a Large Language Model (LLM), based on clarity, detail,...FalseTrue['model']{'min_threshold': {'type': '_empty', 'default': 7}, 'judge_llm': {'type': '_empty', 'default': None}}['llm', 'zero_shot', 'few_shot']['text_classification', 'text_summarization']
validmind.unit_metrics.classification.AccuracyAccuracyCalculates the accuracy of a modelFalseFalse['dataset', 'model']{}['classification']['classification']
validmind.unit_metrics.classification.F1F1Calculates the F1 score for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.PrecisionPrecisionCalculates the precision for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.ROC_AUCROC AUCCalculates the ROC AUC for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.RecallRecallCalculates the recall for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.regression.AdjustedRSquaredScoreAdjusted R Squared ScoreCalculates the adjusted R-squared score for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.GiniCoefficientGini CoefficientCalculates the Gini coefficient for a regression model.FalseFalse['dataset', 'model']{}['regression']['regression']
validmind.unit_metrics.regression.HuberLossHuber LossCalculates the Huber loss for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.KolmogorovSmirnovStatisticKolmogorov Smirnov StatisticCalculates the Kolmogorov-Smirnov statistic for a regression model.FalseFalse['dataset', 'model']{}['regression']['regression']
validmind.unit_metrics.regression.MeanAbsoluteErrorMean Absolute ErrorCalculates the mean absolute error for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.MeanAbsolutePercentageErrorMean Absolute Percentage ErrorCalculates the mean absolute percentage error for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.MeanBiasDeviationMean Bias DeviationCalculates the mean bias deviation for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.MeanSquaredErrorMean Squared ErrorCalculates the mean squared error for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.QuantileLossQuantile LossCalculates the quantile loss for a regression model.FalseFalse['model', 'dataset']{'quantile': {'type': '_empty', 'default': 0.5}}['regression']['regression']
validmind.unit_metrics.regression.RSquaredScoreR Squared ScoreCalculates the R-squared score for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
validmind.unit_metrics.regression.RootMeanSquaredErrorRoot Mean Squared ErrorCalculates the root mean squared error for a regression model.FalseFalse['model', 'dataset']{}['regression']['regression']
\n" ], - "source": [ - "list_tags()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, to match each task type with its related tags, use the [list_tasks_and_tags()](https://docs.validmind.ai/validmind/validmind/tests.html#list_tasks_and_tags) function:" + "text/plain": [ + "" ] - }, + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list_tests()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Understand tags and task types\n", + "\n", + "Use [list_tasks()](https://docs.validmind.ai/validmind/validmind/tests.html#list_tasks) to view all unique task types used to classify tests in the ValidMind Library.\n", + "\n", + "Understanding `task` types helps you filter tests that match your model’s objective. For example:\n", + "\n", + "- **classification:** Works with Classification Models and Datasets.\n", + "- **regression:** Works with Regression Models and Datasets.\n", + "- **text classification:** Works with Text Classification Models and Datasets.\n", + "- **text summarization:** Works with Text Summarization Models and Datasets." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
TaskTags
regressionsenstivity_analysis, tabular_data, time_series_data, model_predictions, feature_selection, correlation, regression, statsmodels, model_performance, model_training, multiclass_classification, linear_regression, data_quality, text_data, model_explainability, binary_classification, stationarity, bias_and_fairness, numerical_data, sklearn, model_selection, statistical_test, descriptive_statistics, seasonality, analysis, data_validation, data_distribution, metadata, feature_importance, visualization, forecasting, model_diagnosis, model_interpretation, unit_root_test, categorical_data, data_analysis
classificationcalibration, anomaly_detection, classification_metrics, tabular_data, time_series_data, feature_selection, correlation, statsmodels, model_performance, model_validation, model_training, classification, multiclass_classification, linear_regression, data_quality, text_data, binary_classification, threshold_optimization, bias_and_fairness, scorecard, model_comparison, numerical_data, sklearn, statistical_test, descriptive_statistics, feature_importance, data_distribution, metadata, visualization, credit_risk, AUC, logistic_regression, model_diagnosis, categorical_data, data_analysis
text_classificationmodel_performance, feature_importance, multiclass_classification, few_shot, frequency_analysis, zero_shot, text_data, visualization, llm, binary_classification, ragas, model_diagnosis, model_comparison, sklearn, nlp, retrieval_performance, tabular_data, time_series_data
text_summarizationqualitative, few_shot, frequency_analysis, embeddings, zero_shot, text_data, visualization, llm, rag_performance, ragas, retrieval_performance, nlp, dimensionality_reduction, tabular_data, time_series_data
data_validationstationarity, statsmodels, unit_root_test, time_series_data
time_series_forecastingmodel_training, data_validation, metadata, visualization, model_explainability, sklearn, model_performance, model_predictions, time_series_data
nlpdata_validation, frequency_analysis, text_data, visualization, nlp
clusteringclustering, model_performance, kmeans, sklearn
residual_analysisregression
visualizationregression
feature_extractionembeddings, text_data, visualization, llm
text_qaqualitative, embeddings, visualization, llm, rag_performance, ragas, dimensionality_reduction, retrieval_performance
text_generationqualitative, embeddings, visualization, llm, rag_performance, ragas, dimensionality_reduction, retrieval_performance
monitoringvisualization
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list_tasks_and_tags()" + "data": { + "text/plain": [ + "['text_qa',\n", + " 'classification',\n", + " 'data_validation',\n", + " 'text_classification',\n", + " 'feature_extraction',\n", + " 'regression',\n", + " 'visualization',\n", + " 'clustering',\n", + " 'time_series_forecasting',\n", + " 'text_summarization',\n", + " 'nlp',\n", + " 'residual_analysis',\n", + " 'monitoring',\n", + " 'text_generation']" ] - }, + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list_tasks()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use [list_tags()](https://docs.validmind.ai/validmind/validmind/tests.html#list_tags) to view all unique tags used to describe tests in the ValidMind Library.\n", + "\n", + "`Tags` describe what a test applies to and help you filter tests for your use case. Examples include:\n", + "\n", + "- **llm:** Tests that work with Large Language Models.\n", + "- **nlp:** Tests relevant for natural language processing.\n", + "- **binary_classification:** Tests for binary classification tasks.\n", + "- **forecasting:** Tests for forecasting and time-series analysis.\n", + "- **tabular_data:** Tests for tabular data like CSVs and Excel spreadsheets.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Filter tests by tags and task types\n", - "\n", - "While listing all tests is useful, you’ll often want to narrow your search. The [list_tests()](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) function supports `filter`, `task`, and `tags` parameters to assist in refining your results.\n", - "\n", - "Use the `filter` parameter to find tests that match a specific keyword, such as `sklearn`:\n" + "data": { + "text/plain": [ + "['senstivity_analysis',\n", + " 'calibration',\n", + " 'clustering',\n", + " 'anomaly_detection',\n", + " 'nlp',\n", + " 'classification_metrics',\n", + " 'dimensionality_reduction',\n", + " 'tabular_data',\n", + " 'time_series_data',\n", + " 'model_predictions',\n", + " 'feature_selection',\n", + " 'correlation',\n", + " 'frequency_analysis',\n", + " 'embeddings',\n", + " 'regression',\n", + " 'llm',\n", + " 'statsmodels',\n", + " 'ragas',\n", + " 'model_performance',\n", + " 'model_validation',\n", + " 'rag_performance',\n", + " 'model_training',\n", + " 'qualitative',\n", + " 'classification',\n", + " 'kmeans',\n", + " 'multiclass_classification',\n", + " 'linear_regression',\n", + " 'data_quality',\n", + " 'text_data',\n", + " 'binary_classification',\n", + " 'threshold_optimization',\n", + " 'stationarity',\n", + " 'bias_and_fairness',\n", + " 'scorecard',\n", + " 'model_explainability',\n", + " 'model_comparison',\n", + " 'numerical_data',\n", + " 'sklearn',\n", + " 'model_selection',\n", + " 'retrieval_performance',\n", + " 'zero_shot',\n", + " 'statistical_test',\n", + " 'descriptive_statistics',\n", + " 'seasonality',\n", + " 'analysis',\n", + " 'data_validation',\n", + " 'data_distribution',\n", + " 'feature_importance',\n", + " 'metadata',\n", + " 'few_shot',\n", + " 'visualization',\n", + " 'credit_risk',\n", + " 'forecasting',\n", + " 'AUC',\n", + " 'logistic_regression',\n", + " 'model_diagnosis',\n", + " 'model_interpretation',\n", + " 'unit_root_test',\n", + " 'categorical_data',\n", + " 'data_analysis']" ] - }, + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list_tags()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, to match each task type with its related tags, use the [list_tasks_and_tags()](https://docs.validmind.ai/validmind/validmind/tests.html#list_tasks_and_tags) function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
IDNameDescriptionHas FigureHas TableRequired InputsParamsTagsTasks
validmind.model_validation.ClusterSizeDistributionCluster Size DistributionAssesses the performance of clustering models by comparing the distribution of cluster sizes in model predictions...TrueFalse['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.TimeSeriesR2SquareBySegmentsTime Series R2 Square By SegmentsEvaluates the R-Squared values of regression models over specified time segments in time series data to assess...TrueTrue['dataset', 'model']{'segments': {'type': None, 'default': None}}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.AdjustedMutualInformationAdjusted Mutual InformationEvaluates clustering model performance by measuring mutual information between true and predicted labels, adjusting...FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.AdjustedRandIndexAdjusted Rand IndexMeasures the similarity between two data clusters using the Adjusted Rand Index (ARI) metric in clustering machine...FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.CalibrationCurveCalibration CurveEvaluates the calibration of probability estimates by comparing predicted probabilities against observed...TrueFalse['model', 'dataset']{'n_bins': {'type': 'int', 'default': 10}}['sklearn', 'model_performance', 'classification']['classification']
validmind.model_validation.sklearn.ClassifierPerformanceClassifier PerformanceEvaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy,...FalseTrue['dataset', 'model']{'average': {'type': 'str', 'default': 'macro'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ClassifierThresholdOptimizationClassifier Threshold OptimizationAnalyzes and visualizes different threshold optimization methods for binary classification models....FalseTrue['dataset', 'model']{'methods': {'type': None, 'default': None}, 'target_recall': {'type': None, 'default': None}}['model_validation', 'threshold_optimization', 'classification_metrics']['classification']
validmind.model_validation.sklearn.ClusterCosineSimilarityCluster Cosine SimilarityMeasures the intra-cluster similarity of a clustering model using cosine similarity....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.ClusterPerformanceMetricsCluster Performance MetricsEvaluates the performance of clustering machine learning models using multiple established metrics....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.CompletenessScoreCompleteness ScoreEvaluates a clustering model's capacity to categorize instances from a single class into the same cluster....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.ConfusionMatrixConfusion MatrixEvaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix...TrueFalse['dataset', 'model']{'threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.FeatureImportanceFeature ImportanceCompute feature importance scores for a given model and generate a summary table...FalseTrue['dataset', 'model']{'num_features': {'type': 'int', 'default': 3}}['model_explainability', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.FowlkesMallowsScoreFowlkes Mallows ScoreEvaluates the similarity between predicted and actual cluster assignments in a model using the Fowlkes-Mallows...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.HomogeneityScoreHomogeneity ScoreAssesses clustering homogeneity by comparing true and predicted labels, scoring from 0 (heterogeneous) to 1...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.HyperParametersTuningHyper Parameters TuningPerforms exhaustive grid search over specified parameter ranges to find optimal model configurations...FalseTrue['model', 'dataset']{'param_grid': {'type': 'dict', 'default': None}, 'scoring': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}, 'fit_params': {'type': 'dict', 'default': None}}['sklearn', 'model_performance']['clustering', 'classification']
validmind.model_validation.sklearn.KMeansClustersOptimizationK Means Clusters OptimizationOptimizes the number of clusters in K-means models using Elbow and Silhouette methods....TrueFalse['model', 'dataset']{'n_clusters': {'type': None, 'default': None}}['sklearn', 'model_performance', 'kmeans']['clustering']
validmind.model_validation.sklearn.MinimumAccuracyMinimum AccuracyChecks if the model's prediction accuracy meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.7}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumF1ScoreMinimum F1 ScoreAssesses if the model's F1 score on the validation set meets a predefined minimum threshold, ensuring balanced...FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumROCAUCScoreMinimum ROCAUC ScoreValidates model by checking if the ROC AUC score meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ModelParametersModel ParametersExtracts and displays model parameters in a structured format for transparency and reproducibility....FalseTrue['model']{'model_params': {'type': None, 'default': None}}['model_training', 'metadata']['classification', 'regression']
validmind.model_validation.sklearn.ModelsPerformanceComparisonModels Performance ComparisonEvaluates and compares the performance of multiple Machine Learning models using various metrics like accuracy,...FalseTrue['dataset', 'models']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'model_comparison']['classification', 'text_classification']
validmind.model_validation.sklearn.OverfitDiagnosisOverfit DiagnosisAssesses potential overfitting in a model's predictions, identifying regions where performance between training and...TrueTrue['model', 'datasets']{'metric': {'type': 'str', 'default': None}, 'cut_off_threshold': {'type': 'float', 'default': 0.04}}['sklearn', 'binary_classification', 'multiclass_classification', 'linear_regression', 'model_diagnosis']['classification', 'regression']
validmind.model_validation.sklearn.PermutationFeatureImportancePermutation Feature ImportanceAssesses the significance of each feature in a model by evaluating the impact on model performance when feature...TrueFalse['model', 'dataset']{'fontsize': {'type': None, 'default': None}, 'figure_height': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.PopulationStabilityIndexPopulation Stability IndexAssesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across...TrueTrue['datasets', 'model']{'num_bins': {'type': 'int', 'default': 10}, 'mode': {'type': 'str', 'default': 'fixed'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.PrecisionRecallCurvePrecision Recall CurveEvaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve....TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ROCCurveROC CurveEvaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic...TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.RegressionErrorsRegression ErrorsAssesses the performance and error distribution of a regression model using various error metrics....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance']['regression', 'classification']
validmind.model_validation.sklearn.RegressionErrorsComparisonRegression Errors ComparisonAssesses multiple regression error metrics to compare model performance across different datasets, emphasizing...FalseTrue['datasets', 'models']{}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.RegressionPerformanceRegression PerformanceEvaluates the performance of a regression model using five different metrics: MAE, MSE, RMSE, MAPE, and MBD....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance']['regression']
validmind.model_validation.sklearn.RegressionR2SquareRegression R2 SquareAssesses the overall goodness-of-fit of a regression model by evaluating R-squared (R2) and Adjusted R-squared (Adj...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['regression']
validmind.model_validation.sklearn.RegressionR2SquareComparisonRegression R2 Square ComparisonCompares R-Squared and Adjusted R-Squared values for different regression models across multiple datasets to assess...FalseTrue['datasets', 'models']{}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.RobustnessDiagnosisRobustness DiagnosisAssesses the robustness of a machine learning model by evaluating performance decay under noisy conditions....TrueTrue['datasets', 'model']{'metric': {'type': 'str', 'default': None}, 'scaling_factor_std_dev_list': {'type': None, 'default': [0.1, 0.2, 0.3, 0.4, 0.5]}, 'performance_decay_threshold': {'type': 'float', 'default': 0.05}}['sklearn', 'model_diagnosis', 'visualization']['classification', 'regression']
validmind.model_validation.sklearn.SHAPGlobalImportanceSHAP Global ImportanceEvaluates and visualizes global feature importance using SHAP values for model explanation and risk identification....FalseTrue['model', 'dataset']{'kernel_explainer_samples': {'type': 'int', 'default': 10}, 'tree_or_linear_explainer_samples': {'type': 'int', 'default': 200}, 'class_of_interest': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ScoreProbabilityAlignmentScore Probability AlignmentAnalyzes the alignment between credit scores and predicted probabilities....TrueTrue['model', 'dataset']{'score_column': {'type': 'str', 'default': 'score'}, 'n_bins': {'type': 'int', 'default': 10}}['visualization', 'credit_risk', 'calibration']['classification']
validmind.model_validation.sklearn.SilhouettePlotSilhouette PlotCalculates and visualizes Silhouette Score, assessing the degree of data point suitability to its cluster in ML...TrueTrue['model', 'dataset']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.TrainingTestDegradationTraining Test DegradationTests if model performance degradation between training and test datasets exceeds a predefined threshold....FalseTrue['datasets', 'model']{'max_threshold': {'type': 'float', 'default': 0.1}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.VMeasureV MeasureEvaluates homogeneity and completeness of a clustering model using the V Measure Score....FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.WeakspotsDiagnosisWeakspots DiagnosisIdentifies and visualizes weak spots in a machine learning model's performance across various sections of the...TrueTrue['datasets', 'model']{'features_columns': {'type': None, 'default': None}, 'metrics': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_diagnosis', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.CalibrationCurveDriftCalibration Curve DriftEvaluates changes in probability calibration between reference and monitoring datasets....TrueTrue['datasets', 'model']{'n_bins': {'type': 'int', 'default': 10}, 'drift_pct_threshold': {'type': 'float', 'default': 20}}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassDiscriminationDriftClass Discrimination DriftCompares classification discrimination metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassificationAccuracyDriftClassification Accuracy DriftCompares classification accuracy metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ConfusionMatrixDriftConfusion Matrix DriftCompares confusion matrix metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ROCCurveDriftROC Curve DriftCompares ROC curves between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TaskTags
regressionsenstivity_analysis, tabular_data, time_series_data, model_predictions, feature_selection, correlation, regression, statsmodels, model_performance, model_training, multiclass_classification, linear_regression, data_quality, text_data, model_explainability, binary_classification, stationarity, bias_and_fairness, numerical_data, sklearn, model_selection, statistical_test, descriptive_statistics, seasonality, analysis, data_validation, data_distribution, metadata, feature_importance, visualization, forecasting, model_diagnosis, model_interpretation, unit_root_test, categorical_data, data_analysis
classificationcalibration, anomaly_detection, classification_metrics, tabular_data, time_series_data, feature_selection, correlation, statsmodels, model_performance, model_validation, model_training, classification, multiclass_classification, linear_regression, data_quality, text_data, binary_classification, threshold_optimization, bias_and_fairness, scorecard, model_comparison, numerical_data, sklearn, statistical_test, descriptive_statistics, feature_importance, data_distribution, metadata, visualization, credit_risk, AUC, logistic_regression, model_diagnosis, categorical_data, data_analysis
text_classificationmodel_performance, feature_importance, multiclass_classification, few_shot, frequency_analysis, zero_shot, text_data, visualization, llm, binary_classification, ragas, model_diagnosis, model_comparison, sklearn, nlp, retrieval_performance, tabular_data, time_series_data
text_summarizationqualitative, few_shot, frequency_analysis, embeddings, zero_shot, text_data, visualization, llm, rag_performance, ragas, retrieval_performance, nlp, dimensionality_reduction, tabular_data, time_series_data
data_validationstationarity, statsmodels, unit_root_test, time_series_data
time_series_forecastingmodel_training, data_validation, metadata, visualization, model_explainability, sklearn, model_performance, model_predictions, time_series_data
nlpdata_validation, frequency_analysis, text_data, visualization, nlp
clusteringclustering, model_performance, kmeans, sklearn
residual_analysisregression
visualizationregression
feature_extractionembeddings, text_data, visualization, llm
text_qaqualitative, embeddings, visualization, llm, rag_performance, ragas, dimensionality_reduction, retrieval_performance
text_generationqualitative, embeddings, visualization, llm, rag_performance, ragas, dimensionality_reduction, retrieval_performance
monitoringvisualization
\n" ], - "source": [ - "list_tests(filter=\"sklearn\")" + "text/plain": [ + "" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use the `task` parameter to find tests that match a specific task type, such as `classification`:\n", - "\n", - "\n" - ] - }, + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list_tasks_and_tags()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Filter tests by tags and task types\n", + "\n", + "While listing all tests is useful, you’ll often want to narrow your search. The [list_tests()](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) function supports `filter`, `task`, and `tags` parameters to assist in refining your results.\n", + "\n", + "Use the `filter` parameter to find tests that match a specific keyword, such as `sklearn`:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
IDNameDescriptionHas FigureHas TableRequired InputsParamsTagsTasks
validmind.data_validation.BivariateScatterPlotsBivariate Scatter PlotsGenerates bivariate scatterplots to visually inspect relationships between pairs of numerical predictor variables...TrueFalse['dataset']{}['tabular_data', 'numerical_data', 'visualization']['classification']
validmind.data_validation.ChiSquaredFeaturesTableChi Squared Features TableAssesses the statistical association between categorical features and a target variable using the Chi-Squared test....FalseTrue['dataset']{'p_threshold': {'type': '_empty', 'default': 0.05}}['tabular_data', 'categorical_data', 'statistical_test']['classification']
validmind.data_validation.ClassImbalanceClass ImbalanceEvaluates and quantifies class distribution imbalance in a dataset used by a machine learning model....TrueTrue['dataset']{'min_percent_threshold': {'type': 'int', 'default': 10}}['tabular_data', 'binary_classification', 'multiclass_classification', 'data_quality']['classification']
validmind.data_validation.DatasetDescriptionDataset DescriptionProvides comprehensive analysis and statistical summaries of each column in a machine learning model's dataset....FalseTrue['dataset']{}['tabular_data', 'time_series_data', 'text_data']['classification', 'regression', 'text_classification', 'text_summarization']
validmind.data_validation.DatasetSplitDataset SplitEvaluates and visualizes the distribution proportions among training, testing, and validation datasets of an ML...FalseTrue['datasets']{}['tabular_data', 'time_series_data', 'text_data']['classification', 'regression', 'text_classification', 'text_summarization']
validmind.data_validation.DescriptiveStatisticsDescriptive StatisticsPerforms a detailed descriptive statistical analysis of both numerical and categorical data within a model's...FalseTrue['dataset']{}['tabular_data', 'time_series_data', 'data_quality']['classification', 'regression']
validmind.data_validation.DuplicatesDuplicatesTests dataset for duplicate entries, ensuring model reliability via data quality verification....FalseTrue['dataset']{'min_threshold': {'type': '_empty', 'default': 1}}['tabular_data', 'data_quality', 'text_data']['classification', 'regression']
validmind.data_validation.FeatureTargetCorrelationPlotFeature Target Correlation PlotVisualizes the correlation between input features and the model's target output in a color-coded horizontal bar...TrueFalse['dataset']{'fig_height': {'type': '_empty', 'default': 600}}['tabular_data', 'visualization', 'correlation']['classification', 'regression']
validmind.data_validation.HighCardinalityHigh CardinalityAssesses the number of unique values in categorical columns to detect high cardinality and potential overfitting....FalseTrue['dataset']{'num_threshold': {'type': 'int', 'default': 100}, 'percent_threshold': {'type': 'float', 'default': 0.1}, 'threshold_type': {'type': 'str', 'default': 'percent'}}['tabular_data', 'data_quality', 'categorical_data']['classification', 'regression']
validmind.data_validation.HighPearsonCorrelationHigh Pearson CorrelationIdentifies highly correlated feature pairs in a dataset suggesting feature redundancy or multicollinearity....FalseTrue['dataset']{'max_threshold': {'type': 'float', 'default': 0.3}, 'top_n_correlations': {'type': 'int', 'default': 10}, 'feature_columns': {'type': 'list', 'default': None}}['tabular_data', 'data_quality', 'correlation']['classification', 'regression']
validmind.data_validation.IQROutliersBarPlotIQR Outliers Bar PlotVisualizes outlier distribution across percentiles in numerical data using the Interquartile Range (IQR) method....TrueFalse['dataset']{'threshold': {'type': 'float', 'default': 1.5}, 'fig_width': {'type': 'int', 'default': 800}}['tabular_data', 'visualization', 'numerical_data']['classification', 'regression']
validmind.data_validation.IQROutliersTableIQR Outliers TableDetermines and summarizes outliers in numerical features using the Interquartile Range method....FalseTrue['dataset']{'threshold': {'type': 'float', 'default': 1.5}}['tabular_data', 'numerical_data']['classification', 'regression']
validmind.data_validation.IsolationForestOutliersIsolation Forest OutliersDetects outliers in a dataset using the Isolation Forest algorithm and visualizes results through scatter plots....TrueFalse['dataset']{'random_state': {'type': 'int', 'default': 0}, 'contamination': {'type': 'float', 'default': 0.1}, 'feature_columns': {'type': 'list', 'default': None}}['tabular_data', 'anomaly_detection']['classification']
validmind.data_validation.JarqueBeraJarque BeraAssesses normality of dataset features in an ML model using the Jarque-Bera test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.data_validation.MissingValuesMissing ValuesEvaluates dataset quality by ensuring missing value ratio across all features does not exceed a set threshold....FalseTrue['dataset']{'min_threshold': {'type': 'int', 'default': 1}}['tabular_data', 'data_quality']['classification', 'regression']
validmind.data_validation.MissingValuesBarPlotMissing Values Bar PlotAssesses the percentage and distribution of missing values in the dataset via a bar plot, with emphasis on...TrueFalse['dataset']{'threshold': {'type': 'int', 'default': 80}, 'fig_height': {'type': 'int', 'default': 600}}['tabular_data', 'data_quality', 'visualization']['classification', 'regression']
validmind.data_validation.MutualInformationMutual InformationCalculates mutual information scores between features and target variable to evaluate feature relevance....TrueFalse['dataset']{'min_threshold': {'type': 'float', 'default': 0.01}, 'task': {'type': 'str', 'default': 'classification'}}['feature_selection', 'data_analysis']['classification', 'regression']
validmind.data_validation.PearsonCorrelationMatrixPearson Correlation MatrixEvaluates linear dependency between numerical variables in a dataset via a Pearson Correlation coefficient heat map....TrueFalse['dataset']{}['tabular_data', 'numerical_data', 'correlation']['classification', 'regression']
validmind.data_validation.ProtectedClassesDescriptionProtected Classes DescriptionVisualizes the distribution of protected classes in the dataset relative to the target variable...TrueTrue['dataset']{'protected_classes': {'type': '_empty', 'default': None}}['bias_and_fairness', 'descriptive_statistics']['classification', 'regression']
validmind.data_validation.RunsTestRuns TestExecutes Runs Test on ML model to detect non-random patterns in output data sequence....FalseTrue['dataset']{}['tabular_data', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.data_validation.ScatterPlotScatter PlotAssesses visual relationships, patterns, and outliers among features in a dataset through scatter plot matrices....TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.ScoreBandDefaultRatesScore Band Default RatesAnalyzes default rates and population distribution across credit score bands....FalseTrue['dataset', 'model']{'score_column': {'type': 'str', 'default': 'score'}, 'score_bands': {'type': 'list', 'default': None}}['visualization', 'credit_risk', 'scorecard']['classification']
validmind.data_validation.ShapiroWilkShapiro WilkEvaluates feature-wise normality of training data using the Shapiro-Wilk test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test']['classification', 'regression']
validmind.data_validation.SkewnessSkewnessEvaluates the skewness of numerical data in a dataset to check against a defined threshold, aiming to ensure data...FalseTrue['dataset']{'max_threshold': {'type': '_empty', 'default': 1}}['data_quality', 'tabular_data']['classification', 'regression']
validmind.data_validation.TabularCategoricalBarPlotsTabular Categorical Bar PlotsGenerates and visualizes bar plots for each category in categorical features to evaluate the dataset's composition....TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.TabularDateTimeHistogramsTabular Date Time HistogramsGenerates histograms to provide graphical insight into the distribution of time intervals in a model's datetime...TrueFalse['dataset']{}['time_series_data', 'visualization']['classification', 'regression']
validmind.data_validation.TabularDescriptionTablesTabular Description TablesSummarizes key descriptive statistics for numerical, categorical, and datetime variables in a dataset....FalseTrue['dataset']{}['tabular_data']['classification', 'regression']
validmind.data_validation.TabularNumericalHistogramsTabular Numerical HistogramsGenerates histograms for each numerical feature in a dataset to provide visual insights into data distribution and...TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.TargetRateBarPlotsTarget Rate Bar PlotsGenerates bar plots visualizing the default rates of categorical features for a classification machine learning...TrueFalse['dataset']{}['tabular_data', 'visualization', 'categorical_data']['classification']
validmind.data_validation.TooManyZeroValuesToo Many Zero ValuesIdentifies numerical columns in a dataset that contain an excessive number of zero values, defined by a threshold...FalseTrue['dataset']{'max_percent_threshold': {'type': 'float', 'default': 0.03}}['tabular_data']['regression', 'classification']
validmind.data_validation.UniqueRowsUnique RowsVerifies the diversity of the dataset by ensuring that the count of unique rows exceeds a prescribed threshold....FalseTrue['dataset']{'min_percent_threshold': {'type': 'float', 'default': 1}}['tabular_data']['regression', 'classification']
validmind.data_validation.WOEBinPlotsWOE Bin PlotsGenerates visualizations of Weight of Evidence (WoE) and Information Value (IV) for understanding predictive power...TrueFalse['dataset']{'breaks_adj': {'type': 'list', 'default': None}, 'fig_height': {'type': 'int', 'default': 600}, 'fig_width': {'type': 'int', 'default': 500}}['tabular_data', 'visualization', 'categorical_data']['classification']
validmind.data_validation.WOEBinTableWOE Bin TableAssesses the Weight of Evidence (WoE) and Information Value (IV) of each feature to evaluate its predictive power...FalseTrue['dataset']{'breaks_adj': {'type': 'list', 'default': None}}['tabular_data', 'categorical_data']['classification']
validmind.model_validation.FeaturesAUCFeatures AUCEvaluates the discriminatory power of each individual feature within a binary classification model by calculating...TrueFalse['dataset']{'fontsize': {'type': 'int', 'default': 12}, 'figure_height': {'type': 'int', 'default': 500}}['feature_importance', 'AUC', 'visualization']['classification']
validmind.model_validation.sklearn.CalibrationCurveCalibration CurveEvaluates the calibration of probability estimates by comparing predicted probabilities against observed...TrueFalse['model', 'dataset']{'n_bins': {'type': 'int', 'default': 10}}['sklearn', 'model_performance', 'classification']['classification']
validmind.model_validation.sklearn.ClassifierPerformanceClassifier PerformanceEvaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy,...FalseTrue['dataset', 'model']{'average': {'type': 'str', 'default': 'macro'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ClassifierThresholdOptimizationClassifier Threshold OptimizationAnalyzes and visualizes different threshold optimization methods for binary classification models....FalseTrue['dataset', 'model']{'methods': {'type': None, 'default': None}, 'target_recall': {'type': None, 'default': None}}['model_validation', 'threshold_optimization', 'classification_metrics']['classification']
validmind.model_validation.sklearn.ConfusionMatrixConfusion MatrixEvaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix...TrueFalse['dataset', 'model']{'threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.HyperParametersTuningHyper Parameters TuningPerforms exhaustive grid search over specified parameter ranges to find optimal model configurations...FalseTrue['model', 'dataset']{'param_grid': {'type': 'dict', 'default': None}, 'scoring': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}, 'fit_params': {'type': 'dict', 'default': None}}['sklearn', 'model_performance']['clustering', 'classification']
validmind.model_validation.sklearn.MinimumAccuracyMinimum AccuracyChecks if the model's prediction accuracy meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.7}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumF1ScoreMinimum F1 ScoreAssesses if the model's F1 score on the validation set meets a predefined minimum threshold, ensuring balanced...FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumROCAUCScoreMinimum ROCAUC ScoreValidates model by checking if the ROC AUC score meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ModelParametersModel ParametersExtracts and displays model parameters in a structured format for transparency and reproducibility....FalseTrue['model']{'model_params': {'type': None, 'default': None}}['model_training', 'metadata']['classification', 'regression']
validmind.model_validation.sklearn.ModelsPerformanceComparisonModels Performance ComparisonEvaluates and compares the performance of multiple Machine Learning models using various metrics like accuracy,...FalseTrue['dataset', 'models']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'model_comparison']['classification', 'text_classification']
validmind.model_validation.sklearn.OverfitDiagnosisOverfit DiagnosisAssesses potential overfitting in a model's predictions, identifying regions where performance between training and...TrueTrue['model', 'datasets']{'metric': {'type': 'str', 'default': None}, 'cut_off_threshold': {'type': 'float', 'default': 0.04}}['sklearn', 'binary_classification', 'multiclass_classification', 'linear_regression', 'model_diagnosis']['classification', 'regression']
validmind.model_validation.sklearn.PermutationFeatureImportancePermutation Feature ImportanceAssesses the significance of each feature in a model by evaluating the impact on model performance when feature...TrueFalse['model', 'dataset']{'fontsize': {'type': None, 'default': None}, 'figure_height': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.PopulationStabilityIndexPopulation Stability IndexAssesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across...TrueTrue['datasets', 'model']{'num_bins': {'type': 'int', 'default': 10}, 'mode': {'type': 'str', 'default': 'fixed'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.PrecisionRecallCurvePrecision Recall CurveEvaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve....TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ROCCurveROC CurveEvaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic...TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.RegressionErrorsRegression ErrorsAssesses the performance and error distribution of a regression model using various error metrics....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance']['regression', 'classification']
validmind.model_validation.sklearn.RobustnessDiagnosisRobustness DiagnosisAssesses the robustness of a machine learning model by evaluating performance decay under noisy conditions....TrueTrue['datasets', 'model']{'metric': {'type': 'str', 'default': None}, 'scaling_factor_std_dev_list': {'type': None, 'default': [0.1, 0.2, 0.3, 0.4, 0.5]}, 'performance_decay_threshold': {'type': 'float', 'default': 0.05}}['sklearn', 'model_diagnosis', 'visualization']['classification', 'regression']
validmind.model_validation.sklearn.SHAPGlobalImportanceSHAP Global ImportanceEvaluates and visualizes global feature importance using SHAP values for model explanation and risk identification....FalseTrue['model', 'dataset']{'kernel_explainer_samples': {'type': 'int', 'default': 10}, 'tree_or_linear_explainer_samples': {'type': 'int', 'default': 200}, 'class_of_interest': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ScoreProbabilityAlignmentScore Probability AlignmentAnalyzes the alignment between credit scores and predicted probabilities....TrueTrue['model', 'dataset']{'score_column': {'type': 'str', 'default': 'score'}, 'n_bins': {'type': 'int', 'default': 10}}['visualization', 'credit_risk', 'calibration']['classification']
validmind.model_validation.sklearn.TrainingTestDegradationTraining Test DegradationTests if model performance degradation between training and test datasets exceeds a predefined threshold....FalseTrue['datasets', 'model']{'max_threshold': {'type': 'float', 'default': 0.1}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.WeakspotsDiagnosisWeakspots DiagnosisIdentifies and visualizes weak spots in a machine learning model's performance across various sections of the...TrueTrue['datasets', 'model']{'features_columns': {'type': None, 'default': None}, 'metrics': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_diagnosis', 'visualization']['classification', 'text_classification']
validmind.model_validation.statsmodels.CumulativePredictionProbabilitiesCumulative Prediction ProbabilitiesVisualizes cumulative probabilities of positive and negative classes for both training and testing in classification models....TrueFalse['dataset', 'model']{'title': {'type': 'str', 'default': 'Cumulative Probabilities'}}['visualization', 'credit_risk']['classification']
validmind.model_validation.statsmodels.GINITableGINI TableEvaluates classification model performance using AUC, GINI, and KS metrics for training and test datasets....FalseTrue['dataset', 'model']{}['model_performance']['classification']
validmind.model_validation.statsmodels.KolmogorovSmirnovKolmogorov SmirnovAssesses whether each feature in the dataset aligns with a normal distribution using the Kolmogorov-Smirnov test....FalseTrue['model', 'dataset']{'dist': {'type': 'str', 'default': 'norm'}}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.model_validation.statsmodels.LillieforsLillieforsAssesses the normality of feature distributions in an ML model's training dataset using the Lilliefors test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.model_validation.statsmodels.PredictionProbabilitiesHistogramPrediction Probabilities HistogramAssesses the predictive probability distribution for binary classification to evaluate model performance and...TrueFalse['dataset', 'model']{'title': {'type': 'str', 'default': 'Histogram of Predictive Probabilities'}}['visualization', 'credit_risk']['classification']
validmind.model_validation.statsmodels.ScorecardHistogramScorecard HistogramThe Scorecard Histogram test evaluates the distribution of credit scores between default and non-default instances,...TrueFalse['dataset']{'title': {'type': 'str', 'default': 'Histogram of Scores'}, 'score_column': {'type': 'str', 'default': 'score'}}['visualization', 'credit_risk', 'logistic_regression']['classification']
validmind.ongoing_monitoring.CalibrationCurveDriftCalibration Curve DriftEvaluates changes in probability calibration between reference and monitoring datasets....TrueTrue['datasets', 'model']{'n_bins': {'type': 'int', 'default': 10}, 'drift_pct_threshold': {'type': 'float', 'default': 20}}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassDiscriminationDriftClass Discrimination DriftCompares classification discrimination metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassImbalanceDriftClass Imbalance DriftEvaluates drift in class distribution between reference and monitoring datasets....TrueTrue['datasets']{'drift_pct_threshold': {'type': 'float', 'default': 5.0}, 'title': {'type': 'str', 'default': 'Class Distribution Drift'}}['tabular_data', 'binary_classification', 'multiclass_classification']['classification']
validmind.ongoing_monitoring.ClassificationAccuracyDriftClassification Accuracy DriftCompares classification accuracy metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ConfusionMatrixDriftConfusion Matrix DriftCompares confusion matrix metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.CumulativePredictionProbabilitiesDriftCumulative Prediction Probabilities DriftCompares cumulative prediction probability distributions between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['visualization', 'credit_risk']['classification']
validmind.ongoing_monitoring.PredictionProbabilitiesHistogramDriftPrediction Probabilities Histogram DriftCompares prediction probability distributions between reference and monitoring datasets....TrueTrue['datasets', 'model']{'title': {'type': '_empty', 'default': 'Prediction Probabilities Histogram Drift'}, 'drift_pct_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk']['classification']
validmind.ongoing_monitoring.ROCCurveDriftROC Curve DriftCompares ROC curves between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ScoreBandsDriftScore Bands DriftAnalyzes drift in population distribution and default rates across score bands....FalseTrue['datasets', 'model']{'score_column': {'type': 'str', 'default': 'score'}, 'score_bands': {'type': 'list', 'default': None}, 'drift_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk', 'scorecard']['classification']
validmind.ongoing_monitoring.ScorecardHistogramDriftScorecard Histogram DriftCompares score distributions between reference and monitoring datasets for each class....TrueTrue['datasets']{'score_column': {'type': 'str', 'default': 'score'}, 'title': {'type': 'str', 'default': 'Scorecard Histogram Drift'}, 'drift_pct_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk', 'logistic_regression']['classification']
validmind.unit_metrics.classification.AccuracyAccuracyCalculates the accuracy of a modelFalseFalse['dataset', 'model']{}['classification']['classification']
validmind.unit_metrics.classification.F1F1Calculates the F1 score for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.PrecisionPrecisionCalculates the precision for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.ROC_AUCROC AUCCalculates the ROC AUC for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.RecallRecallCalculates the recall for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDNameDescriptionHas FigureHas TableRequired InputsParamsTagsTasks
validmind.model_validation.ClusterSizeDistributionCluster Size DistributionAssesses the performance of clustering models by comparing the distribution of cluster sizes in model predictions...TrueFalse['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.TimeSeriesR2SquareBySegmentsTime Series R2 Square By SegmentsEvaluates the R-Squared values of regression models over specified time segments in time series data to assess...TrueTrue['dataset', 'model']{'segments': {'type': None, 'default': None}}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.AdjustedMutualInformationAdjusted Mutual InformationEvaluates clustering model performance by measuring mutual information between true and predicted labels, adjusting...FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.AdjustedRandIndexAdjusted Rand IndexMeasures the similarity between two data clusters using the Adjusted Rand Index (ARI) metric in clustering machine...FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.CalibrationCurveCalibration CurveEvaluates the calibration of probability estimates by comparing predicted probabilities against observed...TrueFalse['model', 'dataset']{'n_bins': {'type': 'int', 'default': 10}}['sklearn', 'model_performance', 'classification']['classification']
validmind.model_validation.sklearn.ClassifierPerformanceClassifier PerformanceEvaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy,...FalseTrue['dataset', 'model']{'average': {'type': 'str', 'default': 'macro'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ClassifierThresholdOptimizationClassifier Threshold OptimizationAnalyzes and visualizes different threshold optimization methods for binary classification models....FalseTrue['dataset', 'model']{'methods': {'type': None, 'default': None}, 'target_recall': {'type': None, 'default': None}}['model_validation', 'threshold_optimization', 'classification_metrics']['classification']
validmind.model_validation.sklearn.ClusterCosineSimilarityCluster Cosine SimilarityMeasures the intra-cluster similarity of a clustering model using cosine similarity....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.ClusterPerformanceMetricsCluster Performance MetricsEvaluates the performance of clustering machine learning models using multiple established metrics....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.CompletenessScoreCompleteness ScoreEvaluates a clustering model's capacity to categorize instances from a single class into the same cluster....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance', 'clustering']['clustering']
validmind.model_validation.sklearn.ConfusionMatrixConfusion MatrixEvaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix...TrueFalse['dataset', 'model']{'threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.FeatureImportanceFeature ImportanceCompute feature importance scores for a given model and generate a summary table...FalseTrue['dataset', 'model']{'num_features': {'type': 'int', 'default': 3}}['model_explainability', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.FowlkesMallowsScoreFowlkes Mallows ScoreEvaluates the similarity between predicted and actual cluster assignments in a model using the Fowlkes-Mallows...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.HomogeneityScoreHomogeneity ScoreAssesses clustering homogeneity by comparing true and predicted labels, scoring from 0 (heterogeneous) to 1...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.HyperParametersTuningHyper Parameters TuningPerforms exhaustive grid search over specified parameter ranges to find optimal model configurations...FalseTrue['model', 'dataset']{'param_grid': {'type': 'dict', 'default': None}, 'scoring': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}, 'fit_params': {'type': 'dict', 'default': None}}['sklearn', 'model_performance']['clustering', 'classification']
validmind.model_validation.sklearn.KMeansClustersOptimizationK Means Clusters OptimizationOptimizes the number of clusters in K-means models using Elbow and Silhouette methods....TrueFalse['model', 'dataset']{'n_clusters': {'type': None, 'default': None}}['sklearn', 'model_performance', 'kmeans']['clustering']
validmind.model_validation.sklearn.MinimumAccuracyMinimum AccuracyChecks if the model's prediction accuracy meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.7}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumF1ScoreMinimum F1 ScoreAssesses if the model's F1 score on the validation set meets a predefined minimum threshold, ensuring balanced...FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumROCAUCScoreMinimum ROCAUC ScoreValidates model by checking if the ROC AUC score meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ModelParametersModel ParametersExtracts and displays model parameters in a structured format for transparency and reproducibility....FalseTrue['model']{'model_params': {'type': None, 'default': None}}['model_training', 'metadata']['classification', 'regression']
validmind.model_validation.sklearn.ModelsPerformanceComparisonModels Performance ComparisonEvaluates and compares the performance of multiple Machine Learning models using various metrics like accuracy,...FalseTrue['dataset', 'models']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'model_comparison']['classification', 'text_classification']
validmind.model_validation.sklearn.OverfitDiagnosisOverfit DiagnosisAssesses potential overfitting in a model's predictions, identifying regions where performance between training and...TrueTrue['model', 'datasets']{'metric': {'type': 'str', 'default': None}, 'cut_off_threshold': {'type': 'float', 'default': 0.04}}['sklearn', 'binary_classification', 'multiclass_classification', 'linear_regression', 'model_diagnosis']['classification', 'regression']
validmind.model_validation.sklearn.PermutationFeatureImportancePermutation Feature ImportanceAssesses the significance of each feature in a model by evaluating the impact on model performance when feature...TrueFalse['model', 'dataset']{'fontsize': {'type': None, 'default': None}, 'figure_height': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.PopulationStabilityIndexPopulation Stability IndexAssesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across...TrueTrue['datasets', 'model']{'num_bins': {'type': 'int', 'default': 10}, 'mode': {'type': 'str', 'default': 'fixed'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.PrecisionRecallCurvePrecision Recall CurveEvaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve....TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ROCCurveROC CurveEvaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic...TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.RegressionErrorsRegression ErrorsAssesses the performance and error distribution of a regression model using various error metrics....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance']['regression', 'classification']
validmind.model_validation.sklearn.RegressionErrorsComparisonRegression Errors ComparisonAssesses multiple regression error metrics to compare model performance across different datasets, emphasizing...FalseTrue['datasets', 'models']{}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.RegressionPerformanceRegression PerformanceEvaluates the performance of a regression model using five different metrics: MAE, MSE, RMSE, MAPE, and MBD....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance']['regression']
validmind.model_validation.sklearn.RegressionR2SquareRegression R2 SquareAssesses the overall goodness-of-fit of a regression model by evaluating R-squared (R2) and Adjusted R-squared (Adj...FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['regression']
validmind.model_validation.sklearn.RegressionR2SquareComparisonRegression R2 Square ComparisonCompares R-Squared and Adjusted R-Squared values for different regression models across multiple datasets to assess...FalseTrue['datasets', 'models']{}['model_performance', 'sklearn']['regression', 'time_series_forecasting']
validmind.model_validation.sklearn.RobustnessDiagnosisRobustness DiagnosisAssesses the robustness of a machine learning model by evaluating performance decay under noisy conditions....TrueTrue['datasets', 'model']{'metric': {'type': 'str', 'default': None}, 'scaling_factor_std_dev_list': {'type': None, 'default': [0.1, 0.2, 0.3, 0.4, 0.5]}, 'performance_decay_threshold': {'type': 'float', 'default': 0.05}}['sklearn', 'model_diagnosis', 'visualization']['classification', 'regression']
validmind.model_validation.sklearn.SHAPGlobalImportanceSHAP Global ImportanceEvaluates and visualizes global feature importance using SHAP values for model explanation and risk identification....FalseTrue['model', 'dataset']{'kernel_explainer_samples': {'type': 'int', 'default': 10}, 'tree_or_linear_explainer_samples': {'type': 'int', 'default': 200}, 'class_of_interest': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ScoreProbabilityAlignmentScore Probability AlignmentAnalyzes the alignment between credit scores and predicted probabilities....TrueTrue['model', 'dataset']{'score_column': {'type': 'str', 'default': 'score'}, 'n_bins': {'type': 'int', 'default': 10}}['visualization', 'credit_risk', 'calibration']['classification']
validmind.model_validation.sklearn.SilhouettePlotSilhouette PlotCalculates and visualizes Silhouette Score, assessing the degree of data point suitability to its cluster in ML...TrueTrue['model', 'dataset']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.TrainingTestDegradationTraining Test DegradationTests if model performance degradation between training and test datasets exceeds a predefined threshold....FalseTrue['datasets', 'model']{'max_threshold': {'type': 'float', 'default': 0.1}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.VMeasureV MeasureEvaluates homogeneity and completeness of a clustering model using the V Measure Score....FalseTrue['dataset', 'model']{}['sklearn', 'model_performance']['clustering']
validmind.model_validation.sklearn.WeakspotsDiagnosisWeakspots DiagnosisIdentifies and visualizes weak spots in a machine learning model's performance across various sections of the...TrueTrue['datasets', 'model']{'features_columns': {'type': None, 'default': None}, 'metrics': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_diagnosis', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.CalibrationCurveDriftCalibration Curve DriftEvaluates changes in probability calibration between reference and monitoring datasets....TrueTrue['datasets', 'model']{'n_bins': {'type': 'int', 'default': 10}, 'drift_pct_threshold': {'type': 'float', 'default': 20}}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassDiscriminationDriftClass Discrimination DriftCompares classification discrimination metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassificationAccuracyDriftClassification Accuracy DriftCompares classification accuracy metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ConfusionMatrixDriftConfusion Matrix DriftCompares confusion matrix metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ROCCurveDriftROC Curve DriftCompares ROC curves between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
\n" ], - "source": [ - "list_tests(task=\"classification\")" + "text/plain": [ + "" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use the `tags` parameter to find tests based on their tags, such as `model_performance` or `visualization`:" - ] - }, + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list_tests(filter=\"sklearn\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use the `task` parameter to find tests that match a specific task type, such as `classification`:\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
IDNameDescriptionHas FigureHas TableRequired InputsParamsTagsTasks
validmind.model_validation.RegressionResidualsPlotRegression Residuals PlotEvaluates regression model performance using residual distribution and actual vs. predicted plots....TrueFalse['model', 'dataset']{'bin_size': {'type': 'float', 'default': 0.1}}['model_performance', 'visualization']['regression']
validmind.model_validation.sklearn.ConfusionMatrixConfusion MatrixEvaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix...TrueFalse['dataset', 'model']{'threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.PrecisionRecallCurvePrecision Recall CurveEvaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve....TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ROCCurveROC CurveEvaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic...TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.TrainingTestDegradationTraining Test DegradationTests if model performance degradation between training and test datasets exceeds a predefined threshold....FalseTrue['datasets', 'model']{'max_threshold': {'type': 'float', 'default': 0.1}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.CalibrationCurveDriftCalibration Curve DriftEvaluates changes in probability calibration between reference and monitoring datasets....TrueTrue['datasets', 'model']{'n_bins': {'type': 'int', 'default': 10}, 'drift_pct_threshold': {'type': 'float', 'default': 20}}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ROCCurveDriftROC Curve DriftCompares ROC curves between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDNameDescriptionHas FigureHas TableRequired InputsParamsTagsTasks
validmind.data_validation.BivariateScatterPlotsBivariate Scatter PlotsGenerates bivariate scatterplots to visually inspect relationships between pairs of numerical predictor variables...TrueFalse['dataset']{}['tabular_data', 'numerical_data', 'visualization']['classification']
validmind.data_validation.ChiSquaredFeaturesTableChi Squared Features TableAssesses the statistical association between categorical features and a target variable using the Chi-Squared test....FalseTrue['dataset']{'p_threshold': {'type': '_empty', 'default': 0.05}}['tabular_data', 'categorical_data', 'statistical_test']['classification']
validmind.data_validation.ClassImbalanceClass ImbalanceEvaluates and quantifies class distribution imbalance in a dataset used by a machine learning model....TrueTrue['dataset']{'min_percent_threshold': {'type': 'int', 'default': 10}}['tabular_data', 'binary_classification', 'multiclass_classification', 'data_quality']['classification']
validmind.data_validation.DatasetDescriptionDataset DescriptionProvides comprehensive analysis and statistical summaries of each column in a machine learning model's dataset....FalseTrue['dataset']{}['tabular_data', 'time_series_data', 'text_data']['classification', 'regression', 'text_classification', 'text_summarization']
validmind.data_validation.DatasetSplitDataset SplitEvaluates and visualizes the distribution proportions among training, testing, and validation datasets of an ML...FalseTrue['datasets']{}['tabular_data', 'time_series_data', 'text_data']['classification', 'regression', 'text_classification', 'text_summarization']
validmind.data_validation.DescriptiveStatisticsDescriptive StatisticsPerforms a detailed descriptive statistical analysis of both numerical and categorical data within a model's...FalseTrue['dataset']{}['tabular_data', 'time_series_data', 'data_quality']['classification', 'regression']
validmind.data_validation.DuplicatesDuplicatesTests dataset for duplicate entries, ensuring model reliability via data quality verification....FalseTrue['dataset']{'min_threshold': {'type': '_empty', 'default': 1}}['tabular_data', 'data_quality', 'text_data']['classification', 'regression']
validmind.data_validation.FeatureTargetCorrelationPlotFeature Target Correlation PlotVisualizes the correlation between input features and the model's target output in a color-coded horizontal bar...TrueFalse['dataset']{'fig_height': {'type': '_empty', 'default': 600}}['tabular_data', 'visualization', 'correlation']['classification', 'regression']
validmind.data_validation.HighCardinalityHigh CardinalityAssesses the number of unique values in categorical columns to detect high cardinality and potential overfitting....FalseTrue['dataset']{'num_threshold': {'type': 'int', 'default': 100}, 'percent_threshold': {'type': 'float', 'default': 0.1}, 'threshold_type': {'type': 'str', 'default': 'percent'}}['tabular_data', 'data_quality', 'categorical_data']['classification', 'regression']
validmind.data_validation.HighPearsonCorrelationHigh Pearson CorrelationIdentifies highly correlated feature pairs in a dataset suggesting feature redundancy or multicollinearity....FalseTrue['dataset']{'max_threshold': {'type': 'float', 'default': 0.3}, 'top_n_correlations': {'type': 'int', 'default': 10}, 'feature_columns': {'type': 'list', 'default': None}}['tabular_data', 'data_quality', 'correlation']['classification', 'regression']
validmind.data_validation.IQROutliersBarPlotIQR Outliers Bar PlotVisualizes outlier distribution across percentiles in numerical data using the Interquartile Range (IQR) method....TrueFalse['dataset']{'threshold': {'type': 'float', 'default': 1.5}, 'fig_width': {'type': 'int', 'default': 800}}['tabular_data', 'visualization', 'numerical_data']['classification', 'regression']
validmind.data_validation.IQROutliersTableIQR Outliers TableDetermines and summarizes outliers in numerical features using the Interquartile Range method....FalseTrue['dataset']{'threshold': {'type': 'float', 'default': 1.5}}['tabular_data', 'numerical_data']['classification', 'regression']
validmind.data_validation.IsolationForestOutliersIsolation Forest OutliersDetects outliers in a dataset using the Isolation Forest algorithm and visualizes results through scatter plots....TrueFalse['dataset']{'random_state': {'type': 'int', 'default': 0}, 'contamination': {'type': 'float', 'default': 0.1}, 'feature_columns': {'type': 'list', 'default': None}}['tabular_data', 'anomaly_detection']['classification']
validmind.data_validation.JarqueBeraJarque BeraAssesses normality of dataset features in an ML model using the Jarque-Bera test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.data_validation.MissingValuesMissing ValuesEvaluates dataset quality by ensuring missing value ratio across all features does not exceed a set threshold....FalseTrue['dataset']{'min_threshold': {'type': 'int', 'default': 1}}['tabular_data', 'data_quality']['classification', 'regression']
validmind.data_validation.MissingValuesBarPlotMissing Values Bar PlotAssesses the percentage and distribution of missing values in the dataset via a bar plot, with emphasis on...TrueFalse['dataset']{'threshold': {'type': 'int', 'default': 80}, 'fig_height': {'type': 'int', 'default': 600}}['tabular_data', 'data_quality', 'visualization']['classification', 'regression']
validmind.data_validation.MutualInformationMutual InformationCalculates mutual information scores between features and target variable to evaluate feature relevance....TrueFalse['dataset']{'min_threshold': {'type': 'float', 'default': 0.01}, 'task': {'type': 'str', 'default': 'classification'}}['feature_selection', 'data_analysis']['classification', 'regression']
validmind.data_validation.PearsonCorrelationMatrixPearson Correlation MatrixEvaluates linear dependency between numerical variables in a dataset via a Pearson Correlation coefficient heat map....TrueFalse['dataset']{}['tabular_data', 'numerical_data', 'correlation']['classification', 'regression']
validmind.data_validation.ProtectedClassesDescriptionProtected Classes DescriptionVisualizes the distribution of protected classes in the dataset relative to the target variable...TrueTrue['dataset']{'protected_classes': {'type': '_empty', 'default': None}}['bias_and_fairness', 'descriptive_statistics']['classification', 'regression']
validmind.data_validation.RunsTestRuns TestExecutes Runs Test on ML model to detect non-random patterns in output data sequence....FalseTrue['dataset']{}['tabular_data', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.data_validation.ScatterPlotScatter PlotAssesses visual relationships, patterns, and outliers among features in a dataset through scatter plot matrices....TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.ScoreBandDefaultRatesScore Band Default RatesAnalyzes default rates and population distribution across credit score bands....FalseTrue['dataset', 'model']{'score_column': {'type': 'str', 'default': 'score'}, 'score_bands': {'type': 'list', 'default': None}}['visualization', 'credit_risk', 'scorecard']['classification']
validmind.data_validation.ShapiroWilkShapiro WilkEvaluates feature-wise normality of training data using the Shapiro-Wilk test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test']['classification', 'regression']
validmind.data_validation.SkewnessSkewnessEvaluates the skewness of numerical data in a dataset to check against a defined threshold, aiming to ensure data...FalseTrue['dataset']{'max_threshold': {'type': '_empty', 'default': 1}}['data_quality', 'tabular_data']['classification', 'regression']
validmind.data_validation.TabularCategoricalBarPlotsTabular Categorical Bar PlotsGenerates and visualizes bar plots for each category in categorical features to evaluate the dataset's composition....TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.TabularDateTimeHistogramsTabular Date Time HistogramsGenerates histograms to provide graphical insight into the distribution of time intervals in a model's datetime...TrueFalse['dataset']{}['time_series_data', 'visualization']['classification', 'regression']
validmind.data_validation.TabularDescriptionTablesTabular Description TablesSummarizes key descriptive statistics for numerical, categorical, and datetime variables in a dataset....FalseTrue['dataset']{}['tabular_data']['classification', 'regression']
validmind.data_validation.TabularNumericalHistogramsTabular Numerical HistogramsGenerates histograms for each numerical feature in a dataset to provide visual insights into data distribution and...TrueFalse['dataset']{}['tabular_data', 'visualization']['classification', 'regression']
validmind.data_validation.TargetRateBarPlotsTarget Rate Bar PlotsGenerates bar plots visualizing the default rates of categorical features for a classification machine learning...TrueFalse['dataset']{}['tabular_data', 'visualization', 'categorical_data']['classification']
validmind.data_validation.TooManyZeroValuesToo Many Zero ValuesIdentifies numerical columns in a dataset that contain an excessive number of zero values, defined by a threshold...FalseTrue['dataset']{'max_percent_threshold': {'type': 'float', 'default': 0.03}}['tabular_data']['regression', 'classification']
validmind.data_validation.UniqueRowsUnique RowsVerifies the diversity of the dataset by ensuring that the count of unique rows exceeds a prescribed threshold....FalseTrue['dataset']{'min_percent_threshold': {'type': 'float', 'default': 1}}['tabular_data']['regression', 'classification']
validmind.data_validation.WOEBinPlotsWOE Bin PlotsGenerates visualizations of Weight of Evidence (WoE) and Information Value (IV) for understanding predictive power...TrueFalse['dataset']{'breaks_adj': {'type': 'list', 'default': None}, 'fig_height': {'type': 'int', 'default': 600}, 'fig_width': {'type': 'int', 'default': 500}}['tabular_data', 'visualization', 'categorical_data']['classification']
validmind.data_validation.WOEBinTableWOE Bin TableAssesses the Weight of Evidence (WoE) and Information Value (IV) of each feature to evaluate its predictive power...FalseTrue['dataset']{'breaks_adj': {'type': 'list', 'default': None}}['tabular_data', 'categorical_data']['classification']
validmind.model_validation.FeaturesAUCFeatures AUCEvaluates the discriminatory power of each individual feature within a binary classification model by calculating...TrueFalse['dataset']{'fontsize': {'type': 'int', 'default': 12}, 'figure_height': {'type': 'int', 'default': 500}}['feature_importance', 'AUC', 'visualization']['classification']
validmind.model_validation.sklearn.CalibrationCurveCalibration CurveEvaluates the calibration of probability estimates by comparing predicted probabilities against observed...TrueFalse['model', 'dataset']{'n_bins': {'type': 'int', 'default': 10}}['sklearn', 'model_performance', 'classification']['classification']
validmind.model_validation.sklearn.ClassifierPerformanceClassifier PerformanceEvaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy,...FalseTrue['dataset', 'model']{'average': {'type': 'str', 'default': 'macro'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ClassifierThresholdOptimizationClassifier Threshold OptimizationAnalyzes and visualizes different threshold optimization methods for binary classification models....FalseTrue['dataset', 'model']{'methods': {'type': None, 'default': None}, 'target_recall': {'type': None, 'default': None}}['model_validation', 'threshold_optimization', 'classification_metrics']['classification']
validmind.model_validation.sklearn.ConfusionMatrixConfusion MatrixEvaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix...TrueFalse['dataset', 'model']{'threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.HyperParametersTuningHyper Parameters TuningPerforms exhaustive grid search over specified parameter ranges to find optimal model configurations...FalseTrue['model', 'dataset']{'param_grid': {'type': 'dict', 'default': None}, 'scoring': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}, 'fit_params': {'type': 'dict', 'default': None}}['sklearn', 'model_performance']['clustering', 'classification']
validmind.model_validation.sklearn.MinimumAccuracyMinimum AccuracyChecks if the model's prediction accuracy meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.7}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumF1ScoreMinimum F1 ScoreAssesses if the model's F1 score on the validation set meets a predefined minimum threshold, ensuring balanced...FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.MinimumROCAUCScoreMinimum ROCAUC ScoreValidates model by checking if the ROC AUC score meets or surpasses a specified threshold....FalseTrue['dataset', 'model']{'min_threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.ModelParametersModel ParametersExtracts and displays model parameters in a structured format for transparency and reproducibility....FalseTrue['model']{'model_params': {'type': None, 'default': None}}['model_training', 'metadata']['classification', 'regression']
validmind.model_validation.sklearn.ModelsPerformanceComparisonModels Performance ComparisonEvaluates and compares the performance of multiple Machine Learning models using various metrics like accuracy,...FalseTrue['dataset', 'models']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'model_comparison']['classification', 'text_classification']
validmind.model_validation.sklearn.OverfitDiagnosisOverfit DiagnosisAssesses potential overfitting in a model's predictions, identifying regions where performance between training and...TrueTrue['model', 'datasets']{'metric': {'type': 'str', 'default': None}, 'cut_off_threshold': {'type': 'float', 'default': 0.04}}['sklearn', 'binary_classification', 'multiclass_classification', 'linear_regression', 'model_diagnosis']['classification', 'regression']
validmind.model_validation.sklearn.PermutationFeatureImportancePermutation Feature ImportanceAssesses the significance of each feature in a model by evaluating the impact on model performance when feature...TrueFalse['model', 'dataset']{'fontsize': {'type': None, 'default': None}, 'figure_height': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.PopulationStabilityIndexPopulation Stability IndexAssesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across...TrueTrue['datasets', 'model']{'num_bins': {'type': 'int', 'default': 10}, 'mode': {'type': 'str', 'default': 'fixed'}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.model_validation.sklearn.PrecisionRecallCurvePrecision Recall CurveEvaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve....TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ROCCurveROC CurveEvaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic...TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.RegressionErrorsRegression ErrorsAssesses the performance and error distribution of a regression model using various error metrics....FalseTrue['model', 'dataset']{}['sklearn', 'model_performance']['regression', 'classification']
validmind.model_validation.sklearn.RobustnessDiagnosisRobustness DiagnosisAssesses the robustness of a machine learning model by evaluating performance decay under noisy conditions....TrueTrue['datasets', 'model']{'metric': {'type': 'str', 'default': None}, 'scaling_factor_std_dev_list': {'type': None, 'default': [0.1, 0.2, 0.3, 0.4, 0.5]}, 'performance_decay_threshold': {'type': 'float', 'default': 0.05}}['sklearn', 'model_diagnosis', 'visualization']['classification', 'regression']
validmind.model_validation.sklearn.SHAPGlobalImportanceSHAP Global ImportanceEvaluates and visualizes global feature importance using SHAP values for model explanation and risk identification....FalseTrue['model', 'dataset']{'kernel_explainer_samples': {'type': 'int', 'default': 10}, 'tree_or_linear_explainer_samples': {'type': 'int', 'default': 200}, 'class_of_interest': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ScoreProbabilityAlignmentScore Probability AlignmentAnalyzes the alignment between credit scores and predicted probabilities....TrueTrue['model', 'dataset']{'score_column': {'type': 'str', 'default': 'score'}, 'n_bins': {'type': 'int', 'default': 10}}['visualization', 'credit_risk', 'calibration']['classification']
validmind.model_validation.sklearn.TrainingTestDegradationTraining Test DegradationTests if model performance degradation between training and test datasets exceeds a predefined threshold....FalseTrue['datasets', 'model']{'max_threshold': {'type': 'float', 'default': 0.1}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.WeakspotsDiagnosisWeakspots DiagnosisIdentifies and visualizes weak spots in a machine learning model's performance across various sections of the...TrueTrue['datasets', 'model']{'features_columns': {'type': None, 'default': None}, 'metrics': {'type': None, 'default': None}, 'thresholds': {'type': None, 'default': None}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_diagnosis', 'visualization']['classification', 'text_classification']
validmind.model_validation.statsmodels.CumulativePredictionProbabilitiesCumulative Prediction ProbabilitiesVisualizes cumulative probabilities of positive and negative classes for both training and testing in classification models....TrueFalse['dataset', 'model']{'title': {'type': 'str', 'default': 'Cumulative Probabilities'}}['visualization', 'credit_risk']['classification']
validmind.model_validation.statsmodels.GINITableGINI TableEvaluates classification model performance using AUC, GINI, and KS metrics for training and test datasets....FalseTrue['dataset', 'model']{}['model_performance']['classification']
validmind.model_validation.statsmodels.KolmogorovSmirnovKolmogorov SmirnovAssesses whether each feature in the dataset aligns with a normal distribution using the Kolmogorov-Smirnov test....FalseTrue['model', 'dataset']{'dist': {'type': 'str', 'default': 'norm'}}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.model_validation.statsmodels.LillieforsLillieforsAssesses the normality of feature distributions in an ML model's training dataset using the Lilliefors test....FalseTrue['dataset']{}['tabular_data', 'data_distribution', 'statistical_test', 'statsmodels']['classification', 'regression']
validmind.model_validation.statsmodels.PredictionProbabilitiesHistogramPrediction Probabilities HistogramAssesses the predictive probability distribution for binary classification to evaluate model performance and...TrueFalse['dataset', 'model']{'title': {'type': 'str', 'default': 'Histogram of Predictive Probabilities'}}['visualization', 'credit_risk']['classification']
validmind.model_validation.statsmodels.ScorecardHistogramScorecard HistogramThe Scorecard Histogram test evaluates the distribution of credit scores between default and non-default instances,...TrueFalse['dataset']{'title': {'type': 'str', 'default': 'Histogram of Scores'}, 'score_column': {'type': 'str', 'default': 'score'}}['visualization', 'credit_risk', 'logistic_regression']['classification']
validmind.ongoing_monitoring.CalibrationCurveDriftCalibration Curve DriftEvaluates changes in probability calibration between reference and monitoring datasets....TrueTrue['datasets', 'model']{'n_bins': {'type': 'int', 'default': 10}, 'drift_pct_threshold': {'type': 'float', 'default': 20}}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassDiscriminationDriftClass Discrimination DriftCompares classification discrimination metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ClassImbalanceDriftClass Imbalance DriftEvaluates drift in class distribution between reference and monitoring datasets....TrueTrue['datasets']{'drift_pct_threshold': {'type': 'float', 'default': 5.0}, 'title': {'type': 'str', 'default': 'Class Distribution Drift'}}['tabular_data', 'binary_classification', 'multiclass_classification']['classification']
validmind.ongoing_monitoring.ClassificationAccuracyDriftClassification Accuracy DriftCompares classification accuracy metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.ConfusionMatrixDriftConfusion Matrix DriftCompares confusion matrix metrics between reference and monitoring datasets....FalseTrue['datasets', 'model']{'drift_pct_threshold': {'type': '_empty', 'default': 20}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance']['classification', 'text_classification']
validmind.ongoing_monitoring.CumulativePredictionProbabilitiesDriftCumulative Prediction Probabilities DriftCompares cumulative prediction probability distributions between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['visualization', 'credit_risk']['classification']
validmind.ongoing_monitoring.PredictionProbabilitiesHistogramDriftPrediction Probabilities Histogram DriftCompares prediction probability distributions between reference and monitoring datasets....TrueTrue['datasets', 'model']{'title': {'type': '_empty', 'default': 'Prediction Probabilities Histogram Drift'}, 'drift_pct_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk']['classification']
validmind.ongoing_monitoring.ROCCurveDriftROC Curve DriftCompares ROC curves between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ScoreBandsDriftScore Bands DriftAnalyzes drift in population distribution and default rates across score bands....FalseTrue['datasets', 'model']{'score_column': {'type': 'str', 'default': 'score'}, 'score_bands': {'type': 'list', 'default': None}, 'drift_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk', 'scorecard']['classification']
validmind.ongoing_monitoring.ScorecardHistogramDriftScorecard Histogram DriftCompares score distributions between reference and monitoring datasets for each class....TrueTrue['datasets']{'score_column': {'type': 'str', 'default': 'score'}, 'title': {'type': 'str', 'default': 'Scorecard Histogram Drift'}, 'drift_pct_threshold': {'type': 'float', 'default': 20.0}}['visualization', 'credit_risk', 'logistic_regression']['classification']
validmind.unit_metrics.classification.AccuracyAccuracyCalculates the accuracy of a modelFalseFalse['dataset', 'model']{}['classification']['classification']
validmind.unit_metrics.classification.F1F1Calculates the F1 score for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.PrecisionPrecisionCalculates the precision for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.ROC_AUCROC AUCCalculates the ROC AUC for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
validmind.unit_metrics.classification.RecallRecallCalculates the recall for a classification model.FalseFalse['model', 'dataset']{}['classification']['classification']
\n" ], - "source": [ - "list_tests(tags=[\"model_performance\", \"visualization\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use `filter`, `task`, and `tags` together to create more specific queries.\n", - "\n", - "For example, apply all three to find tests compatible with `sklearn` models, designed for `classification` tasks:" + "text/plain": [ + "" ] - }, + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list_tests(task=\"classification\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use the `tags` parameter to find tests based on their tags, such as `model_performance` or `visualization`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
IDNameDescriptionHas FigureHas TableRequired InputsParamsTagsTasks
validmind.model_validation.sklearn.ConfusionMatrixConfusion MatrixEvaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix...TrueFalse['dataset', 'model']{'threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.PrecisionRecallCurvePrecision Recall CurveEvaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve....TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ROCCurveROC CurveEvaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic...TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.TrainingTestDegradationTraining Test DegradationTests if model performance degradation between training and test datasets exceeds a predefined threshold....FalseTrue['datasets', 'model']{'max_threshold': {'type': 'float', 'default': 0.1}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.CalibrationCurveDriftCalibration Curve DriftEvaluates changes in probability calibration between reference and monitoring datasets....TrueTrue['datasets', 'model']{'n_bins': {'type': 'int', 'default': 10}, 'drift_pct_threshold': {'type': 'float', 'default': 20}}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ROCCurveDriftROC Curve DriftCompares ROC curves between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDNameDescriptionHas FigureHas TableRequired InputsParamsTagsTasks
validmind.model_validation.RegressionResidualsPlotRegression Residuals PlotEvaluates regression model performance using residual distribution and actual vs. predicted plots....TrueFalse['model', 'dataset']{'bin_size': {'type': 'float', 'default': 0.1}}['model_performance', 'visualization']['regression']
validmind.model_validation.sklearn.ConfusionMatrixConfusion MatrixEvaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix...TrueFalse['dataset', 'model']{'threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.PrecisionRecallCurvePrecision Recall CurveEvaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve....TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ROCCurveROC CurveEvaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic...TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.TrainingTestDegradationTraining Test DegradationTests if model performance degradation between training and test datasets exceeds a predefined threshold....FalseTrue['datasets', 'model']{'max_threshold': {'type': 'float', 'default': 0.1}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.CalibrationCurveDriftCalibration Curve DriftEvaluates changes in probability calibration between reference and monitoring datasets....TrueTrue['datasets', 'model']{'n_bins': {'type': 'int', 'default': 10}, 'drift_pct_threshold': {'type': 'float', 'default': 20}}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ROCCurveDriftROC Curve DriftCompares ROC curves between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
\n" ], - "source": [ - "list_tests(filter=\"sklearn\",\n", - " tags=[\"model_performance\", \"visualization\"], task=\"classification\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Store test sets for use\n", - "\n", - "Once you've identified specific sets of tests you'd like to run, you can store the tests in variables, enabling you to easily reuse those tests in later steps.\n", - "\n", - "For example, if you're validating a summarization model, use [`list_tests()`](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to retrieve all tests tagged for text summarization and save them to `text_summarization_tests` for later use:\n" + "text/plain": [ + "" ] - }, + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list_tests(tags=[\"model_performance\", \"visualization\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use `filter`, `task`, and `tags` together to create more specific queries.\n", + "\n", + "For example, apply all three to find tests compatible with `sklearn` models, designed for `classification` tasks:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['validmind.data_validation.DatasetDescription',\n", - " 'validmind.data_validation.DatasetSplit',\n", - " 'validmind.data_validation.nlp.CommonWords',\n", - " 'validmind.data_validation.nlp.Hashtags',\n", - " 'validmind.data_validation.nlp.LanguageDetection',\n", - " 'validmind.data_validation.nlp.Mentions',\n", - " 'validmind.data_validation.nlp.Punctuations',\n", - " 'validmind.data_validation.nlp.StopWords',\n", - " 'validmind.data_validation.nlp.TextDescription',\n", - " 'validmind.model_validation.BertScore',\n", - " 'validmind.model_validation.BleuScore',\n", - " 'validmind.model_validation.ContextualRecall',\n", - " 'validmind.model_validation.MeteorScore',\n", - " 'validmind.model_validation.RegardScore',\n", - " 'validmind.model_validation.RougeScore',\n", - " 'validmind.model_validation.TokenDisparity',\n", - " 'validmind.model_validation.ToxicityScore',\n", - " 'validmind.model_validation.embeddings.CosineSimilarityComparison',\n", - " 'validmind.model_validation.embeddings.CosineSimilarityHeatmap',\n", - " 'validmind.model_validation.embeddings.EuclideanDistanceComparison',\n", - " 'validmind.model_validation.embeddings.EuclideanDistanceHeatmap',\n", - " 'validmind.model_validation.embeddings.PCAComponentsPairwisePlots',\n", - " 'validmind.model_validation.embeddings.TSNEComponentsPairwisePlots',\n", - " 'validmind.model_validation.ragas.AnswerCorrectness',\n", - " 'validmind.model_validation.ragas.AspectCritic',\n", - " 'validmind.model_validation.ragas.ContextEntityRecall',\n", - " 'validmind.model_validation.ragas.ContextPrecision',\n", - " 'validmind.model_validation.ragas.ContextPrecisionWithoutReference',\n", - " 'validmind.model_validation.ragas.ContextRecall',\n", - " 'validmind.model_validation.ragas.Faithfulness',\n", - " 'validmind.model_validation.ragas.NoiseSensitivity',\n", - " 'validmind.model_validation.ragas.ResponseRelevancy',\n", - " 'validmind.model_validation.ragas.SemanticSimilarity',\n", - " 'validmind.prompt_validation.Bias',\n", - " 'validmind.prompt_validation.Clarity',\n", - " 'validmind.prompt_validation.Conciseness',\n", - " 'validmind.prompt_validation.Delimitation',\n", - " 'validmind.prompt_validation.NegativeInstruction',\n", - " 'validmind.prompt_validation.Robustness',\n", - " 'validmind.prompt_validation.Specificity']" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDNameDescriptionHas FigureHas TableRequired InputsParamsTagsTasks
validmind.model_validation.sklearn.ConfusionMatrixConfusion MatrixEvaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix...TrueFalse['dataset', 'model']{'threshold': {'type': 'float', 'default': 0.5}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.PrecisionRecallCurvePrecision Recall CurveEvaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve....TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.ROCCurveROC CurveEvaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic...TrueFalse['model', 'dataset']{}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.model_validation.sklearn.TrainingTestDegradationTraining Test DegradationTests if model performance degradation between training and test datasets exceeds a predefined threshold....FalseTrue['datasets', 'model']{'max_threshold': {'type': 'float', 'default': 0.1}}['sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.CalibrationCurveDriftCalibration Curve DriftEvaluates changes in probability calibration between reference and monitoring datasets....TrueTrue['datasets', 'model']{'n_bins': {'type': 'int', 'default': 10}, 'drift_pct_threshold': {'type': 'float', 'default': 20}}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
validmind.ongoing_monitoring.ROCCurveDriftROC Curve DriftCompares ROC curves between reference and monitoring datasets....TrueFalse['datasets', 'model']{}['sklearn', 'binary_classification', 'model_performance', 'visualization']['classification', 'text_classification']
\n" ], - "source": [ - "text_summarization_tests = list_tests(task=\"text_summarization\", pretty=False)\n", - "text_summarization_tests" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Next steps\n", - "\n", - "Now that you know how to browse and filter tests in the ValidMind Library, you’re ready to take the next step. Use the test IDs you’ve selected to either run individual tests or batch run them with custom test suites.\n", - "\n", - "
Learn about the tests suites available in the ValidMind Library.\n", - "

\n", - "Check out our Explore test suites notebook for more code examples and usage of key functions.
\n", - "\n", - "\n", - "\n", - "### Discover more learning resources\n", - "\n", - "We offer many interactive notebooks to help you document models:\n", - "\n", - "- [Run tests & test suites](https://docs.validmind.ai/developer/model-testing/testing-overview.html)\n", - "- [Code samples](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)\n", - "\n", - "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind.\n" + "text/plain": [ + "" ] - }, + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list_tests(filter=\"sklearn\",\n", + " tags=[\"model_performance\", \"visualization\"], task=\"classification\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Store test sets for use\n", + "\n", + "Once you've identified specific sets of tests you'd like to run, you can store the tests in variables, enabling you to easily reuse those tests in later steps.\n", + "\n", + "For example, if you're validating a summarization model, use [`list_tests()`](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to retrieve all tests tagged for text summarization and save them to `text_summarization_tests` for later use:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "***\n", - "\n", - "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", - "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + "data": { + "text/plain": [ + "['validmind.data_validation.DatasetDescription',\n", + " 'validmind.data_validation.DatasetSplit',\n", + " 'validmind.data_validation.nlp.CommonWords',\n", + " 'validmind.data_validation.nlp.Hashtags',\n", + " 'validmind.data_validation.nlp.LanguageDetection',\n", + " 'validmind.data_validation.nlp.Mentions',\n", + " 'validmind.data_validation.nlp.Punctuations',\n", + " 'validmind.data_validation.nlp.StopWords',\n", + " 'validmind.data_validation.nlp.TextDescription',\n", + " 'validmind.model_validation.BertScore',\n", + " 'validmind.model_validation.BleuScore',\n", + " 'validmind.model_validation.ContextualRecall',\n", + " 'validmind.model_validation.MeteorScore',\n", + " 'validmind.model_validation.RegardScore',\n", + " 'validmind.model_validation.RougeScore',\n", + " 'validmind.model_validation.TokenDisparity',\n", + " 'validmind.model_validation.ToxicityScore',\n", + " 'validmind.model_validation.embeddings.CosineSimilarityComparison',\n", + " 'validmind.model_validation.embeddings.CosineSimilarityHeatmap',\n", + " 'validmind.model_validation.embeddings.EuclideanDistanceComparison',\n", + " 'validmind.model_validation.embeddings.EuclideanDistanceHeatmap',\n", + " 'validmind.model_validation.embeddings.PCAComponentsPairwisePlots',\n", + " 'validmind.model_validation.embeddings.TSNEComponentsPairwisePlots',\n", + " 'validmind.model_validation.ragas.AnswerCorrectness',\n", + " 'validmind.model_validation.ragas.AspectCritic',\n", + " 'validmind.model_validation.ragas.ContextEntityRecall',\n", + " 'validmind.model_validation.ragas.ContextPrecision',\n", + " 'validmind.model_validation.ragas.ContextPrecisionWithoutReference',\n", + " 'validmind.model_validation.ragas.ContextRecall',\n", + " 'validmind.model_validation.ragas.Faithfulness',\n", + " 'validmind.model_validation.ragas.NoiseSensitivity',\n", + " 'validmind.model_validation.ragas.ResponseRelevancy',\n", + " 'validmind.model_validation.ragas.SemanticSimilarity',\n", + " 'validmind.prompt_validation.Bias',\n", + " 'validmind.prompt_validation.Clarity',\n", + " 'validmind.prompt_validation.Conciseness',\n", + " 'validmind.prompt_validation.Delimitation',\n", + " 'validmind.prompt_validation.NegativeInstruction',\n", + " 'validmind.prompt_validation.Robustness',\n", + " 'validmind.prompt_validation.Specificity']" ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.20" - } + ], + "source": [ + "text_summarization_tests = list_tests(task=\"text_summarization\", pretty=False)\n", + "text_summarization_tests" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Next steps\n", + "\n", + "Now that you know how to browse and filter tests in the ValidMind Library, you’re ready to take the next step. Use the test IDs you’ve selected to either run individual tests or batch run them with custom test suites.\n", + "\n", + "
Learn about the tests suites available in the ValidMind Library.\n", + "

\n", + "Check out our Explore test suites notebook for more code examples and usage of key functions.
\n", + "\n", + "\n", + "\n", + "### Discover more learning resources\n", + "\n", + "We offer many interactive notebooks to help you document models:\n", + "\n", + "- [Run tests & test suites](https://docs.validmind.ai/developer/model-testing/testing-overview.html)\n", + "- [Code samples](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)\n", + "\n", + "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind.\n" + ] + }, + { + "cell_type": "markdown", + "id": "bdfaacd9", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" }, - "nbformat": 4, - "nbformat_minor": 4 + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.20" + } + }, + "nbformat": 4, + "nbformat_minor": 4 } From 2d7c6dffda9532440c3474da6cfb8a834afd79e8 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:42:42 -0800 Subject: [PATCH 16/31] Updating with cell ID --- scripts/copyright_notebooks.py | 106 ++++++++++++++------------------- 1 file changed, 44 insertions(+), 62 deletions(-) diff --git a/scripts/copyright_notebooks.py b/scripts/copyright_notebooks.py index 444d17761..048d8720a 100644 --- a/scripts/copyright_notebooks.py +++ b/scripts/copyright_notebooks.py @@ -20,6 +20,7 @@ import argparse import sys +import uuid from dataclasses import dataclass from pathlib import Path from typing import Iterable, Optional, Tuple @@ -27,12 +28,11 @@ import nbformat MARKER = "" +CELL_ID_PREFIX = "copyright-" # Assumes: # repo/ # notebooks/ -# templates/ -# _copyright.ipynb # scripts/ # sync_copyright.py REPO_ROOT = Path(__file__).resolve().parents[1] @@ -54,17 +54,18 @@ def _write_notebook(path: Path, nb: nbformat.NotebookNode) -> None: def _normalize_source(s: str) -> str: - """ - Normalize for stable comparisons: - - Convert Windows/Mac newlines to \n - - Strip trailing whitespace on each line - - Strip trailing newlines at end - """ s = s.replace("\r\n", "\n").replace("\r", "\n") s = "\n".join(line.rstrip() for line in s.split("\n")) return s.rstrip("\n") +def _new_copyright_cell(source: str) -> nbformat.NotebookNode: + return nbformat.v4.new_markdown_cell( + source=source, + id=f"{CELL_ID_PREFIX}{uuid.uuid4().hex}", + ) + + def _find_marked_markdown_cell_index(nb: nbformat.NotebookNode) -> Optional[int]: for i, cell in enumerate(nb.cells): if cell.get("cell_type") == "markdown": @@ -74,34 +75,30 @@ def _find_marked_markdown_cell_index(nb: nbformat.NotebookNode) -> Optional[int] return None -def _load_canonical_cell_source(copyright_nb_path: Path) -> str: - if not copyright_nb_path.exists(): - raise FileNotFoundError(f"Copyright notebook not found: {copyright_nb_path}") +def _load_canonical_cell_source(path: Path) -> str: + if not path.exists(): + raise FileNotFoundError(f"Copyright notebook not found: {path}") - nb = _read_notebook(copyright_nb_path) + nb = _read_notebook(path) idx = _find_marked_markdown_cell_index(nb) if idx is None: raise ValueError( - f"Could not find a markdown cell containing marker {MARKER!r} in {copyright_nb_path}" + f"Could not find a markdown cell containing marker {MARKER!r} in {path}" ) canonical = nb.cells[idx].get("source", "") if not isinstance(canonical, str) or not canonical.strip(): - raise ValueError(f"Canonical cell in {copyright_nb_path} is empty or invalid.") + raise ValueError(f"Canonical cell in {path} is empty or invalid.") return canonical def _iter_ipynb_files(root: Path, exclude_dirs: Tuple[str, ...]) -> Iterable[Path]: for path in root.rglob("*.ipynb"): - # Skip typical noisy directories. if any(ex in path.parts for ex in exclude_dirs): continue - - # Only check notebooks whose filename does NOT start with "_" if path.name.startswith("_"): continue - yield path @@ -120,12 +117,14 @@ def process_notebook( canonical_norm = _normalize_source(canonical_source) idx = _find_marked_markdown_cell_index(nb) + + # ---------- Missing cell ---------- if idx is None: if check_only: return Result(nb_path, "would-append", "marker not found") if not dry_run: - nb.cells.append(nbformat.v4.new_markdown_cell(source=canonical_source)) + nb.cells.append(_new_copyright_cell(canonical_source)) try: _write_notebook(nb_path, nb) except Exception as e: @@ -133,17 +132,21 @@ def process_notebook( return Result(nb_path, "appended" if not dry_run else "would-append", "marker not found") - existing = nb.cells[idx].get("source", "") - existing_norm = _normalize_source(existing if isinstance(existing, str) else "") + # ---------- Existing cell ---------- + cell = nb.cells[idx] + existing_source = cell.get("source", "") + existing_norm = _normalize_source(existing_source if isinstance(existing_source, str) else "") + + id_ok = isinstance(cell.get("id"), str) and cell["id"].startswith(CELL_ID_PREFIX) - if existing_norm == canonical_norm: + if existing_norm == canonical_norm and id_ok: return Result(nb_path, "unchanged", "already matches canonical") if check_only: - return Result(nb_path, "would-update", "marker found but content differs") + return Result(nb_path, "would-update", "content or id differs") if not dry_run: - nb.cells[idx]["source"] = canonical_source + nb.cells[idx] = _new_copyright_cell(canonical_source) try: _write_notebook(nb_path, nb) except Exception as e: @@ -152,7 +155,7 @@ def process_notebook( return Result( nb_path, "updated" if not dry_run else "would-update", - "marker found but content differed", + "content or id differed", ) @@ -168,7 +171,7 @@ def main() -> int: "--copyright", type=Path, default=REPO_ROOT / "notebooks" / "templates" / "_copyright.ipynb", - help="Notebook containing the canonical copyright cell (default: repo_root/notebooks/templates/_copyright.ipynb)", + help="Canonical copyright notebook", ) p.add_argument( "--exclude-dirs", @@ -176,17 +179,8 @@ def main() -> int: default=[".ipynb_checkpoints", ".git", ".venv", "venv", "node_modules", "__pycache__"], help="Directory names to exclude anywhere in the path", ) - p.add_argument("--dry-run", action="store_true", help="Show what would change but don't write") - p.add_argument( - "--check", - action="store_true", - help="Exit non-zero if any notebook would be updated/appended (no writes)", - ) - p.add_argument( - "--include-copyright-notebook", - action="store_true", - help="Also process the copyright notebook itself (default: skip it)", - ) + p.add_argument("--dry-run", action="store_true") + p.add_argument("--check", action="store_true") args = p.parse_args() root = args.root.resolve() @@ -198,38 +192,26 @@ def main() -> int: print(f"ERROR: {e}", file=sys.stderr) return 2 - results: list[Result] = [] + counts: dict[str, int] = {} for nb_path in _iter_ipynb_files(root, tuple(args.exclude_dirs)): - if not args.include_copyright_notebook and nb_path.resolve() == copyright_nb: - continue - results.append( - process_notebook( - nb_path, - canonical_source, - dry_run=args.dry_run, - check_only=args.check, - ) + result = process_notebook( + nb_path, + canonical_source, + dry_run=args.dry_run, + check_only=args.check, ) - - # Print per-file changes/errors - counts: dict[str, int] = {} - for r in results: - counts[r.status] = counts.get(r.status, 0) + 1 - if r.status in {"appended", "updated", "would-append", "would-update", "error"}: - print(f"{r.status:12} {r.path} ({r.detail})") + counts[result.status] = counts.get(result.status, 0) + 1 + if result.status not in {"unchanged"}: + print(f"{result.status:12} {result.path} ({result.detail})") print("\nSummary:") - for k in sorted(counts.keys()): + for k in sorted(counts): print(f" {k:12}: {counts[k]}") - if args.check: - # Any diff means failure (useful for CI) - if counts.get("would-append", 0) or counts.get("would-update", 0) or counts.get("error", 0): - return 1 - - if counts.get("error", 0): + if args.check and any(k in counts for k in ("would-append", "would-update", "error")): + return 1 + if counts.get("error"): return 3 - return 0 From 91f56303e23c05af1b17a92f89d868a303614d01 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:43:47 -0800 Subject: [PATCH 17/31] Rerunning script --- .../agents/langgraph_agent_simple_banking_demo.ipynb | 1 + .../capital_markets/quickstart_option_pricing_models.ipynb | 2 +- .../quickstart_option_pricing_models_quantlib.ipynb | 2 +- .../code_explainer/quickstart_code_explainer_demo.ipynb | 1 + .../credit_risk/application_scorecard_executive.ipynb | 1 + .../credit_risk/application_scorecard_full_suite.ipynb | 1 + .../credit_risk/application_scorecard_with_bias.ipynb | 1 + .../credit_risk/application_scorecard_with_ml.ipynb | 1 + .../credit_risk/document_excel_application_scorecard.ipynb | 1 + .../code_samples/custom_tests/implement_custom_tests.ipynb | 1 + .../custom_tests/integrate_external_test_providers.ipynb | 1 + .../model_validation/validate_application_scorecard.ipynb | 1 + .../nlp_and_llm/foundation_models_integration_demo.ipynb | 1 + .../nlp_and_llm/foundation_models_summarization_demo.ipynb | 1 + .../nlp_and_llm/hugging_face_integration_demo.ipynb | 1 + .../nlp_and_llm/hugging_face_summarization_demo.ipynb | 1 + .../code_samples/nlp_and_llm/llm_summarization_demo.ipynb | 1 + .../code_samples/nlp_and_llm/prompt_validation_demo.ipynb | 1 + notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb | 1 + .../code_samples/nlp_and_llm/rag_documentation_demo.ipynb | 1 + .../application_scorecard_ongoing_monitoring.ipynb | 1 + .../quickstart_customer_churn_ongoing_monitoring.ipynb | 1 + .../regression/quickstart_regression_full_suite.ipynb | 1 + .../time_series/quickstart_time_series_full_suite.ipynb | 1 + .../time_series/quickstart_time_series_high_code.ipynb | 1 + .../code_sharing/clustering/quickstart_cluster_demo.ipynb | 1 + .../credit_risk/assign_prediction_probabilities.ipynb | 1 + notebooks/code_sharing/deepeval_integration_demo.ipynb | 2 +- .../embeddings/quickstart_embeddings_demo.ipynb | 6 +++++- .../code_sharing/geval_deepeval_integration_demo.ipynb | 2 +- .../insurance_mortality/insurance_validation_demo.ipynb | 4 ++-- .../insurance_mortality/validmind_insurance_POC.ipynb | 2 +- notebooks/code_sharing/llm/rag_llamaindex.ipynb | 1 + notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb | 1 + .../llm/rag_rfp_answer_generation_langchain.ipynb | 1 + .../code_sharing/llm/rag_rfp_question_similarity.ipynb | 1 + .../code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb | 1 + .../genai_vendor_contract_agent_usecase_poc.ipynb | 1 + notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb | 1 + .../operational_deposit/operational_deposit_poc.ipynb | 1 + .../operational_deposit/synthetic_data_generation.ipynb | 1 + .../customizing_tests_with_output_templates.ipynb | 1 + notebooks/code_sharing/plots_and_stats_demo.ipynb | 2 +- notebooks/code_sharing/post_processing_functions.ipynb | 1 + notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb | 1 + notebooks/code_sharing/r_demo/r-ecm-demo.ipynb | 2 +- notebooks/code_sharing/r_demo/r-ecm-model.ipynb | 2 +- .../code_sharing/regression/regression_unit_metrics.ipynb | 1 + .../code_sharing/test_configuration_updates_demo.ipynb | 1 + notebooks/how_to/assign_scores_complete_tutorial.ipynb | 2 +- notebooks/how_to/configure_dataset_features.ipynb | 4 ++-- notebooks/how_to/customize_test_result_descriptions.ipynb | 1 + .../document_multiple_results_for_the_same_test.ipynb | 1 + notebooks/how_to/enable_pii_detection.ipynb | 2 +- notebooks/how_to/explore_test_suites.ipynb | 1 + notebooks/how_to/explore_tests.ipynb | 2 +- notebooks/how_to/filter_input_columns.ipynb | 1 + notebooks/how_to/load_datasets_predictions.ipynb | 1 + notebooks/how_to/log_metrics_over_time.ipynb | 1 + notebooks/how_to/run_documentation_sections.ipynb | 1 + notebooks/how_to/run_documentation_tests_with_config.ipynb | 1 + notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb | 1 + notebooks/how_to/run_tests/2_run_comparison_tests.ipynb | 1 + .../how_to/run_tests_that_require_multiple_datasets.ipynb | 1 + notebooks/how_to/run_unit_metrics.ipynb | 4 ++-- notebooks/how_to/understand_utilize_rawdata.ipynb | 2 +- notebooks/how_to/use_dataset_model_objects.ipynb | 1 + notebooks/quickstart/quickstart_model_documentation.ipynb | 2 +- notebooks/quickstart/quickstart_model_validation.ipynb | 2 +- notebooks/templates/e2e-notebook.ipynb | 1 + .../tutorials/model_development/1-set_up_validmind.ipynb | 2 +- .../model_development/2-start_development_process.ipynb | 1 + .../model_development/3-integrate_custom_tests.ipynb | 1 + .../4-finalize_testing_documentation.ipynb | 1 + .../1-set_up_validmind_for_validation.ipynb | 2 +- .../model_validation/2-start_validation_process.ipynb | 1 + .../model_validation/3-developing_challenger_model.ipynb | 1 + .../model_validation/4-finalize_validation_reporting.ipynb | 1 + 78 files changed, 85 insertions(+), 23 deletions(-) diff --git a/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb b/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb index 27f0cfb78..bbdab6a21 100644 --- a/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb +++ b/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb @@ -1462,6 +1462,7 @@ }, { "cell_type": "markdown", + "id": "copyright-4a8b954ce80b451ea981617ae01863bb", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb index a5aa91c81..989d919e9 100644 --- a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb +++ b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb @@ -2061,7 +2061,7 @@ }, { "cell_type": "markdown", - "id": "adb15089", + "id": "copyright-08fc00557dce4ceb9b88415e92b96f9d", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb index 8311bdf9c..c00c90b30 100644 --- a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb +++ b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb @@ -1291,7 +1291,7 @@ }, { "cell_type": "markdown", - "id": "cab6911c", + "id": "copyright-04d764b723b24d2c91fb160889ce5686", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb b/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb index 5665049d1..efd3d86af 100644 --- a/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb +++ b/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb @@ -823,6 +823,7 @@ }, { "cell_type": "markdown", + "id": "copyright-374a03611aa34d17809511af22876130", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb index 1801b0d7c..15129a0a0 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb @@ -343,6 +343,7 @@ }, { "cell_type": "markdown", + "id": "copyright-c6f1e357abb641589e3e838c42700469", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb index 056980c20..52bcb220b 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb @@ -863,6 +863,7 @@ }, { "cell_type": "markdown", + "id": "copyright-2bb3d61d89b24097acfe3a930a7738c3", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb index f0e321f68..8a67d6cbc 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb @@ -1508,6 +1508,7 @@ }, { "cell_type": "markdown", + "id": "copyright-325f517ff94f433da6edeb715e0b18b7", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb index f5ad09c8d..ad80673ec 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb @@ -1958,6 +1958,7 @@ }, { "cell_type": "markdown", + "id": "copyright-8dc8c0e2d94e466aa3662abea59ae320", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb b/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb index bba44f438..3827a806e 100644 --- a/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb +++ b/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb @@ -967,6 +967,7 @@ }, { "cell_type": "markdown", + "id": "copyright-7999db9e05c141ffbb1aebc359e5e991", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb b/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb index 897541235..a32403e00 100644 --- a/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb +++ b/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb @@ -1056,6 +1056,7 @@ }, { "cell_type": "markdown", + "id": "copyright-dc1c13abf083439bb35183d74f1e3906", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb b/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb index 7b1417dfc..51177b8bb 100644 --- a/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb +++ b/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb @@ -956,6 +956,7 @@ }, { "cell_type": "markdown", + "id": "copyright-8ae7040f97944f0a921aa73c496c4028", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb b/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb index f91c759b1..fc89bc7f5 100644 --- a/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb +++ b/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb @@ -1837,6 +1837,7 @@ }, { "cell_type": "markdown", + "id": "copyright-0da375a81eb24d1e9262c75cefa11fe0", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb b/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb index 2ff9e7eef..07f859248 100644 --- a/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb @@ -459,6 +459,7 @@ }, { "cell_type": "markdown", + "id": "copyright-618bc2e28e5b4c1a86fe7ec816670ecf", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb index 5807cb645..23e13310e 100644 --- a/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb @@ -506,6 +506,7 @@ }, { "cell_type": "markdown", + "id": "copyright-7712d42b5de9414ab0ada1a836ec292b", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb b/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb index 27b686021..3b1e12198 100644 --- a/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb @@ -443,6 +443,7 @@ }, { "cell_type": "markdown", + "id": "copyright-36b4de610d8346c2bd39f77034353c5b", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb index f6aeacb36..d0cbc8338 100644 --- a/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb @@ -464,6 +464,7 @@ }, { "cell_type": "markdown", + "id": "copyright-21fc7c08e9eb4c49b870329276d6ee59", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb index 2e45b1e35..bfb28f6a4 100644 --- a/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb @@ -981,6 +981,7 @@ }, { "cell_type": "markdown", + "id": "copyright-2fa8cf438e7846c795d86136e2ff8737", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb b/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb index cc305d9c8..ded04b612 100644 --- a/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb @@ -514,6 +514,7 @@ }, { "cell_type": "markdown", + "id": "copyright-06bb1f319953451ebe45f9b6d3212588", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb b/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb index 2dbd36a41..21cc88f56 100644 --- a/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb @@ -1818,6 +1818,7 @@ }, { "cell_type": "markdown", + "id": "copyright-19e39bd8bb4849baa4ad1e25b19c9005", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb b/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb index 60090eb16..60564fda3 100644 --- a/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb @@ -1642,6 +1642,7 @@ }, { "cell_type": "markdown", + "id": "copyright-c3c7f66d7ff34b20acc6ce689b035a8d", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb b/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb index a6957f059..a5cfaf318 100644 --- a/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb +++ b/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb @@ -1320,6 +1320,7 @@ }, { "cell_type": "markdown", + "id": "copyright-454ac221f7b54eda9b3ef276d72c98d8", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb b/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb index cc730ad5f..74b00edf5 100644 --- a/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb +++ b/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb @@ -847,6 +847,7 @@ }, { "cell_type": "markdown", + "id": "copyright-24962d7e133d4c49ba80f951b9596bf7", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb b/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb index d233cf04a..3fc8e6b56 100644 --- a/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb +++ b/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb @@ -548,6 +548,7 @@ }, { "cell_type": "markdown", + "id": "copyright-247efd904e2c4c73ac8b4c2a3c3d72c3", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb b/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb index 0f1b611ab..03a27fb33 100644 --- a/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb +++ b/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb @@ -710,6 +710,7 @@ }, { "cell_type": "markdown", + "id": "copyright-e1bfae645c344efb893cc91068d739a0", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb b/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb index ad9bd4997..8bb58015a 100644 --- a/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb +++ b/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb @@ -968,6 +968,7 @@ }, { "cell_type": "markdown", + "id": "copyright-0424783f00b54a1ba194d091f228cd00", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb b/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb index 1ece1c15c..5a0ab5a31 100644 --- a/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb +++ b/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb @@ -449,6 +449,7 @@ }, { "cell_type": "markdown", + "id": "copyright-2c09c52a30194ca995811da83001409c", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb b/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb index 2238ebf3b..fc0bd3156 100644 --- a/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb +++ b/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb @@ -737,6 +737,7 @@ }, { "cell_type": "markdown", + "id": "copyright-2b381c75df7a43e5ad28a83638c094b8", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/deepeval_integration_demo.ipynb b/notebooks/code_sharing/deepeval_integration_demo.ipynb index c2698e31f..3722b3d2c 100644 --- a/notebooks/code_sharing/deepeval_integration_demo.ipynb +++ b/notebooks/code_sharing/deepeval_integration_demo.ipynb @@ -1063,7 +1063,7 @@ }, { "cell_type": "markdown", - "id": "22f216ac", + "id": "copyright-6cbf89e4a1c84ffab4754f30d29e3122", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb b/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb index a1968716e..35827864e 100644 --- a/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb +++ b/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb @@ -273,6 +273,7 @@ }, { "cell_type": "markdown", + "id": "8304fc80", "metadata": {}, "source": [ "## Upgrade ValidMind\n", @@ -285,6 +286,7 @@ { "cell_type": "code", "execution_count": null, + "id": "b1a2ec55", "metadata": {}, "outputs": [], "source": [ @@ -293,6 +295,7 @@ }, { "cell_type": "markdown", + "id": "a37344ed", "metadata": {}, "source": [ "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n", @@ -304,6 +307,7 @@ }, { "cell_type": "markdown", + "id": "e6eea130", "metadata": {}, "source": [ "You may need to restart your kernel after running the upgrade package for changes to be applied." @@ -311,7 +315,7 @@ }, { "cell_type": "markdown", - "id": "0a827b65", + "id": "copyright-1c3f12e2b05445f1a5ad8082d340e711", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb b/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb index f9c068dc5..6a3f30173 100644 --- a/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb +++ b/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb @@ -485,7 +485,7 @@ }, { "cell_type": "markdown", - "id": "bb785b65", + "id": "copyright-405304d2fe4b4a7d834c03e119a598d0", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb b/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb index 1929e0a5d..f35d439b4 100644 --- a/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb +++ b/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb @@ -502,6 +502,7 @@ }, { "cell_type": "markdown", + "id": "copyright-5791d70b8f4544f7b2fb51a8882ce98d", "metadata": {}, "source": [ "\n", @@ -533,8 +534,7 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.16" - }, - "orig_nbformat": 4 + } }, "nbformat": 4, "nbformat_minor": 2 diff --git a/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb b/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb index 48d860478..e4f842328 100644 --- a/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb +++ b/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb @@ -1431,7 +1431,7 @@ }, { "cell_type": "markdown", - "id": "f7176334", + "id": "copyright-05fe74a970a94a0295319bfc930a3c6c", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/llm/rag_llamaindex.ipynb b/notebooks/code_sharing/llm/rag_llamaindex.ipynb index 7eeffbd86..24682b197 100644 --- a/notebooks/code_sharing/llm/rag_llamaindex.ipynb +++ b/notebooks/code_sharing/llm/rag_llamaindex.ipynb @@ -552,6 +552,7 @@ }, { "cell_type": "markdown", + "id": "copyright-60f202c7445a4917b9a98bd414b17872", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb b/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb index 2466d6b40..cc8a145f0 100644 --- a/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb @@ -298,6 +298,7 @@ }, { "cell_type": "markdown", + "id": "copyright-5495b7f913c648519071bbf905a995e1", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb b/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb index e6746b00a..fa2fe4d34 100644 --- a/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb @@ -805,6 +805,7 @@ }, { "cell_type": "markdown", + "id": "copyright-d5c28b30a0da46aeb27736a7fa3190ed", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb b/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb index be6de753b..a36136df8 100644 --- a/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb @@ -1097,6 +1097,7 @@ }, { "cell_type": "markdown", + "id": "copyright-ef260866f86949d9ba4fb00e404a4ae3", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb b/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb index dfc494b30..5b399febd 100644 --- a/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb +++ b/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb @@ -1411,6 +1411,7 @@ }, { "cell_type": "markdown", + "id": "copyright-1164a14a67624ae69117bbf0776a9e1c", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb b/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb index c6be300b1..c01914a23 100644 --- a/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb +++ b/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb @@ -509,6 +509,7 @@ }, { "cell_type": "markdown", + "id": "copyright-9816bc693dae41fc9a730d0a633e95be", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb b/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb index 36ea66d13..dbf2fe329 100644 --- a/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb +++ b/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb @@ -344,6 +344,7 @@ }, { "cell_type": "markdown", + "id": "copyright-56711472201d464da9fe65aa9c86df27", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb b/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb index d15c60429..66d7b1562 100644 --- a/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb +++ b/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb @@ -1160,6 +1160,7 @@ }, { "cell_type": "markdown", + "id": "copyright-902f7c9c77e24d1f90fa8dd92f8ed6a1", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb b/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb index fb7c4d9d9..ccaea82bb 100644 --- a/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb +++ b/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb @@ -164,6 +164,7 @@ }, { "cell_type": "markdown", + "id": "copyright-d352a5bbccf04edeaf0904a097ce401b", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb b/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb index 4b3da0479..8dd51796b 100644 --- a/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb +++ b/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb @@ -818,6 +818,7 @@ }, { "cell_type": "markdown", + "id": "copyright-f975b807debb41b08bc76ec15d6b5270", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/plots_and_stats_demo.ipynb b/notebooks/code_sharing/plots_and_stats_demo.ipynb index 02b2c29a3..f3cb1b1a6 100644 --- a/notebooks/code_sharing/plots_and_stats_demo.ipynb +++ b/notebooks/code_sharing/plots_and_stats_demo.ipynb @@ -685,7 +685,7 @@ }, { "cell_type": "markdown", - "id": "ce1de6a7", + "id": "copyright-43bb005394714716a8d86388849bada5", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/post_processing_functions.ipynb b/notebooks/code_sharing/post_processing_functions.ipynb index cdc2b792a..a6fe1cdb3 100644 --- a/notebooks/code_sharing/post_processing_functions.ipynb +++ b/notebooks/code_sharing/post_processing_functions.ipynb @@ -610,6 +610,7 @@ }, { "cell_type": "markdown", + "id": "copyright-d1340daee29f4698bc8c027822ac9024", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb b/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb index 756e08952..e1ed683fb 100644 --- a/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb +++ b/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb @@ -484,6 +484,7 @@ }, { "cell_type": "markdown", + "id": "copyright-0b8a5215b8ae4772a7dcaee5a7aaff5a", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb b/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb index c16e03eaf..407fc0fbd 100644 --- a/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb +++ b/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb @@ -123,7 +123,7 @@ }, { "cell_type": "markdown", - "id": "8fb2b3e2", + "id": "copyright-df542aeb3c1c4cf49417bfcfbc5181c2", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/r_demo/r-ecm-model.ipynb b/notebooks/code_sharing/r_demo/r-ecm-model.ipynb index e0a8a886a..b2ee3e8b8 100644 --- a/notebooks/code_sharing/r_demo/r-ecm-model.ipynb +++ b/notebooks/code_sharing/r_demo/r-ecm-model.ipynb @@ -211,7 +211,7 @@ }, { "cell_type": "markdown", - "id": "49991d76", + "id": "copyright-1208c6ec921a4c2287ed552fefb9d492", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/regression/regression_unit_metrics.ipynb b/notebooks/code_sharing/regression/regression_unit_metrics.ipynb index bcf287279..96f03d8cd 100644 --- a/notebooks/code_sharing/regression/regression_unit_metrics.ipynb +++ b/notebooks/code_sharing/regression/regression_unit_metrics.ipynb @@ -761,6 +761,7 @@ }, { "cell_type": "markdown", + "id": "copyright-60fa6601fb06405588c55d115d751712", "metadata": {}, "source": [ "\n", diff --git a/notebooks/code_sharing/test_configuration_updates_demo.ipynb b/notebooks/code_sharing/test_configuration_updates_demo.ipynb index ec3e8f1db..9d9aa1bdc 100644 --- a/notebooks/code_sharing/test_configuration_updates_demo.ipynb +++ b/notebooks/code_sharing/test_configuration_updates_demo.ipynb @@ -56,6 +56,7 @@ }, { "cell_type": "markdown", + "id": "copyright-039c4c36fba14271bb77914fd32cd1ac", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/assign_scores_complete_tutorial.ipynb b/notebooks/how_to/assign_scores_complete_tutorial.ipynb index 24adfa9ce..be0b6c5e3 100644 --- a/notebooks/how_to/assign_scores_complete_tutorial.ipynb +++ b/notebooks/how_to/assign_scores_complete_tutorial.ipynb @@ -774,7 +774,7 @@ }, { "cell_type": "markdown", - "id": "c1b1dd04", + "id": "copyright-558d40f9618f443a902f6e82b19ef637", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/configure_dataset_features.ipynb b/notebooks/how_to/configure_dataset_features.ipynb index 70bef799a..7511a3c6e 100644 --- a/notebooks/how_to/configure_dataset_features.ipynb +++ b/notebooks/how_to/configure_dataset_features.ipynb @@ -432,6 +432,7 @@ }, { "cell_type": "markdown", + "id": "copyright-f866c2325f194718af7e8b6ff634d396", "metadata": {}, "source": [ "\n", @@ -463,8 +464,7 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" - }, - "orig_nbformat": 4 + } }, "nbformat": 4, "nbformat_minor": 2 diff --git a/notebooks/how_to/customize_test_result_descriptions.ipynb b/notebooks/how_to/customize_test_result_descriptions.ipynb index eb6ad34e7..2ce816e19 100644 --- a/notebooks/how_to/customize_test_result_descriptions.ipynb +++ b/notebooks/how_to/customize_test_result_descriptions.ipynb @@ -1002,6 +1002,7 @@ }, { "cell_type": "markdown", + "id": "copyright-1d37be74d95e4962834aef4bca1dcf5b", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb b/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb index 8f0b6bc2a..14228c6f2 100644 --- a/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb +++ b/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb @@ -583,6 +583,7 @@ }, { "cell_type": "markdown", + "id": "copyright-70d2c811a5404d36a3a64fe7ec71e73b", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/enable_pii_detection.ipynb b/notebooks/how_to/enable_pii_detection.ipynb index 0ad69ac0a..eeb3028c4 100644 --- a/notebooks/how_to/enable_pii_detection.ipynb +++ b/notebooks/how_to/enable_pii_detection.ipynb @@ -600,7 +600,7 @@ }, { "cell_type": "markdown", - "id": "f1e696a9", + "id": "copyright-b5e6bd73e17d4dad8f08bd2d610ea3ae", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/explore_test_suites.ipynb b/notebooks/how_to/explore_test_suites.ipynb index cebf3977b..6c8cbc620 100644 --- a/notebooks/how_to/explore_test_suites.ipynb +++ b/notebooks/how_to/explore_test_suites.ipynb @@ -688,6 +688,7 @@ }, { "cell_type": "markdown", + "id": "copyright-1a5266dc56d84720a8a64e3566c1d0ea", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/explore_tests.ipynb b/notebooks/how_to/explore_tests.ipynb index 6ced61976..a6c7cabcc 100644 --- a/notebooks/how_to/explore_tests.ipynb +++ b/notebooks/how_to/explore_tests.ipynb @@ -4387,7 +4387,7 @@ }, { "cell_type": "markdown", - "id": "bdfaacd9", + "id": "copyright-37e237f2c9a9471ba73341a731c3f9e5", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/filter_input_columns.ipynb b/notebooks/how_to/filter_input_columns.ipynb index 20fc2875e..1d792ea11 100644 --- a/notebooks/how_to/filter_input_columns.ipynb +++ b/notebooks/how_to/filter_input_columns.ipynb @@ -64,6 +64,7 @@ }, { "cell_type": "markdown", + "id": "copyright-ac723356fc904b3ea2324be034e1a4d3", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/load_datasets_predictions.ipynb b/notebooks/how_to/load_datasets_predictions.ipynb index 0a2b2d2b9..659363c3b 100644 --- a/notebooks/how_to/load_datasets_predictions.ipynb +++ b/notebooks/how_to/load_datasets_predictions.ipynb @@ -1017,6 +1017,7 @@ }, { "cell_type": "markdown", + "id": "copyright-f0f5f5c8197b4ef4b4f50dd7b19d4d0d", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/log_metrics_over_time.ipynb b/notebooks/how_to/log_metrics_over_time.ipynb index 2cc5da6a3..7f79ecf32 100644 --- a/notebooks/how_to/log_metrics_over_time.ipynb +++ b/notebooks/how_to/log_metrics_over_time.ipynb @@ -919,6 +919,7 @@ }, { "cell_type": "markdown", + "id": "copyright-3ade6e49f883409ca0bf318bc762a074", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/run_documentation_sections.ipynb b/notebooks/how_to/run_documentation_sections.ipynb index 44e27ca20..a77b8219f 100644 --- a/notebooks/how_to/run_documentation_sections.ipynb +++ b/notebooks/how_to/run_documentation_sections.ipynb @@ -555,6 +555,7 @@ }, { "cell_type": "markdown", + "id": "copyright-91227660a8554880b1fd48c405811806", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/run_documentation_tests_with_config.ipynb b/notebooks/how_to/run_documentation_tests_with_config.ipynb index 0795316e3..40cf4b08a 100644 --- a/notebooks/how_to/run_documentation_tests_with_config.ipynb +++ b/notebooks/how_to/run_documentation_tests_with_config.ipynb @@ -680,6 +680,7 @@ }, { "cell_type": "markdown", + "id": "copyright-7bfb4adf3fa74d4798cef42604f5d6f1", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb b/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb index 1b909773b..b1272dd7c 100644 --- a/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb +++ b/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb @@ -567,6 +567,7 @@ }, { "cell_type": "markdown", + "id": "copyright-4a0bae9136454ace8c99adf8edf1e817", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb b/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb index 7c27e48de..10368ee89 100644 --- a/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb +++ b/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb @@ -751,6 +751,7 @@ }, { "cell_type": "markdown", + "id": "copyright-ed5da38da21649069130bbd93ea62f44", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb b/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb index 6a5d57476..4a7152f54 100644 --- a/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb +++ b/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb @@ -532,6 +532,7 @@ }, { "cell_type": "markdown", + "id": "copyright-2cce0683b80246f6ada2eb0bbd3fd098", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/run_unit_metrics.ipynb b/notebooks/how_to/run_unit_metrics.ipynb index 7242c67a9..f82746374 100644 --- a/notebooks/how_to/run_unit_metrics.ipynb +++ b/notebooks/how_to/run_unit_metrics.ipynb @@ -756,6 +756,7 @@ }, { "cell_type": "markdown", + "id": "copyright-9e3803f1f4f141c6bbb19b62c82c4244", "metadata": {}, "source": [ "\n", @@ -787,8 +788,7 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" - }, - "orig_nbformat": 4 + } }, "nbformat": 4, "nbformat_minor": 2 diff --git a/notebooks/how_to/understand_utilize_rawdata.ipynb b/notebooks/how_to/understand_utilize_rawdata.ipynb index 72425f2c2..fbc10fc2a 100644 --- a/notebooks/how_to/understand_utilize_rawdata.ipynb +++ b/notebooks/how_to/understand_utilize_rawdata.ipynb @@ -702,7 +702,7 @@ }, { "cell_type": "markdown", - "id": "16d292fe", + "id": "copyright-b30ff7e5445141fda373c9d60f3387ee", "metadata": {}, "source": [ "\n", diff --git a/notebooks/how_to/use_dataset_model_objects.ipynb b/notebooks/how_to/use_dataset_model_objects.ipynb index 402542003..b512b9e1a 100644 --- a/notebooks/how_to/use_dataset_model_objects.ipynb +++ b/notebooks/how_to/use_dataset_model_objects.ipynb @@ -966,6 +966,7 @@ }, { "cell_type": "markdown", + "id": "copyright-15e20dc385d14d9ebd8fa506dee33251", "metadata": {}, "source": [ "\n", diff --git a/notebooks/quickstart/quickstart_model_documentation.ipynb b/notebooks/quickstart/quickstart_model_documentation.ipynb index 6369bb8ec..ec713790f 100644 --- a/notebooks/quickstart/quickstart_model_documentation.ipynb +++ b/notebooks/quickstart/quickstart_model_documentation.ipynb @@ -852,7 +852,7 @@ }, { "cell_type": "markdown", - "id": "09c906f3", + "id": "copyright-9b5d15f077ba48b9af65474494f123a0", "metadata": {}, "source": [ "\n", diff --git a/notebooks/quickstart/quickstart_model_validation.ipynb b/notebooks/quickstart/quickstart_model_validation.ipynb index 367263b9b..d54934c3a 100644 --- a/notebooks/quickstart/quickstart_model_validation.ipynb +++ b/notebooks/quickstart/quickstart_model_validation.ipynb @@ -1173,7 +1173,7 @@ }, { "cell_type": "markdown", - "id": "f33d4214", + "id": "copyright-a62891c4333546a68021886fbab3d9ce", "metadata": {}, "source": [ "\n", diff --git a/notebooks/templates/e2e-notebook.ipynb b/notebooks/templates/e2e-notebook.ipynb index c6641a3a8..be779df91 100644 --- a/notebooks/templates/e2e-notebook.ipynb +++ b/notebooks/templates/e2e-notebook.ipynb @@ -281,6 +281,7 @@ }, { "cell_type": "markdown", + "id": "copyright-76242bf35234457daa55eb4f995f6cc8", "metadata": {}, "source": [ "\n", diff --git a/notebooks/tutorials/model_development/1-set_up_validmind.ipynb b/notebooks/tutorials/model_development/1-set_up_validmind.ipynb index 832b69ad5..db2ae274b 100644 --- a/notebooks/tutorials/model_development/1-set_up_validmind.ipynb +++ b/notebooks/tutorials/model_development/1-set_up_validmind.ipynb @@ -440,7 +440,7 @@ }, { "cell_type": "markdown", - "id": "39c974f0", + "id": "copyright-3020855baec547b79ddd967f29b32248", "metadata": {}, "source": [ "\n", diff --git a/notebooks/tutorials/model_development/2-start_development_process.ipynb b/notebooks/tutorials/model_development/2-start_development_process.ipynb index 26010cb73..ae38cbfb3 100644 --- a/notebooks/tutorials/model_development/2-start_development_process.ipynb +++ b/notebooks/tutorials/model_development/2-start_development_process.ipynb @@ -989,6 +989,7 @@ }, { "cell_type": "markdown", + "id": "copyright-e5e5871f076545839c2d502658ac7e33", "metadata": {}, "source": [ "\n", diff --git a/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb b/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb index 18f7e8348..6d54af45f 100644 --- a/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb +++ b/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb @@ -968,6 +968,7 @@ }, { "cell_type": "markdown", + "id": "copyright-a43192a3977347558fcbe85bcfaec9da", "metadata": {}, "source": [ "\n", diff --git a/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb b/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb index 5b0521886..b3a86d96e 100644 --- a/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb +++ b/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb @@ -967,6 +967,7 @@ }, { "cell_type": "markdown", + "id": "copyright-6540b3502fe94eed919c7109a67f31cd", "metadata": {}, "source": [ "\n", diff --git a/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb b/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb index 7275fd2ea..43ee7a3e5 100644 --- a/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb +++ b/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb @@ -491,7 +491,7 @@ }, { "cell_type": "markdown", - "id": "47e90901", + "id": "copyright-01931314846445b888659f6f15b2189b", "metadata": {}, "source": [ "\n", diff --git a/notebooks/tutorials/model_validation/2-start_validation_process.ipynb b/notebooks/tutorials/model_validation/2-start_validation_process.ipynb index b5d10b3ac..71aedda0f 100644 --- a/notebooks/tutorials/model_validation/2-start_validation_process.ipynb +++ b/notebooks/tutorials/model_validation/2-start_validation_process.ipynb @@ -862,6 +862,7 @@ }, { "cell_type": "markdown", + "id": "copyright-9f5daf1c86b346fd87ad7fa53417c7fb", "metadata": {}, "source": [ "\n", diff --git a/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb b/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb index 70ab94083..b312f4735 100644 --- a/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb +++ b/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb @@ -865,6 +865,7 @@ }, { "cell_type": "markdown", + "id": "copyright-9dae2efaa1024dee9020ae673b5817a7", "metadata": {}, "source": [ "\n", diff --git a/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb b/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb index 0566bddba..d1fe90923 100644 --- a/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb +++ b/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb @@ -1200,6 +1200,7 @@ }, { "cell_type": "markdown", + "id": "copyright-20b8614cb05a45f4bb5adb7ceb939482", "metadata": {}, "source": [ "\n", From 4fff6421e0197f72e7524cd5db9fd4e70000bd37 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:44:48 -0800 Subject: [PATCH 18/31] Combining script & README instructions --- Makefile | 2 -- README.md | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0a511591c..9ddfbc7fa 100644 --- a/Makefile +++ b/Makefile @@ -77,8 +77,6 @@ generate-test-id-types: copyright: poetry run python scripts/copyright_files.py - -copyright-nb: poetry run python scripts/copyright_notebooks.py verify-copyright: diff --git a/README.md b/README.md index 1c467e9d0..48324858c 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ poetry run python scripts/add_test_description.py review validmind/tests/ongoing ## Adding a copyright header -When adding new Python files to the project, you can add the ValidMind copyright header to any files that +When adding new Python or Jupyter Notebook files to the project, you can add the ValidMind copyright header to any files that are missing it by running: ```bash From 68e5d61f8d9ed9369f855afafd32b4719a8777e3 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:49:03 -0800 Subject: [PATCH 19/31] Editing returns --- scripts/copyright_notebooks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/copyright_notebooks.py b/scripts/copyright_notebooks.py index 048d8720a..457cae97c 100644 --- a/scripts/copyright_notebooks.py +++ b/scripts/copyright_notebooks.py @@ -121,7 +121,7 @@ def process_notebook( # ---------- Missing cell ---------- if idx is None: if check_only: - return Result(nb_path, "would-append", "marker not found") + return Result(nb_path, "would-append", "Copyright marker not found") if not dry_run: nb.cells.append(_new_copyright_cell(canonical_source)) @@ -130,7 +130,7 @@ def process_notebook( except Exception as e: return Result(nb_path, "error", f"failed to write: {e}") - return Result(nb_path, "appended" if not dry_run else "would-append", "marker not found") + return Result(nb_path, "appended" if not dry_run else "would-append", "Copyright marker not found") # ---------- Existing cell ---------- cell = nb.cells[idx] @@ -204,7 +204,7 @@ def main() -> int: if result.status not in {"unchanged"}: print(f"{result.status:12} {result.path} ({result.detail})") - print("\nSummary:") + print("\nNotebook summary:") for k in sorted(counts): print(f" {k:12}: {counts[k]}") From 2eec5d83c173ecad8dcd43ab3d40a56351155492 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:52:23 -0800 Subject: [PATCH 20/31] Editing READMEs --- README.md | 2 +- notebooks/templates/README.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 48324858c..487780fc9 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ poetry run python scripts/add_test_description.py review validmind/tests/ongoing ## Adding a copyright header -When adding new Python or Jupyter Notebook files to the project, you can add the ValidMind copyright header to any files that +When adding new Python or stand-alone Jupyter Notebook files to the project, you can add the ValidMind copyright header to any files that are missing it by running: ```bash diff --git a/notebooks/templates/README.md b/notebooks/templates/README.md index b4d9db71e..e90a1c39d 100644 --- a/notebooks/templates/README.md +++ b/notebooks/templates/README.md @@ -18,6 +18,8 @@ The template generation script/notebook draws from the following mini-templates, - [`_upgrade-validmind.ipynb`](_upgrade-validmind.ipynb): Instructions for comparing & upgrading versions of the ValidMind Library. - [`_copyright.ipynb`](_copyright.ipynb): Copyright and licensing info — mandatory at the bottom of every notebook owned by ValidMind. +Mini-templates are not meant to stand alone, and follow the filename convention of `_filename.ipynb`. + ## Add table of contents For lengthy notebooks, we recommend that you add a table of contents with the [**Simplified table of contents for Jupyter Notebooks extension**](https://github.com/validbeck/jupyter-notebook-toc/tree/main/installation). From c881862d57b40996ae2e5f9bd6ee047d8c8f16ad Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:55:42 -0800 Subject: [PATCH 21/31] Verification script --- Makefile | 3 + scripts/verify_notebook_copyright.py | 132 +++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 scripts/verify_notebook_copyright.py diff --git a/Makefile b/Makefile index 9ddfbc7fa..a4b81760f 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,9 @@ copyright: verify-copyright: poetry run python scripts/verify_copyright.py +verify-nb-copyright: + poetry run python scripts/verify_notebook_copyright.py + verify-exposed-credentials: poetry run python scripts/credentials_check.py diff --git a/scripts/verify_notebook_copyright.py b/scripts/verify_notebook_copyright.py new file mode 100644 index 000000000..4375d3727 --- /dev/null +++ b/scripts/verify_notebook_copyright.py @@ -0,0 +1,132 @@ +# This software is proprietary and confidential. Unauthorized copying, +# modification, distribution or use of this software is strictly prohibited. +# Please refer to the LICENSE file in the root directory of this repository +# for more information. +# +# Copyright © 2023 ValidMind Inc. All rights reserved. + +""" +This script verifies that all notebooks under a directory have the +ValidMind copyright cell. + +How to use: + poetry run python scripts/verify_notebook_copyright.py + +Notes: +- Checks for a markdown cell containing: +- Compares that cell's content to notebooks/templates/_copyright.ipynb +- Only checks .ipynb files whose filename does NOT start with "_" +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Optional + +import nbformat + +MARKER = "" +CELL_ID_PREFIX = "copyright-" + + +def normalize_source(s: str) -> str: + s = s.replace("\r\n", "\n").replace("\r", "\n") + s = "\n".join(line.rstrip() for line in s.split("\n")) + return s.rstrip("\n") + + +def find_marked_markdown_cell_index(nb: nbformat.NotebookNode) -> Optional[int]: + for i, cell in enumerate(nb.cells): + if cell.get("cell_type") == "markdown": + src = cell.get("source", "") + if isinstance(src, str) and MARKER in src: + return i + return None + + +def load_canonical_cell_source(copyright_nb_path: Path) -> str: + if not copyright_nb_path.exists(): + raise FileNotFoundError(f"Canonical copyright notebook not found: {copyright_nb_path}") + + nb = nbformat.read(str(copyright_nb_path), as_version=4) + idx = find_marked_markdown_cell_index(nb) + if idx is None: + raise ValueError( + f"Could not find a markdown cell containing marker {MARKER!r} in {copyright_nb_path}" + ) + + canonical = nb.cells[idx].get("source", "") + if not isinstance(canonical, str) or not canonical.strip(): + raise ValueError(f"Canonical cell in {copyright_nb_path} is empty or invalid.") + + return canonical + + +def main() -> int: + repo_root = Path(os.getcwd()).resolve() + + # Align with the existing script style: + # - default paths rooted at cwd (repo root when run via poetry/make) + notebooks_dir = repo_root / "notebooks" + copyright_nb_path = repo_root / "notebooks" / "templates" / "_copyright.ipynb" + + try: + canonical_source = load_canonical_cell_source(copyright_nb_path) + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + return 2 + + canonical_norm = normalize_source(canonical_source) + + errors: list[str] = [] + + for root, dirs, files in os.walk(notebooks_dir): + # Skip noisy directories + dirs[:] = [d for d in dirs if d not in {".ipynb_checkpoints", ".git", "__pycache__"}] + + for file in files: + if not file.endswith(".ipynb"): + continue + if file.startswith("_"): + continue + + nb_path = Path(root) / file + + try: + nb = nbformat.read(str(nb_path), as_version=4) + except Exception as e: + errors.append(f"Notebook {nb_path} could not be read: {e}") + continue + + idx = find_marked_markdown_cell_index(nb) + if idx is None: + errors.append(f"Notebook {nb_path} is missing the copyright cell.") + continue + + cell = nb.cells[idx] + src = cell.get("source", "") + src_norm = normalize_source(src if isinstance(src, str) else "") + + if src_norm != canonical_norm: + errors.append( + f"Notebook {nb_path} has a copyright cell, but its content does not match canonical." + ) + + cell_id = cell.get("id") + if not (isinstance(cell_id, str) and cell_id.startswith(CELL_ID_PREFIX)): + errors.append( + f"Notebook {nb_path} copyright cell is missing a valid id (expected prefix '{CELL_ID_PREFIX}')." + ) + + if errors: + print("\n".join(errors)) + print("\nPlease fix the errors above by running `make copyright-notebooks`") + return 1 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 282c97d3a55aa1543d4e1a79ecdb77f6d11ef127 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:57:59 -0800 Subject: [PATCH 22/31] Save point --- Makefile | 2 -- scripts/verify_notebook_copyright.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a4b81760f..dc0c017b5 100644 --- a/Makefile +++ b/Makefile @@ -81,8 +81,6 @@ copyright: verify-copyright: poetry run python scripts/verify_copyright.py - -verify-nb-copyright: poetry run python scripts/verify_notebook_copyright.py verify-exposed-credentials: diff --git a/scripts/verify_notebook_copyright.py b/scripts/verify_notebook_copyright.py index 4375d3727..b976a4995 100644 --- a/scripts/verify_notebook_copyright.py +++ b/scripts/verify_notebook_copyright.py @@ -122,7 +122,7 @@ def main() -> int: if errors: print("\n".join(errors)) - print("\nPlease fix the errors above by running `make copyright-notebooks`") + print("\nPlease fix the errors above by running `make copyright`") return 1 return 0 From 72488a2d3bc0e3834896bf1f51df86fd55f1028f Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 15:13:28 -0800 Subject: [PATCH 23/31] Update notebooks/templates/_copyright.ipynb Co-authored-by: Nik Richers --- notebooks/templates/_copyright.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/templates/_copyright.ipynb b/notebooks/templates/_copyright.ipynb index 1c5cb574e..54065c853 100644 --- a/notebooks/templates/_copyright.ipynb +++ b/notebooks/templates/_copyright.ipynb @@ -12,7 +12,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } From 9f6f953807e47d8e753c18b7c64ff237471532d1 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 15:22:34 -0800 Subject: [PATCH 24/31] Reverting weird changes to load.py --- validmind/tests/load.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/validmind/tests/load.py b/validmind/tests/load.py index 9a9f13c53..4dc97d11a 100644 --- a/validmind/tests/load.py +++ b/validmind/tests/load.py @@ -127,9 +127,7 @@ def _inspect_signature( return inputs, params -def _get_test_function_from_provider( - test_id: str, namespace: str -) -> Callable[..., Any]: +def _get_test_function_from_provider(test_id: str, namespace: str) -> Callable[..., Any]: """Load a test function from the appropriate provider or scorer store. Args: @@ -148,7 +146,9 @@ def _get_test_function_from_provider( return custom_scorer if not test_provider_store.has_test_provider(namespace): - raise LoadTestError(f"No test provider found for namespace: {namespace}") + raise LoadTestError( + f"No test provider found for namespace: {namespace}" + ) provider = test_provider_store.get_test_provider(namespace) From 1a98746e0c71ee921d38fa2ab271de9c13a21731 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 15:23:26 -0800 Subject: [PATCH 25/31] Updating notebooks with simplified license --- .../agents/langgraph_agent_simple_banking_demo.ipynb | 4 ++-- .../capital_markets/quickstart_option_pricing_models.ipynb | 4 ++-- .../quickstart_option_pricing_models_quantlib.ipynb | 4 ++-- .../code_explainer/quickstart_code_explainer_demo.ipynb | 4 ++-- .../credit_risk/application_scorecard_executive.ipynb | 4 ++-- .../credit_risk/application_scorecard_full_suite.ipynb | 4 ++-- .../credit_risk/application_scorecard_with_bias.ipynb | 4 ++-- .../credit_risk/application_scorecard_with_ml.ipynb | 4 ++-- .../credit_risk/document_excel_application_scorecard.ipynb | 4 ++-- .../code_samples/custom_tests/implement_custom_tests.ipynb | 4 ++-- .../custom_tests/integrate_external_test_providers.ipynb | 4 ++-- .../model_validation/validate_application_scorecard.ipynb | 4 ++-- .../nlp_and_llm/foundation_models_integration_demo.ipynb | 4 ++-- .../nlp_and_llm/foundation_models_summarization_demo.ipynb | 4 ++-- .../nlp_and_llm/hugging_face_integration_demo.ipynb | 4 ++-- .../nlp_and_llm/hugging_face_summarization_demo.ipynb | 4 ++-- .../code_samples/nlp_and_llm/llm_summarization_demo.ipynb | 4 ++-- .../code_samples/nlp_and_llm/prompt_validation_demo.ipynb | 4 ++-- notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb | 4 ++-- .../code_samples/nlp_and_llm/rag_documentation_demo.ipynb | 4 ++-- .../application_scorecard_ongoing_monitoring.ipynb | 4 ++-- .../quickstart_customer_churn_ongoing_monitoring.ipynb | 4 ++-- .../regression/quickstart_regression_full_suite.ipynb | 4 ++-- .../time_series/quickstart_time_series_full_suite.ipynb | 4 ++-- .../time_series/quickstart_time_series_high_code.ipynb | 4 ++-- .../code_sharing/clustering/quickstart_cluster_demo.ipynb | 4 ++-- .../credit_risk/assign_prediction_probabilities.ipynb | 4 ++-- notebooks/code_sharing/deepeval_integration_demo.ipynb | 4 ++-- .../code_sharing/embeddings/quickstart_embeddings_demo.ipynb | 4 ++-- notebooks/code_sharing/geval_deepeval_integration_demo.ipynb | 4 ++-- .../insurance_mortality/insurance_validation_demo.ipynb | 4 ++-- .../insurance_mortality/validmind_insurance_POC.ipynb | 4 ++-- notebooks/code_sharing/llm/rag_llamaindex.ipynb | 4 ++-- notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb | 4 ++-- .../llm/rag_rfp_answer_generation_langchain.ipynb | 4 ++-- notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb | 4 ++-- .../code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb | 4 ++-- .../genai_vendor_contract_agent_usecase_poc.ipynb | 4 ++-- notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb | 4 ++-- .../operational_deposit/operational_deposit_poc.ipynb | 4 ++-- .../operational_deposit/synthetic_data_generation.ipynb | 4 ++-- .../customizing_tests_with_output_templates.ipynb | 4 ++-- notebooks/code_sharing/plots_and_stats_demo.ipynb | 4 ++-- notebooks/code_sharing/post_processing_functions.ipynb | 4 ++-- notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb | 4 ++-- notebooks/code_sharing/r_demo/r-ecm-demo.ipynb | 4 ++-- notebooks/code_sharing/r_demo/r-ecm-model.ipynb | 4 ++-- .../code_sharing/regression/regression_unit_metrics.ipynb | 4 ++-- notebooks/code_sharing/test_configuration_updates_demo.ipynb | 4 ++-- notebooks/how_to/assign_scores_complete_tutorial.ipynb | 4 ++-- notebooks/how_to/configure_dataset_features.ipynb | 4 ++-- notebooks/how_to/customize_test_result_descriptions.ipynb | 4 ++-- .../how_to/document_multiple_results_for_the_same_test.ipynb | 4 ++-- notebooks/how_to/enable_pii_detection.ipynb | 4 ++-- notebooks/how_to/explore_test_suites.ipynb | 4 ++-- notebooks/how_to/explore_tests.ipynb | 4 ++-- notebooks/how_to/filter_input_columns.ipynb | 4 ++-- notebooks/how_to/load_datasets_predictions.ipynb | 4 ++-- notebooks/how_to/log_metrics_over_time.ipynb | 4 ++-- notebooks/how_to/run_documentation_sections.ipynb | 4 ++-- notebooks/how_to/run_documentation_tests_with_config.ipynb | 4 ++-- notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb | 4 ++-- notebooks/how_to/run_tests/2_run_comparison_tests.ipynb | 4 ++-- .../how_to/run_tests_that_require_multiple_datasets.ipynb | 4 ++-- notebooks/how_to/run_unit_metrics.ipynb | 4 ++-- notebooks/how_to/understand_utilize_rawdata.ipynb | 4 ++-- notebooks/how_to/use_dataset_model_objects.ipynb | 4 ++-- notebooks/quickstart/quickstart_model_documentation.ipynb | 4 ++-- notebooks/quickstart/quickstart_model_validation.ipynb | 4 ++-- notebooks/templates/e2e-notebook.ipynb | 4 ++-- .../tutorials/model_development/1-set_up_validmind.ipynb | 4 ++-- .../model_development/2-start_development_process.ipynb | 4 ++-- .../model_development/3-integrate_custom_tests.ipynb | 4 ++-- .../model_development/4-finalize_testing_documentation.ipynb | 4 ++-- .../model_validation/1-set_up_validmind_for_validation.ipynb | 4 ++-- .../model_validation/2-start_validation_process.ipynb | 4 ++-- .../model_validation/3-developing_challenger_model.ipynb | 4 ++-- .../model_validation/4-finalize_validation_reporting.ipynb | 4 ++-- 78 files changed, 156 insertions(+), 156 deletions(-) diff --git a/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb b/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb index bbdab6a21..7e80b877a 100644 --- a/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb +++ b/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb @@ -1462,7 +1462,7 @@ }, { "cell_type": "markdown", - "id": "copyright-4a8b954ce80b451ea981617ae01863bb", + "id": "copyright-e7184e5605bb4f85b3d7b8306aaaef78", "metadata": {}, "source": [ "\n", @@ -1472,7 +1472,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb index 989d919e9..695395924 100644 --- a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb +++ b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb @@ -2061,7 +2061,7 @@ }, { "cell_type": "markdown", - "id": "copyright-08fc00557dce4ceb9b88415e92b96f9d", + "id": "copyright-a23adf093a60485ea005cf8fc18545a5", "metadata": {}, "source": [ "\n", @@ -2071,7 +2071,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb index c00c90b30..1269254ad 100644 --- a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb +++ b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb @@ -1291,7 +1291,7 @@ }, { "cell_type": "markdown", - "id": "copyright-04d764b723b24d2c91fb160889ce5686", + "id": "copyright-de5d1e182b09403abddabc2850f2dd05", "metadata": {}, "source": [ "\n", @@ -1301,7 +1301,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb b/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb index efd3d86af..1eb1ef747 100644 --- a/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb +++ b/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb @@ -823,7 +823,7 @@ }, { "cell_type": "markdown", - "id": "copyright-374a03611aa34d17809511af22876130", + "id": "copyright-ccbede139a26452183291a108b791513", "metadata": {}, "source": [ "\n", @@ -833,7 +833,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb index 15129a0a0..2115a88b5 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb @@ -343,7 +343,7 @@ }, { "cell_type": "markdown", - "id": "copyright-c6f1e357abb641589e3e838c42700469", + "id": "copyright-97f68fa25e694a059b7028ce3ec374cc", "metadata": {}, "source": [ "\n", @@ -353,7 +353,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb index 52bcb220b..83a6d276c 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb @@ -863,7 +863,7 @@ }, { "cell_type": "markdown", - "id": "copyright-2bb3d61d89b24097acfe3a930a7738c3", + "id": "copyright-ce253f0d12144a08847d4a65a250a85f", "metadata": {}, "source": [ "\n", @@ -873,7 +873,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb index 8a67d6cbc..a91560831 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb @@ -1508,7 +1508,7 @@ }, { "cell_type": "markdown", - "id": "copyright-325f517ff94f433da6edeb715e0b18b7", + "id": "copyright-6ee1a1ce0bd74036a8890be21965bfd2", "metadata": {}, "source": [ "\n", @@ -1518,7 +1518,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb index ad80673ec..357d572b7 100644 --- a/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb +++ b/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb @@ -1958,7 +1958,7 @@ }, { "cell_type": "markdown", - "id": "copyright-8dc8c0e2d94e466aa3662abea59ae320", + "id": "copyright-00e4b240625f4af29adb179235912142", "metadata": {}, "source": [ "\n", @@ -1968,7 +1968,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb b/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb index 3827a806e..cc7685c9c 100644 --- a/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb +++ b/notebooks/code_samples/credit_risk/document_excel_application_scorecard.ipynb @@ -967,7 +967,7 @@ }, { "cell_type": "markdown", - "id": "copyright-7999db9e05c141ffbb1aebc359e5e991", + "id": "copyright-b04bb3907b774a148a65d78c539ffb4d", "metadata": {}, "source": [ "\n", @@ -977,7 +977,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb b/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb index a32403e00..17385b8e1 100644 --- a/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb +++ b/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb @@ -1056,7 +1056,7 @@ }, { "cell_type": "markdown", - "id": "copyright-dc1c13abf083439bb35183d74f1e3906", + "id": "copyright-997b933948594ddd929ee9419957dfe3", "metadata": {}, "source": [ "\n", @@ -1066,7 +1066,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb b/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb index 51177b8bb..ca1403fb6 100644 --- a/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb +++ b/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb @@ -956,7 +956,7 @@ }, { "cell_type": "markdown", - "id": "copyright-8ae7040f97944f0a921aa73c496c4028", + "id": "copyright-e9a85f828fdb448ba20c565dec9a0b75", "metadata": {}, "source": [ "\n", @@ -966,7 +966,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb b/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb index fc89bc7f5..22ab661f7 100644 --- a/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb +++ b/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb @@ -1837,7 +1837,7 @@ }, { "cell_type": "markdown", - "id": "copyright-0da375a81eb24d1e9262c75cefa11fe0", + "id": "copyright-3441e79421624bc7819dc2c2957bd93f", "metadata": {}, "source": [ "\n", @@ -1847,7 +1847,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb b/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb index 07f859248..633c99e0c 100644 --- a/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb @@ -459,7 +459,7 @@ }, { "cell_type": "markdown", - "id": "copyright-618bc2e28e5b4c1a86fe7ec816670ecf", + "id": "copyright-114aa3e23e7e44318b66971760526b6f", "metadata": {}, "source": [ "\n", @@ -469,7 +469,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb index 23e13310e..a7f06ab15 100644 --- a/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb @@ -506,7 +506,7 @@ }, { "cell_type": "markdown", - "id": "copyright-7712d42b5de9414ab0ada1a836ec292b", + "id": "copyright-b34e4189be964082bb87b10aa94dcf6a", "metadata": {}, "source": [ "\n", @@ -516,7 +516,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb b/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb index 3b1e12198..dd7e03d05 100644 --- a/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb @@ -443,7 +443,7 @@ }, { "cell_type": "markdown", - "id": "copyright-36b4de610d8346c2bd39f77034353c5b", + "id": "copyright-8e54f1f3ff334b529685ef57073abb3e", "metadata": {}, "source": [ "\n", @@ -453,7 +453,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb index d0cbc8338..8209e65c2 100644 --- a/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb @@ -464,7 +464,7 @@ }, { "cell_type": "markdown", - "id": "copyright-21fc7c08e9eb4c49b870329276d6ee59", + "id": "copyright-14c6f04dcd164deb8a1db44fde050729", "metadata": {}, "source": [ "\n", @@ -474,7 +474,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb index bfb28f6a4..159a16e79 100644 --- a/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb @@ -981,7 +981,7 @@ }, { "cell_type": "markdown", - "id": "copyright-2fa8cf438e7846c795d86136e2ff8737", + "id": "copyright-f38869c3ab0c4de7989f536d06b51773", "metadata": {}, "source": [ "\n", @@ -991,7 +991,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb b/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb index ded04b612..49633b3a9 100644 --- a/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb @@ -514,7 +514,7 @@ }, { "cell_type": "markdown", - "id": "copyright-06bb1f319953451ebe45f9b6d3212588", + "id": "copyright-4101a5c77a954664ba8d8a682bee1a1c", "metadata": {}, "source": [ "\n", @@ -524,7 +524,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb b/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb index 21cc88f56..35969b1eb 100644 --- a/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb @@ -1818,7 +1818,7 @@ }, { "cell_type": "markdown", - "id": "copyright-19e39bd8bb4849baa4ad1e25b19c9005", + "id": "copyright-09e315440ca84258abe1aaefaca3a3d0", "metadata": {}, "source": [ "\n", @@ -1828,7 +1828,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb b/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb index 60564fda3..485c81ffd 100644 --- a/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb +++ b/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb @@ -1642,7 +1642,7 @@ }, { "cell_type": "markdown", - "id": "copyright-c3c7f66d7ff34b20acc6ce689b035a8d", + "id": "copyright-397fa35a68a34dc38f5d84d797fb5331", "metadata": {}, "source": [ "\n", @@ -1652,7 +1652,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb b/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb index a5cfaf318..1ff6ca3e0 100644 --- a/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb +++ b/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb @@ -1320,7 +1320,7 @@ }, { "cell_type": "markdown", - "id": "copyright-454ac221f7b54eda9b3ef276d72c98d8", + "id": "copyright-4c5ddb7cde514c958ea0048f5e472de5", "metadata": {}, "source": [ "\n", @@ -1330,7 +1330,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb b/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb index 74b00edf5..04f64a31e 100644 --- a/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb +++ b/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb @@ -847,7 +847,7 @@ }, { "cell_type": "markdown", - "id": "copyright-24962d7e133d4c49ba80f951b9596bf7", + "id": "copyright-c2ce5d3e3aa04c4ab7f6401ac810c67f", "metadata": {}, "source": [ "\n", @@ -857,7 +857,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb b/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb index 3fc8e6b56..90992448e 100644 --- a/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb +++ b/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb @@ -548,7 +548,7 @@ }, { "cell_type": "markdown", - "id": "copyright-247efd904e2c4c73ac8b4c2a3c3d72c3", + "id": "copyright-ad12b1c3e98d435ea8cc57eadf4f5a76", "metadata": {}, "source": [ "\n", @@ -558,7 +558,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb b/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb index 03a27fb33..27e572717 100644 --- a/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb +++ b/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb @@ -710,7 +710,7 @@ }, { "cell_type": "markdown", - "id": "copyright-e1bfae645c344efb893cc91068d739a0", + "id": "copyright-3e1111bc137c44b9866472934320b128", "metadata": {}, "source": [ "\n", @@ -720,7 +720,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb b/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb index 8bb58015a..33a2e7220 100644 --- a/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb +++ b/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb @@ -968,7 +968,7 @@ }, { "cell_type": "markdown", - "id": "copyright-0424783f00b54a1ba194d091f228cd00", + "id": "copyright-0e30120b73cd445b92ff0637c3467c01", "metadata": {}, "source": [ "\n", @@ -978,7 +978,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb b/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb index 5a0ab5a31..5fdb3cb21 100644 --- a/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb +++ b/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb @@ -449,7 +449,7 @@ }, { "cell_type": "markdown", - "id": "copyright-2c09c52a30194ca995811da83001409c", + "id": "copyright-ab09192f89b6461dab8c821b43a5b591", "metadata": {}, "source": [ "\n", @@ -459,7 +459,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb b/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb index fc0bd3156..cb4a7195e 100644 --- a/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb +++ b/notebooks/code_sharing/credit_risk/assign_prediction_probabilities.ipynb @@ -737,7 +737,7 @@ }, { "cell_type": "markdown", - "id": "copyright-2b381c75df7a43e5ad28a83638c094b8", + "id": "copyright-ec110f5334fc4d4e917b2c3d9e25fd65", "metadata": {}, "source": [ "\n", @@ -747,7 +747,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/deepeval_integration_demo.ipynb b/notebooks/code_sharing/deepeval_integration_demo.ipynb index 3722b3d2c..af26d7501 100644 --- a/notebooks/code_sharing/deepeval_integration_demo.ipynb +++ b/notebooks/code_sharing/deepeval_integration_demo.ipynb @@ -1063,7 +1063,7 @@ }, { "cell_type": "markdown", - "id": "copyright-6cbf89e4a1c84ffab4754f30d29e3122", + "id": "copyright-94c232c772c1435aa4529b67cfcc0bb2", "metadata": {}, "source": [ "\n", @@ -1073,7 +1073,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb b/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb index 35827864e..2aed999fd 100644 --- a/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb +++ b/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb @@ -315,7 +315,7 @@ }, { "cell_type": "markdown", - "id": "copyright-1c3f12e2b05445f1a5ad8082d340e711", + "id": "copyright-639ea318e72743eabfd7979b7994814f", "metadata": {}, "source": [ "\n", @@ -325,7 +325,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb b/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb index 6a3f30173..4f2a8dabc 100644 --- a/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb +++ b/notebooks/code_sharing/geval_deepeval_integration_demo.ipynb @@ -485,7 +485,7 @@ }, { "cell_type": "markdown", - "id": "copyright-405304d2fe4b4a7d834c03e119a598d0", + "id": "copyright-bc8fa92bf77149ecbc9fbce7b817acce", "metadata": {}, "source": [ "\n", @@ -495,7 +495,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb b/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb index f35d439b4..995f87013 100644 --- a/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb +++ b/notebooks/code_sharing/insurance_mortality/insurance_validation_demo.ipynb @@ -502,7 +502,7 @@ }, { "cell_type": "markdown", - "id": "copyright-5791d70b8f4544f7b2fb51a8882ce98d", + "id": "copyright-151aeb12b20c41ce951d6a1948ccfeac", "metadata": {}, "source": [ "\n", @@ -512,7 +512,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb b/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb index e4f842328..176ad30af 100644 --- a/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb +++ b/notebooks/code_sharing/insurance_mortality/validmind_insurance_POC.ipynb @@ -1431,7 +1431,7 @@ }, { "cell_type": "markdown", - "id": "copyright-05fe74a970a94a0295319bfc930a3c6c", + "id": "copyright-a91c22ff34ff4c8e8f63d1a9f3056ffe", "metadata": {}, "source": [ "\n", @@ -1441,7 +1441,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/rag_llamaindex.ipynb b/notebooks/code_sharing/llm/rag_llamaindex.ipynb index 24682b197..e00bec1cf 100644 --- a/notebooks/code_sharing/llm/rag_llamaindex.ipynb +++ b/notebooks/code_sharing/llm/rag_llamaindex.ipynb @@ -552,7 +552,7 @@ }, { "cell_type": "markdown", - "id": "copyright-60f202c7445a4917b9a98bd414b17872", + "id": "copyright-135ac40a507742419d0be8828ca69a43", "metadata": {}, "source": [ "\n", @@ -562,7 +562,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb b/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb index cc8a145f0..dd3521d88 100644 --- a/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_answer_generation.ipynb @@ -298,7 +298,7 @@ }, { "cell_type": "markdown", - "id": "copyright-5495b7f913c648519071bbf905a995e1", + "id": "copyright-03772d6afa1343d9adf9885cd1522f24", "metadata": {}, "source": [ "\n", @@ -308,7 +308,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb b/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb index fa2fe4d34..cac1232ca 100644 --- a/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_answer_generation_langchain.ipynb @@ -805,7 +805,7 @@ }, { "cell_type": "markdown", - "id": "copyright-d5c28b30a0da46aeb27736a7fa3190ed", + "id": "copyright-c312e313d66e4267bf134354d2e6af2b", "metadata": {}, "source": [ "\n", @@ -815,7 +815,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb b/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb index a36136df8..1c7cd1831 100644 --- a/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb +++ b/notebooks/code_sharing/llm/rag_rfp_question_similarity.ipynb @@ -1097,7 +1097,7 @@ }, { "cell_type": "markdown", - "id": "copyright-ef260866f86949d9ba4fb00e404a4ae3", + "id": "copyright-c81156e9f371416f867b5c3a6b5708a5", "metadata": {}, "source": [ "\n", @@ -1107,7 +1107,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb b/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb index 5b399febd..81450fa53 100644 --- a/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb +++ b/notebooks/code_sharing/llm/rag_vendor_contracts_llamaindex.ipynb @@ -1411,7 +1411,7 @@ }, { "cell_type": "markdown", - "id": "copyright-1164a14a67624ae69117bbf0776a9e1c", + "id": "copyright-7da783ce5cfa4cbc981b1a7ff8303606", "metadata": {}, "source": [ "\n", @@ -1421,7 +1421,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb b/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb index c01914a23..50a70d851 100644 --- a/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb +++ b/notebooks/code_sharing/llm/vendor_contract_agent/genai_vendor_contract_agent_usecase_poc.ipynb @@ -509,7 +509,7 @@ }, { "cell_type": "markdown", - "id": "copyright-9816bc693dae41fc9a730d0a633e95be", + "id": "copyright-3237936bfca7442a8d69058c607510ea", "metadata": {}, "source": [ "\n", @@ -519,7 +519,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb b/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb index dbf2fe329..50e53d53f 100644 --- a/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb +++ b/notebooks/code_sharing/market_basket_analysis/mba_poc.ipynb @@ -344,7 +344,7 @@ }, { "cell_type": "markdown", - "id": "copyright-56711472201d464da9fe65aa9c86df27", + "id": "copyright-c73a07d5c7bc4bdba38ab3a458c54e5a", "metadata": {}, "source": [ "\n", @@ -354,7 +354,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb b/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb index 66d7b1562..fc885bb19 100644 --- a/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb +++ b/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb @@ -1160,7 +1160,7 @@ }, { "cell_type": "markdown", - "id": "copyright-902f7c9c77e24d1f90fa8dd92f8ed6a1", + "id": "copyright-150aeb3474a94fa282029aed47eb2834", "metadata": {}, "source": [ "\n", @@ -1170,7 +1170,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb b/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb index ccaea82bb..6f853ea6e 100644 --- a/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb +++ b/notebooks/code_sharing/operational_deposit/synthetic_data_generation.ipynb @@ -164,7 +164,7 @@ }, { "cell_type": "markdown", - "id": "copyright-d352a5bbccf04edeaf0904a097ce401b", + "id": "copyright-cab796ab33f342aeae5cbb97e45d7674", "metadata": {}, "source": [ "\n", @@ -174,7 +174,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb b/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb index 8dd51796b..40ef8e454 100644 --- a/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb +++ b/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb @@ -818,7 +818,7 @@ }, { "cell_type": "markdown", - "id": "copyright-f975b807debb41b08bc76ec15d6b5270", + "id": "copyright-3bef29e042fe42af9d5a90752889fef1", "metadata": {}, "source": [ "\n", @@ -828,7 +828,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/plots_and_stats_demo.ipynb b/notebooks/code_sharing/plots_and_stats_demo.ipynb index f3cb1b1a6..8bc70d77e 100644 --- a/notebooks/code_sharing/plots_and_stats_demo.ipynb +++ b/notebooks/code_sharing/plots_and_stats_demo.ipynb @@ -685,7 +685,7 @@ }, { "cell_type": "markdown", - "id": "copyright-43bb005394714716a8d86388849bada5", + "id": "copyright-45adba169fed412bb9ef9b4389e4ee1a", "metadata": {}, "source": [ "\n", @@ -695,7 +695,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/post_processing_functions.ipynb b/notebooks/code_sharing/post_processing_functions.ipynb index a6fe1cdb3..e8dbdee5e 100644 --- a/notebooks/code_sharing/post_processing_functions.ipynb +++ b/notebooks/code_sharing/post_processing_functions.ipynb @@ -610,7 +610,7 @@ }, { "cell_type": "markdown", - "id": "copyright-d1340daee29f4698bc8c027822ac9024", + "id": "copyright-f2272c0f0bc6407eb6e368cf88972fb0", "metadata": {}, "source": [ "\n", @@ -620,7 +620,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb b/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb index e1ed683fb..40c8c6df9 100644 --- a/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb +++ b/notebooks/code_sharing/r_demo/r-customer-churn-model.ipynb @@ -484,7 +484,7 @@ }, { "cell_type": "markdown", - "id": "copyright-0b8a5215b8ae4772a7dcaee5a7aaff5a", + "id": "copyright-5bd8d9bb6e7241f3b0fabb188c85b380", "metadata": {}, "source": [ "\n", @@ -494,7 +494,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb b/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb index 407fc0fbd..27042b51f 100644 --- a/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb +++ b/notebooks/code_sharing/r_demo/r-ecm-demo.ipynb @@ -123,7 +123,7 @@ }, { "cell_type": "markdown", - "id": "copyright-df542aeb3c1c4cf49417bfcfbc5181c2", + "id": "copyright-33e6984e8b924bc08859eb8ad610184c", "metadata": {}, "source": [ "\n", @@ -133,7 +133,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/r_demo/r-ecm-model.ipynb b/notebooks/code_sharing/r_demo/r-ecm-model.ipynb index b2ee3e8b8..c2e7c783d 100644 --- a/notebooks/code_sharing/r_demo/r-ecm-model.ipynb +++ b/notebooks/code_sharing/r_demo/r-ecm-model.ipynb @@ -211,7 +211,7 @@ }, { "cell_type": "markdown", - "id": "copyright-1208c6ec921a4c2287ed552fefb9d492", + "id": "copyright-e47c2f27632947eebeb97978bcf50bb8", "metadata": {}, "source": [ "\n", @@ -221,7 +221,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/regression/regression_unit_metrics.ipynb b/notebooks/code_sharing/regression/regression_unit_metrics.ipynb index 96f03d8cd..3af977523 100644 --- a/notebooks/code_sharing/regression/regression_unit_metrics.ipynb +++ b/notebooks/code_sharing/regression/regression_unit_metrics.ipynb @@ -761,7 +761,7 @@ }, { "cell_type": "markdown", - "id": "copyright-60fa6601fb06405588c55d115d751712", + "id": "copyright-3ad307f0c325424782cdc45a37badc9c", "metadata": {}, "source": [ "\n", @@ -771,7 +771,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/code_sharing/test_configuration_updates_demo.ipynb b/notebooks/code_sharing/test_configuration_updates_demo.ipynb index 9d9aa1bdc..528f4de99 100644 --- a/notebooks/code_sharing/test_configuration_updates_demo.ipynb +++ b/notebooks/code_sharing/test_configuration_updates_demo.ipynb @@ -56,7 +56,7 @@ }, { "cell_type": "markdown", - "id": "copyright-039c4c36fba14271bb77914fd32cd1ac", + "id": "copyright-e9a1d468f954434aaf061db9e18b44f7", "metadata": {}, "source": [ "\n", @@ -66,7 +66,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/assign_scores_complete_tutorial.ipynb b/notebooks/how_to/assign_scores_complete_tutorial.ipynb index be0b6c5e3..2b96099df 100644 --- a/notebooks/how_to/assign_scores_complete_tutorial.ipynb +++ b/notebooks/how_to/assign_scores_complete_tutorial.ipynb @@ -774,7 +774,7 @@ }, { "cell_type": "markdown", - "id": "copyright-558d40f9618f443a902f6e82b19ef637", + "id": "copyright-25e9888fae40481b9007f6a2e2bdd60b", "metadata": {}, "source": [ "\n", @@ -784,7 +784,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/configure_dataset_features.ipynb b/notebooks/how_to/configure_dataset_features.ipynb index 7511a3c6e..30d86a1ee 100644 --- a/notebooks/how_to/configure_dataset_features.ipynb +++ b/notebooks/how_to/configure_dataset_features.ipynb @@ -432,7 +432,7 @@ }, { "cell_type": "markdown", - "id": "copyright-f866c2325f194718af7e8b6ff634d396", + "id": "copyright-77bd4876ab7945d6a1f592e432d81ca5", "metadata": {}, "source": [ "\n", @@ -442,7 +442,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/customize_test_result_descriptions.ipynb b/notebooks/how_to/customize_test_result_descriptions.ipynb index 2ce816e19..c3aca95e1 100644 --- a/notebooks/how_to/customize_test_result_descriptions.ipynb +++ b/notebooks/how_to/customize_test_result_descriptions.ipynb @@ -1002,7 +1002,7 @@ }, { "cell_type": "markdown", - "id": "copyright-1d37be74d95e4962834aef4bca1dcf5b", + "id": "copyright-5559bb769ab34ad5b09176ce718a7b6c", "metadata": {}, "source": [ "\n", @@ -1012,7 +1012,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb b/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb index 14228c6f2..74e04e1ab 100644 --- a/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb +++ b/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb @@ -583,7 +583,7 @@ }, { "cell_type": "markdown", - "id": "copyright-70d2c811a5404d36a3a64fe7ec71e73b", + "id": "copyright-6ce412276b6244aab16b2e3443c6a861", "metadata": {}, "source": [ "\n", @@ -593,7 +593,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/enable_pii_detection.ipynb b/notebooks/how_to/enable_pii_detection.ipynb index eeb3028c4..3d648e82c 100644 --- a/notebooks/how_to/enable_pii_detection.ipynb +++ b/notebooks/how_to/enable_pii_detection.ipynb @@ -600,7 +600,7 @@ }, { "cell_type": "markdown", - "id": "copyright-b5e6bd73e17d4dad8f08bd2d610ea3ae", + "id": "copyright-b91f78083ca449d7a755ccfb02d001b1", "metadata": {}, "source": [ "\n", @@ -610,7 +610,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/explore_test_suites.ipynb b/notebooks/how_to/explore_test_suites.ipynb index 6c8cbc620..f9f0a7286 100644 --- a/notebooks/how_to/explore_test_suites.ipynb +++ b/notebooks/how_to/explore_test_suites.ipynb @@ -688,7 +688,7 @@ }, { "cell_type": "markdown", - "id": "copyright-1a5266dc56d84720a8a64e3566c1d0ea", + "id": "copyright-a1609733aeef47c78c2de0ece28f59fa", "metadata": {}, "source": [ "\n", @@ -698,7 +698,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/explore_tests.ipynb b/notebooks/how_to/explore_tests.ipynb index a6c7cabcc..9fc1d3292 100644 --- a/notebooks/how_to/explore_tests.ipynb +++ b/notebooks/how_to/explore_tests.ipynb @@ -4387,7 +4387,7 @@ }, { "cell_type": "markdown", - "id": "copyright-37e237f2c9a9471ba73341a731c3f9e5", + "id": "copyright-1b1ae13be5d84b3b84274b9e7789f11b", "metadata": {}, "source": [ "\n", @@ -4397,7 +4397,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/filter_input_columns.ipynb b/notebooks/how_to/filter_input_columns.ipynb index 1d792ea11..1f1bb65c5 100644 --- a/notebooks/how_to/filter_input_columns.ipynb +++ b/notebooks/how_to/filter_input_columns.ipynb @@ -64,7 +64,7 @@ }, { "cell_type": "markdown", - "id": "copyright-ac723356fc904b3ea2324be034e1a4d3", + "id": "copyright-814bdeab9bdf4b1c87f3fc2ea3b33fbc", "metadata": {}, "source": [ "\n", @@ -74,7 +74,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/load_datasets_predictions.ipynb b/notebooks/how_to/load_datasets_predictions.ipynb index 659363c3b..a2dacdeea 100644 --- a/notebooks/how_to/load_datasets_predictions.ipynb +++ b/notebooks/how_to/load_datasets_predictions.ipynb @@ -1017,7 +1017,7 @@ }, { "cell_type": "markdown", - "id": "copyright-f0f5f5c8197b4ef4b4f50dd7b19d4d0d", + "id": "copyright-0763ff57c8834b5e80d683d17186580e", "metadata": {}, "source": [ "\n", @@ -1027,7 +1027,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/log_metrics_over_time.ipynb b/notebooks/how_to/log_metrics_over_time.ipynb index 7f79ecf32..f3bbfbdfa 100644 --- a/notebooks/how_to/log_metrics_over_time.ipynb +++ b/notebooks/how_to/log_metrics_over_time.ipynb @@ -919,7 +919,7 @@ }, { "cell_type": "markdown", - "id": "copyright-3ade6e49f883409ca0bf318bc762a074", + "id": "copyright-2b6b06cfa3254951b8ebbc1178037888", "metadata": {}, "source": [ "\n", @@ -929,7 +929,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_documentation_sections.ipynb b/notebooks/how_to/run_documentation_sections.ipynb index a77b8219f..fb4412df9 100644 --- a/notebooks/how_to/run_documentation_sections.ipynb +++ b/notebooks/how_to/run_documentation_sections.ipynb @@ -555,7 +555,7 @@ }, { "cell_type": "markdown", - "id": "copyright-91227660a8554880b1fd48c405811806", + "id": "copyright-f4756a1f66ab49598b696ed86685fcc6", "metadata": {}, "source": [ "\n", @@ -565,7 +565,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_documentation_tests_with_config.ipynb b/notebooks/how_to/run_documentation_tests_with_config.ipynb index 40cf4b08a..a96df1a5b 100644 --- a/notebooks/how_to/run_documentation_tests_with_config.ipynb +++ b/notebooks/how_to/run_documentation_tests_with_config.ipynb @@ -680,7 +680,7 @@ }, { "cell_type": "markdown", - "id": "copyright-7bfb4adf3fa74d4798cef42604f5d6f1", + "id": "copyright-d1660c97181e4b7e9470b3529ffef83d", "metadata": {}, "source": [ "\n", @@ -690,7 +690,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb b/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb index b1272dd7c..a901849c4 100644 --- a/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb +++ b/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb @@ -567,7 +567,7 @@ }, { "cell_type": "markdown", - "id": "copyright-4a0bae9136454ace8c99adf8edf1e817", + "id": "copyright-25b1e88bb9d849c8a1f3fc48aac4b6a4", "metadata": {}, "source": [ "\n", @@ -577,7 +577,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb b/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb index 10368ee89..5aa8d78db 100644 --- a/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb +++ b/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb @@ -751,7 +751,7 @@ }, { "cell_type": "markdown", - "id": "copyright-ed5da38da21649069130bbd93ea62f44", + "id": "copyright-d655a3bbc6e14b15b7466f3808556ef8", "metadata": {}, "source": [ "\n", @@ -761,7 +761,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb b/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb index 4a7152f54..d292c4461 100644 --- a/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb +++ b/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb @@ -532,7 +532,7 @@ }, { "cell_type": "markdown", - "id": "copyright-2cce0683b80246f6ada2eb0bbd3fd098", + "id": "copyright-b40eb05198f94039ae2f66d120320271", "metadata": {}, "source": [ "\n", @@ -542,7 +542,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/run_unit_metrics.ipynb b/notebooks/how_to/run_unit_metrics.ipynb index f82746374..235c538a9 100644 --- a/notebooks/how_to/run_unit_metrics.ipynb +++ b/notebooks/how_to/run_unit_metrics.ipynb @@ -756,7 +756,7 @@ }, { "cell_type": "markdown", - "id": "copyright-9e3803f1f4f141c6bbb19b62c82c4244", + "id": "copyright-238ef79bf0f045d9a112ffd540267e96", "metadata": {}, "source": [ "\n", @@ -766,7 +766,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/understand_utilize_rawdata.ipynb b/notebooks/how_to/understand_utilize_rawdata.ipynb index fbc10fc2a..5fe3c360d 100644 --- a/notebooks/how_to/understand_utilize_rawdata.ipynb +++ b/notebooks/how_to/understand_utilize_rawdata.ipynb @@ -702,7 +702,7 @@ }, { "cell_type": "markdown", - "id": "copyright-b30ff7e5445141fda373c9d60f3387ee", + "id": "copyright-d9a502e868ba4fc1a70056873609b472", "metadata": {}, "source": [ "\n", @@ -712,7 +712,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/how_to/use_dataset_model_objects.ipynb b/notebooks/how_to/use_dataset_model_objects.ipynb index b512b9e1a..12431963c 100644 --- a/notebooks/how_to/use_dataset_model_objects.ipynb +++ b/notebooks/how_to/use_dataset_model_objects.ipynb @@ -966,7 +966,7 @@ }, { "cell_type": "markdown", - "id": "copyright-15e20dc385d14d9ebd8fa506dee33251", + "id": "copyright-340a990e20194848af0efb0c965e219a", "metadata": {}, "source": [ "\n", @@ -976,7 +976,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/quickstart/quickstart_model_documentation.ipynb b/notebooks/quickstart/quickstart_model_documentation.ipynb index ec713790f..e88d9a803 100644 --- a/notebooks/quickstart/quickstart_model_documentation.ipynb +++ b/notebooks/quickstart/quickstart_model_documentation.ipynb @@ -852,7 +852,7 @@ }, { "cell_type": "markdown", - "id": "copyright-9b5d15f077ba48b9af65474494f123a0", + "id": "copyright-e26871efeffc48e386d68e90db1838e7", "metadata": {}, "source": [ "\n", @@ -862,7 +862,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/quickstart/quickstart_model_validation.ipynb b/notebooks/quickstart/quickstart_model_validation.ipynb index d54934c3a..ffa1bc42a 100644 --- a/notebooks/quickstart/quickstart_model_validation.ipynb +++ b/notebooks/quickstart/quickstart_model_validation.ipynb @@ -1173,7 +1173,7 @@ }, { "cell_type": "markdown", - "id": "copyright-a62891c4333546a68021886fbab3d9ce", + "id": "copyright-09361e6dcd874470819686c63660be51", "metadata": {}, "source": [ "\n", @@ -1183,7 +1183,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/templates/e2e-notebook.ipynb b/notebooks/templates/e2e-notebook.ipynb index be779df91..f19fc436e 100644 --- a/notebooks/templates/e2e-notebook.ipynb +++ b/notebooks/templates/e2e-notebook.ipynb @@ -281,7 +281,7 @@ }, { "cell_type": "markdown", - "id": "copyright-76242bf35234457daa55eb4f995f6cc8", + "id": "copyright-fcacdbff91e64143a630e9efa43bbe6e", "metadata": {}, "source": [ "\n", @@ -291,7 +291,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_development/1-set_up_validmind.ipynb b/notebooks/tutorials/model_development/1-set_up_validmind.ipynb index db2ae274b..f85b592d8 100644 --- a/notebooks/tutorials/model_development/1-set_up_validmind.ipynb +++ b/notebooks/tutorials/model_development/1-set_up_validmind.ipynb @@ -440,7 +440,7 @@ }, { "cell_type": "markdown", - "id": "copyright-3020855baec547b79ddd967f29b32248", + "id": "copyright-3e02e70c1d4d4840bf8d9ef44e2cf20c", "metadata": {}, "source": [ "\n", @@ -450,7 +450,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_development/2-start_development_process.ipynb b/notebooks/tutorials/model_development/2-start_development_process.ipynb index ae38cbfb3..1d62e9c08 100644 --- a/notebooks/tutorials/model_development/2-start_development_process.ipynb +++ b/notebooks/tutorials/model_development/2-start_development_process.ipynb @@ -989,7 +989,7 @@ }, { "cell_type": "markdown", - "id": "copyright-e5e5871f076545839c2d502658ac7e33", + "id": "copyright-a258273ac8eb41a29a39e52e6f3b4341", "metadata": {}, "source": [ "\n", @@ -999,7 +999,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb b/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb index 6d54af45f..49df9a5b5 100644 --- a/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb +++ b/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb @@ -968,7 +968,7 @@ }, { "cell_type": "markdown", - "id": "copyright-a43192a3977347558fcbe85bcfaec9da", + "id": "copyright-2f589b81f04949f6a2c6764859b8bc86", "metadata": {}, "source": [ "\n", @@ -978,7 +978,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb b/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb index b3a86d96e..0af95d90a 100644 --- a/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb +++ b/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb @@ -967,7 +967,7 @@ }, { "cell_type": "markdown", - "id": "copyright-6540b3502fe94eed919c7109a67f31cd", + "id": "copyright-6ef1ef2b30eb44c29b99e625c58826ee", "metadata": {}, "source": [ "\n", @@ -977,7 +977,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb b/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb index 43ee7a3e5..b9154b80c 100644 --- a/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb +++ b/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb @@ -491,7 +491,7 @@ }, { "cell_type": "markdown", - "id": "copyright-01931314846445b888659f6f15b2189b", + "id": "copyright-5d7a1c159e4840fca79011d1c0380725", "metadata": {}, "source": [ "\n", @@ -501,7 +501,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_validation/2-start_validation_process.ipynb b/notebooks/tutorials/model_validation/2-start_validation_process.ipynb index 71aedda0f..48e4dbff6 100644 --- a/notebooks/tutorials/model_validation/2-start_validation_process.ipynb +++ b/notebooks/tutorials/model_validation/2-start_validation_process.ipynb @@ -862,7 +862,7 @@ }, { "cell_type": "markdown", - "id": "copyright-9f5daf1c86b346fd87ad7fa53417c7fb", + "id": "copyright-149e9401b9df40d393a2a1958331dea6", "metadata": {}, "source": [ "\n", @@ -872,7 +872,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb b/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb index b312f4735..5f7d7e8fd 100644 --- a/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb +++ b/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb @@ -865,7 +865,7 @@ }, { "cell_type": "markdown", - "id": "copyright-9dae2efaa1024dee9020ae673b5817a7", + "id": "copyright-58757a9cc9de45069a5e6c57c8aaff14", "metadata": {}, "source": [ "\n", @@ -875,7 +875,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } diff --git a/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb b/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb index d1fe90923..fe4e221c9 100644 --- a/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb +++ b/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb @@ -1200,7 +1200,7 @@ }, { "cell_type": "markdown", - "id": "copyright-20b8614cb05a45f4bb5adb7ceb939482", + "id": "copyright-53539c44edf54e4e9bc4d959ffbddd2c", "metadata": {}, "source": [ "\n", @@ -1210,7 +1210,7 @@ "***\n", "\n", "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE in the root of the GitHub `validmind-library` repository](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" ] } From 4ef319ffe505f20ae9ae6f3cb2b7d091d45142e5 Mon Sep 17 00:00:00 2001 From: Nik Richers Date: Mon, 19 Jan 2026 16:22:22 -0800 Subject: [PATCH 26/31] Switch template script to use Cursor --- notebooks/templates/e2e_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/templates/e2e_template.py b/notebooks/templates/e2e_template.py index 3f3a08db5..f50710ad1 100644 --- a/notebooks/templates/e2e_template.py +++ b/notebooks/templates/e2e_template.py @@ -51,7 +51,7 @@ def create_notebook(): nbformat.write(notebook, f) print(f"Created '{filepath}'") - subprocess.run(["code", filepath], check=True) + subprocess.run(["cursor", filepath], check=True) except Exception as e: print(f"Error creating or opening notebook: {e}") From 0f71f4bc759c7c0ad8c173c7195b7ab0e1bbdaaa Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 16:42:39 -0800 Subject: [PATCH 27/31] Testing editor detection --- notebooks/templates/e2e_template.py | 73 ++++++++++++++++++++++++++++- validmind/tests/__types__.py | 24 ++++++++++ validmind/tests/load.py | 8 ++-- 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/notebooks/templates/e2e_template.py b/notebooks/templates/e2e_template.py index f50710ad1..3a392bf12 100644 --- a/notebooks/templates/e2e_template.py +++ b/notebooks/templates/e2e_template.py @@ -4,6 +4,8 @@ import uuid import subprocess import json +import sys +import platform from typing import Callable, Dict, Iterable, Optional, Set, Tuple def ensure_ids(notebook): @@ -13,8 +15,75 @@ def ensure_ids(notebook): cell["id"] = str(uuid.uuid4()) return notebook +def detect_editor(): + """Detect the currently running editor from environment variables.""" + # Check common environment variables set by IDEs + env_checks = [ + ("CURSOR_PATH", "cursor"), + ("VSCODE_PID", "code"), + ("VSCODE_IPC_HOOK", "code"), + ("VSCODE_CWD", "code"), + ("TERM_PROGRAM", { + "cursor": "cursor", + "vscode": "code", + "Apple_Terminal": None, + "iTerm.app": None, + }), + ("JUPYTER_SERVER_ROOT", "jupyter"), + ] + + for env_var, editor in env_checks: + value = os.environ.get(env_var) + if value: + if isinstance(editor, dict): + # For TERM_PROGRAM, map the value to the command + for term_value, cmd in editor.items(): + if term_value.lower() in value.lower(): + return cmd + else: + return editor + + return None + +def open_in(filepath): + """Try to open a file in the current editor or system default application.""" + # Try to detect the current editor first + detected_editor = detect_editor() + + # Build the list of commands, prioritizing the detected editor + ide_commands = ["cursor", "code", "jupyter", "jupyter-lab", "jupyter-notebook"] + + if detected_editor and detected_editor in ide_commands: + # Move detected editor to the front of the list + ide_commands.remove(detected_editor) + ide_commands.insert(0, detected_editor) + + # Try each command + for cmd in ide_commands: + try: + subprocess.run([cmd, filepath], check=True, capture_output=True, timeout=2) + detected_msg = " (detected)" if cmd == detected_editor else "" + print(f"Opened in {cmd}{detected_msg}") + return + except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired): + continue + + # Fallback to system default application + try: + system = platform.system() + if system == "Darwin": # macOS + subprocess.run(["open", filepath], check=True) + elif system == "Windows": + subprocess.run(["start", filepath], shell=True, check=True) + elif system == "Linux": + subprocess.run(["xdg-open", filepath], check=True) + print(f"Opened with system default application") + except Exception as e: + print(f"Could not automatically open file. Please open manually: {filepath}") + print(f"Error: {e}") + def create_notebook(): - """Creates a new Jupyter Notebook file by asking the user for a filename and opens it in VS Code.""" + """Creates a new Jupyter Notebook file by asking the user for a filename and opens it.""" filename = input("Enter the name for the new notebook (without .ipynb extension): ").strip() if not filename: print("Filename cannot be empty, file not created") @@ -51,7 +120,7 @@ def create_notebook(): nbformat.write(notebook, f) print(f"Created '{filepath}'") - subprocess.run(["cursor", filepath], check=True) + open_in(filepath) except Exception as e: print(f"Error creating or opening notebook: {e}") diff --git a/validmind/tests/__types__.py b/validmind/tests/__types__.py index 74c247209..d936d38f6 100644 --- a/validmind/tests/__types__.py +++ b/validmind/tests/__types__.py @@ -198,6 +198,30 @@ "validmind.prompt_validation.NegativeInstruction", "validmind.prompt_validation.Robustness", "validmind.prompt_validation.Specificity", + "validmind.scorers.classification.AbsoluteError", + "validmind.scorers.classification.BrierScore", + "validmind.scorers.classification.CalibrationError", + "validmind.scorers.classification.ClassBalance", + "validmind.scorers.classification.Confidence", + "validmind.scorers.classification.Correctness", + "validmind.scorers.classification.LogLoss", + "validmind.scorers.classification.OutlierScore", + "validmind.scorers.classification.ProbabilityError", + "validmind.scorers.classification.Uncertainty", + "validmind.scorers.llm.deepeval.AnswerRelevancy", + "validmind.scorers.llm.deepeval.ArgumentCorrectness", + "validmind.scorers.llm.deepeval.Bias", + "validmind.scorers.llm.deepeval.ContextualPrecision", + "validmind.scorers.llm.deepeval.ContextualRecall", + "validmind.scorers.llm.deepeval.ContextualRelevancy", + "validmind.scorers.llm.deepeval.Faithfulness", + "validmind.scorers.llm.deepeval.GEval", + "validmind.scorers.llm.deepeval.Hallucination", + "validmind.scorers.llm.deepeval.PlanAdherence", + "validmind.scorers.llm.deepeval.PlanQuality", + "validmind.scorers.llm.deepeval.Summarization", + "validmind.scorers.llm.deepeval.TaskCompletion", + "validmind.scorers.llm.deepeval.ToolCorrectness", "validmind.stats.CorrelationAnalysis", "validmind.stats.DescriptiveStats", "validmind.stats.NormalityTests", diff --git a/validmind/tests/load.py b/validmind/tests/load.py index 4dc97d11a..9a9f13c53 100644 --- a/validmind/tests/load.py +++ b/validmind/tests/load.py @@ -127,7 +127,9 @@ def _inspect_signature( return inputs, params -def _get_test_function_from_provider(test_id: str, namespace: str) -> Callable[..., Any]: +def _get_test_function_from_provider( + test_id: str, namespace: str +) -> Callable[..., Any]: """Load a test function from the appropriate provider or scorer store. Args: @@ -146,9 +148,7 @@ def _get_test_function_from_provider(test_id: str, namespace: str) -> Callable[. return custom_scorer if not test_provider_store.has_test_provider(namespace): - raise LoadTestError( - f"No test provider found for namespace: {namespace}" - ) + raise LoadTestError(f"No test provider found for namespace: {namespace}") provider = test_provider_store.get_test_provider(namespace) From 0239094a9c7d32d408f58d5f6d1cc9f5a2cc8f0f Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 16:47:26 -0800 Subject: [PATCH 28/31] Tweak --- notebooks/templates/e2e_template.py | 76 +++++++++++++++++++---------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/notebooks/templates/e2e_template.py b/notebooks/templates/e2e_template.py index 3a392bf12..2dbd2cb38 100644 --- a/notebooks/templates/e2e_template.py +++ b/notebooks/templates/e2e_template.py @@ -16,32 +16,56 @@ def ensure_ids(notebook): return notebook def detect_editor(): - """Detect the currently running editor from environment variables.""" - # Check common environment variables set by IDEs - env_checks = [ - ("CURSOR_PATH", "cursor"), - ("VSCODE_PID", "code"), - ("VSCODE_IPC_HOOK", "code"), - ("VSCODE_CWD", "code"), - ("TERM_PROGRAM", { - "cursor": "cursor", - "vscode": "code", - "Apple_Terminal": None, - "iTerm.app": None, - }), - ("JUPYTER_SERVER_ROOT", "jupyter"), - ] - - for env_var, editor in env_checks: - value = os.environ.get(env_var) - if value: - if isinstance(editor, dict): - # For TERM_PROGRAM, map the value to the command - for term_value, cmd in editor.items(): - if term_value.lower() in value.lower(): - return cmd - else: - return editor + """Detect the currently running editor from environment variables and process info.""" + # First check TERM_PROGRAM which is most reliable for distinguishing Cursor from VS Code + term_program = os.environ.get("TERM_PROGRAM", "") + if "cursor" in term_program.lower(): + return "cursor" + elif "vscode" in term_program.lower(): + return "code" + + # Check for Cursor-specific environment variables + if os.environ.get("CURSOR_PATH"): + return "cursor" + + # Check the parent process to see what's actually running + try: + # Get parent process ID + ppid = os.getppid() + if platform.system() == "Darwin": # macOS + # Use ps to get the parent process name + result = subprocess.run( + ["ps", "-p", str(ppid), "-o", "comm="], + capture_output=True, + text=True, + timeout=1 + ) + process_name = result.stdout.strip().lower() + if "cursor" in process_name: + return "cursor" + elif "code" in process_name or "vscode" in process_name: + return "code" + elif platform.system() == "Linux": + # Check /proc on Linux + try: + with open(f"/proc/{ppid}/comm", "r") as f: + process_name = f.read().strip().lower() + if "cursor" in process_name: + return "cursor" + elif "code" in process_name or "vscode" in process_name: + return "code" + except: + pass + except: + pass + + # Check other VS Code environment variables (will catch VS Code but not distinguish from Cursor) + if os.environ.get("VSCODE_PID") or os.environ.get("VSCODE_IPC_HOOK") or os.environ.get("VSCODE_CWD"): + return "code" + + # Check for Jupyter + if os.environ.get("JUPYTER_SERVER_ROOT"): + return "jupyter" return None From e0a351488c74416c334027643e7464c694e55e10 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 16:56:05 -0800 Subject: [PATCH 29/31] Tweak 2 --- notebooks/templates/e2e_template.py | 261 ++++++++++++++++++++++++---- 1 file changed, 229 insertions(+), 32 deletions(-) diff --git a/notebooks/templates/e2e_template.py b/notebooks/templates/e2e_template.py index 2dbd2cb38..21293b000 100644 --- a/notebooks/templates/e2e_template.py +++ b/notebooks/templates/e2e_template.py @@ -17,51 +17,142 @@ def ensure_ids(notebook): def detect_editor(): """Detect the currently running editor from environment variables and process info.""" - # First check TERM_PROGRAM which is most reliable for distinguishing Cursor from VS Code - term_program = os.environ.get("TERM_PROGRAM", "") - if "cursor" in term_program.lower(): - return "cursor" - elif "vscode" in term_program.lower(): - return "code" - - # Check for Cursor-specific environment variables - if os.environ.get("CURSOR_PATH"): - return "cursor" + # Walk up the process tree FIRST (most reliable for distinguishing Cursor from VS Code in terminal) + vscode_found = False - # Check the parent process to see what's actually running try: - # Get parent process ID - ppid = os.getppid() if platform.system() == "Darwin": # macOS - # Use ps to get the parent process name - result = subprocess.run( - ["ps", "-p", str(ppid), "-o", "comm="], - capture_output=True, - text=True, - timeout=1 - ) - process_name = result.stdout.strip().lower() - if "cursor" in process_name: - return "cursor" - elif "code" in process_name or "vscode" in process_name: + current_pid = os.getpid() + for _ in range(20): # Check up to 20 levels up + # Get info for this specific PID + result = subprocess.run( + ["ps", "-p", str(current_pid), "-o", "ppid=,comm="], + capture_output=True, + text=True, + timeout=1 + ) + + if result.returncode != 0: + break + + output = result.stdout.strip() + if not output: + break + + parts = output.split(None, 1) + if len(parts) < 2: + break + + ppid = int(parts[0]) + comm = parts[1].lower() + + # Check for Cursor first (it's more specific) + if "cursor" in comm: + return "cursor" + elif "code" in comm or "vscode" in comm: + # Found code/vscode, but keep looking for Cursor + vscode_found = True + + current_pid = ppid + if ppid <= 1: # Reached init/launchd + break + + # If we found vscode but not cursor, return code + if vscode_found: return "code" + elif platform.system() == "Linux": - # Check /proc on Linux + # Walk up the process tree on Linux using /proc + current_pid = os.getpid() + for _ in range(20): # Check up to 20 levels up + try: + # Read process name + with open(f"/proc/{current_pid}/comm", "r") as f: + process_name = f.read().strip().lower() + if "cursor" in process_name: + return "cursor" + elif "code" in process_name or "vscode" in process_name: + vscode_found = True + + # Get parent PID from stat file + with open(f"/proc/{current_pid}/stat", "r") as f: + stat = f.read().split() + current_pid = int(stat[3]) # ppid is 4th field + + if current_pid <= 1: + break + except: + break + + if vscode_found: + return "code" + + elif platform.system() == "Windows": + # Windows process tree detection using wmic or PowerShell try: - with open(f"/proc/{ppid}/comm", "r") as f: - process_name = f.read().strip().lower() - if "cursor" in process_name: + current_pid = os.getpid() + for _ in range(20): + # Use wmic to get parent process info + result = subprocess.run( + ["wmic", "process", "where", f"ProcessId={current_pid}", + "get", "ParentProcessId,Name", "/format:list"], + capture_output=True, + text=True, + timeout=2 + ) + + if result.returncode != 0: + break + + # Parse wmic output + name = "" + ppid = None + for line in result.stdout.split("\n"): + if "Name=" in line: + name = line.split("=", 1)[1].strip().lower() + elif "ParentProcessId=" in line: + ppid_str = line.split("=", 1)[1].strip() + if ppid_str: + ppid = int(ppid_str) + + if not name or ppid is None: + break + + # Check for Cursor or VS Code + if "cursor" in name: return "cursor" - elif "code" in process_name or "vscode" in process_name: - return "code" + elif "code" in name or "vscode" in name: + vscode_found = True + + current_pid = ppid + if ppid <= 0: + break + + if vscode_found: + return "code" except: pass except: pass - # Check other VS Code environment variables (will catch VS Code but not distinguish from Cursor) + # Check TERM_PROGRAM (but this is less reliable for Cursor terminal) + term_program = os.environ.get("TERM_PROGRAM", "") + if "cursor" in term_program.lower(): + return "cursor" + + # Check for Cursor-specific environment variables + if os.environ.get("CURSOR_PATH"): + return "cursor" + + # Check other VS Code environment variables if os.environ.get("VSCODE_PID") or os.environ.get("VSCODE_IPC_HOOK") or os.environ.get("VSCODE_CWD"): - return "code" + # Try to find cursor in the path + vscode_ipc = os.environ.get("VSCODE_IPC_HOOK", "") + if "cursor" in vscode_ipc.lower(): + return "cursor" + # Only return code if TERM_PROGRAM confirms it's vscode (not cursor) + if term_program.lower() == "vscode": + return "code" # Check for Jupyter if os.environ.get("JUPYTER_SERVER_ROOT"): @@ -588,6 +679,112 @@ def add_copyright(filepath: str) -> None: script_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(script_dir) + + # Debug: show detected editor and relevant environment variables + print("=== Debug Info ===") + print(f"TERM_PROGRAM: {os.environ.get('TERM_PROGRAM', 'not set')}") + print(f"CURSOR_PATH: {os.environ.get('CURSOR_PATH', 'not set')}") + print(f"VSCODE_IPC_HOOK: {os.environ.get('VSCODE_IPC_HOOK', 'not set')}") + + # Show process tree + system = platform.system() + try: + print("\nProcess tree:") + current_pid = os.getpid() + + if system == "Darwin": # macOS + for i in range(20): + result = subprocess.run( + ["ps", "-p", str(current_pid), "-o", "ppid=,comm="], + capture_output=True, + text=True, + timeout=1 + ) + + if result.returncode != 0: + print(f" {i}: (process not found)") + break + + output = result.stdout.strip() + if not output: + break + + parts = output.split(None, 1) + if len(parts) < 2: + break + + ppid = int(parts[0]) + comm = parts[1] + + print(f" {i}: {comm} (PID: {current_pid}, Parent: {ppid})") + + current_pid = ppid + if ppid <= 1: + print(f" Reached init/launchd") + break + + elif system == "Linux": + for i in range(20): + try: + with open(f"/proc/{current_pid}/comm", "r") as f: + comm = f.read().strip() + + with open(f"/proc/{current_pid}/stat", "r") as f: + stat = f.read().split() + ppid = int(stat[3]) + + print(f" {i}: {comm} (PID: {current_pid}, Parent: {ppid})") + + current_pid = ppid + if ppid <= 1: + print(f" Reached init") + break + except: + print(f" {i}: (could not read process)") + break + + elif system == "Windows": + for i in range(20): + result = subprocess.run( + ["wmic", "process", "where", f"ProcessId={current_pid}", + "get", "ParentProcessId,Name", "/format:list"], + capture_output=True, + text=True, + timeout=2 + ) + + if result.returncode != 0: + print(f" {i}: (process not found)") + break + + name = "" + ppid = None + for line in result.stdout.split("\n"): + if "Name=" in line: + name = line.split("=", 1)[1].strip() + elif "ParentProcessId=" in line: + ppid_str = line.split("=", 1)[1].strip() + if ppid_str: + ppid = int(ppid_str) + + if not name or ppid is None: + break + + print(f" {i}: {name} (PID: {current_pid}, Parent: {ppid})") + + current_pid = ppid + if ppid <= 0: + print(f" Reached system process") + break + else: + print(f" Process tree inspection not supported on {system}") + + except Exception as e: + print(f"Could not get process tree: {e}") + + detected = detect_editor() + print(f"\nDetected editor: {detected or 'none'}") + print("==================\n") filepath = create_notebook() if filepath: From b0736e26d6d40581b241c8c1903fa7c393b1e933 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Mon, 19 Jan 2026 16:58:06 -0800 Subject: [PATCH 30/31] Remove debugging info --- notebooks/templates/e2e_template.py | 154 +++++----------------------- 1 file changed, 24 insertions(+), 130 deletions(-) diff --git a/notebooks/templates/e2e_template.py b/notebooks/templates/e2e_template.py index 21293b000..be4914c32 100644 --- a/notebooks/templates/e2e_template.py +++ b/notebooks/templates/e2e_template.py @@ -19,7 +19,7 @@ def detect_editor(): """Detect the currently running editor from environment variables and process info.""" # Walk up the process tree FIRST (most reliable for distinguishing Cursor from VS Code in terminal) vscode_found = False - + try: if platform.system() == "Darwin": # macOS current_pid = os.getpid() @@ -31,36 +31,36 @@ def detect_editor(): text=True, timeout=1 ) - + if result.returncode != 0: break - + output = result.stdout.strip() if not output: break - + parts = output.split(None, 1) if len(parts) < 2: break - + ppid = int(parts[0]) comm = parts[1].lower() - + # Check for Cursor first (it's more specific) if "cursor" in comm: return "cursor" elif "code" in comm or "vscode" in comm: # Found code/vscode, but keep looking for Cursor vscode_found = True - + current_pid = ppid if ppid <= 1: # Reached init/launchd break - + # If we found vscode but not cursor, return code if vscode_found: return "code" - + elif platform.system() == "Linux": # Walk up the process tree on Linux using /proc current_pid = os.getpid() @@ -73,20 +73,20 @@ def detect_editor(): return "cursor" elif "code" in process_name or "vscode" in process_name: vscode_found = True - + # Get parent PID from stat file with open(f"/proc/{current_pid}/stat", "r") as f: stat = f.read().split() current_pid = int(stat[3]) # ppid is 4th field - + if current_pid <= 1: break except: break - + if vscode_found: return "code" - + elif platform.system() == "Windows": # Windows process tree detection using wmic or PowerShell try: @@ -94,16 +94,16 @@ def detect_editor(): for _ in range(20): # Use wmic to get parent process info result = subprocess.run( - ["wmic", "process", "where", f"ProcessId={current_pid}", + ["wmic", "process", "where", f"ProcessId={current_pid}", "get", "ParentProcessId,Name", "/format:list"], capture_output=True, text=True, timeout=2 ) - + if result.returncode != 0: break - + # Parse wmic output name = "" ppid = None @@ -114,36 +114,36 @@ def detect_editor(): ppid_str = line.split("=", 1)[1].strip() if ppid_str: ppid = int(ppid_str) - + if not name or ppid is None: break - + # Check for Cursor or VS Code if "cursor" in name: return "cursor" elif "code" in name or "vscode" in name: vscode_found = True - + current_pid = ppid if ppid <= 0: break - + if vscode_found: return "code" except: pass except: pass - + # Check TERM_PROGRAM (but this is less reliable for Cursor terminal) term_program = os.environ.get("TERM_PROGRAM", "") if "cursor" in term_program.lower(): return "cursor" - + # Check for Cursor-specific environment variables if os.environ.get("CURSOR_PATH"): return "cursor" - + # Check other VS Code environment variables if os.environ.get("VSCODE_PID") or os.environ.get("VSCODE_IPC_HOOK") or os.environ.get("VSCODE_CWD"): # Try to find cursor in the path @@ -153,7 +153,7 @@ def detect_editor(): # Only return code if TERM_PROGRAM confirms it's vscode (not cursor) if term_program.lower() == "vscode": return "code" - + # Check for Jupyter if os.environ.get("JUPYTER_SERVER_ROOT"): return "jupyter" @@ -679,112 +679,6 @@ def add_copyright(filepath: str) -> None: script_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(script_dir) - - # Debug: show detected editor and relevant environment variables - print("=== Debug Info ===") - print(f"TERM_PROGRAM: {os.environ.get('TERM_PROGRAM', 'not set')}") - print(f"CURSOR_PATH: {os.environ.get('CURSOR_PATH', 'not set')}") - print(f"VSCODE_IPC_HOOK: {os.environ.get('VSCODE_IPC_HOOK', 'not set')}") - - # Show process tree - system = platform.system() - try: - print("\nProcess tree:") - current_pid = os.getpid() - - if system == "Darwin": # macOS - for i in range(20): - result = subprocess.run( - ["ps", "-p", str(current_pid), "-o", "ppid=,comm="], - capture_output=True, - text=True, - timeout=1 - ) - - if result.returncode != 0: - print(f" {i}: (process not found)") - break - - output = result.stdout.strip() - if not output: - break - - parts = output.split(None, 1) - if len(parts) < 2: - break - - ppid = int(parts[0]) - comm = parts[1] - - print(f" {i}: {comm} (PID: {current_pid}, Parent: {ppid})") - - current_pid = ppid - if ppid <= 1: - print(f" Reached init/launchd") - break - - elif system == "Linux": - for i in range(20): - try: - with open(f"/proc/{current_pid}/comm", "r") as f: - comm = f.read().strip() - - with open(f"/proc/{current_pid}/stat", "r") as f: - stat = f.read().split() - ppid = int(stat[3]) - - print(f" {i}: {comm} (PID: {current_pid}, Parent: {ppid})") - - current_pid = ppid - if ppid <= 1: - print(f" Reached init") - break - except: - print(f" {i}: (could not read process)") - break - - elif system == "Windows": - for i in range(20): - result = subprocess.run( - ["wmic", "process", "where", f"ProcessId={current_pid}", - "get", "ParentProcessId,Name", "/format:list"], - capture_output=True, - text=True, - timeout=2 - ) - - if result.returncode != 0: - print(f" {i}: (process not found)") - break - - name = "" - ppid = None - for line in result.stdout.split("\n"): - if "Name=" in line: - name = line.split("=", 1)[1].strip() - elif "ParentProcessId=" in line: - ppid_str = line.split("=", 1)[1].strip() - if ppid_str: - ppid = int(ppid_str) - - if not name or ppid is None: - break - - print(f" {i}: {name} (PID: {current_pid}, Parent: {ppid})") - - current_pid = ppid - if ppid <= 0: - print(f" Reached system process") - break - else: - print(f" Process tree inspection not supported on {system}") - - except Exception as e: - print(f"Could not get process tree: {e}") - - detected = detect_editor() - print(f"\nDetected editor: {detected or 'none'}") - print("==================\n") filepath = create_notebook() if filepath: From 829ec8e0415742f6895969a8d6fd3bfc0a69a5f3 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Tue, 20 Jan 2026 10:00:23 -0800 Subject: [PATCH 31/31] Revert validmind/tests/ to main? --- validmind/tests/__types__.py | 24 ------------------------ validmind/tests/load.py | 8 ++++---- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/validmind/tests/__types__.py b/validmind/tests/__types__.py index d936d38f6..74c247209 100644 --- a/validmind/tests/__types__.py +++ b/validmind/tests/__types__.py @@ -198,30 +198,6 @@ "validmind.prompt_validation.NegativeInstruction", "validmind.prompt_validation.Robustness", "validmind.prompt_validation.Specificity", - "validmind.scorers.classification.AbsoluteError", - "validmind.scorers.classification.BrierScore", - "validmind.scorers.classification.CalibrationError", - "validmind.scorers.classification.ClassBalance", - "validmind.scorers.classification.Confidence", - "validmind.scorers.classification.Correctness", - "validmind.scorers.classification.LogLoss", - "validmind.scorers.classification.OutlierScore", - "validmind.scorers.classification.ProbabilityError", - "validmind.scorers.classification.Uncertainty", - "validmind.scorers.llm.deepeval.AnswerRelevancy", - "validmind.scorers.llm.deepeval.ArgumentCorrectness", - "validmind.scorers.llm.deepeval.Bias", - "validmind.scorers.llm.deepeval.ContextualPrecision", - "validmind.scorers.llm.deepeval.ContextualRecall", - "validmind.scorers.llm.deepeval.ContextualRelevancy", - "validmind.scorers.llm.deepeval.Faithfulness", - "validmind.scorers.llm.deepeval.GEval", - "validmind.scorers.llm.deepeval.Hallucination", - "validmind.scorers.llm.deepeval.PlanAdherence", - "validmind.scorers.llm.deepeval.PlanQuality", - "validmind.scorers.llm.deepeval.Summarization", - "validmind.scorers.llm.deepeval.TaskCompletion", - "validmind.scorers.llm.deepeval.ToolCorrectness", "validmind.stats.CorrelationAnalysis", "validmind.stats.DescriptiveStats", "validmind.stats.NormalityTests", diff --git a/validmind/tests/load.py b/validmind/tests/load.py index 9a9f13c53..4dc97d11a 100644 --- a/validmind/tests/load.py +++ b/validmind/tests/load.py @@ -127,9 +127,7 @@ def _inspect_signature( return inputs, params -def _get_test_function_from_provider( - test_id: str, namespace: str -) -> Callable[..., Any]: +def _get_test_function_from_provider(test_id: str, namespace: str) -> Callable[..., Any]: """Load a test function from the appropriate provider or scorer store. Args: @@ -148,7 +146,9 @@ def _get_test_function_from_provider( return custom_scorer if not test_provider_store.has_test_provider(namespace): - raise LoadTestError(f"No test provider found for namespace: {namespace}") + raise LoadTestError( + f"No test provider found for namespace: {namespace}" + ) provider = test_provider_store.get_test_provider(namespace)