diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index da46c4020..def6dfbc1 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -10,7 +10,7 @@ on:
- prod
- release-v1
paths-ignore:
- - 'docs/_build/**'
+ - 'docs/**'
permissions:
contents: read
diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml
index d137a5b07..2905fb354 100644
--- a/.github/workflows/python.yaml
+++ b/.github/workflows/python.yaml
@@ -7,13 +7,13 @@ on:
push:
branches: [main]
paths-ignore:
- - 'docs/_build/**'
+ - 'docs/**'
- 'notebooks/**'
- 'scripts/**'
pull_request:
branches: ['*']
paths-ignore:
- - 'docs/_build/**'
+ - 'docs/**'
- 'notebooks/**'
- 'scripts/**'
@@ -79,3 +79,7 @@ jobs:
# Tests that we can build the docs
- name: Generate Docs
run: make docs
+
+ # Tests that we can build the quarto docs
+ - name: Generate Quarto Docs
+ run: make quarto-docs
diff --git a/.github/workflows/docs.yaml b/.github/workflows/quarto-docs.yaml
similarity index 57%
rename from .github/workflows/docs.yaml
rename to .github/workflows/quarto-docs.yaml
index 8db50953f..4cfdabaac 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/quarto-docs.yaml
@@ -1,27 +1,26 @@
-# This workflow will install Python dependencies and generate Markdown
-# documentation from docstrings using Sphinx. We generate the HTML
-# documentation to keep it up to date with the Markdown files
-name: Python Library API docs
+# This workflow will install Python dependencies and generate
+# Quarto documentation using Griffe for API extraction and
+# Jinja2 templates for the docs and navigation.
+name: Python Library API docs for Quarto
on:
push:
branches:
- main
- - release-v1
paths-ignore:
- - 'docs/_build/**'
+ - "docs/**"
workflow_dispatch:
inputs:
note:
- description: 'Provide a description of the changes'
+ description: "Provide a description of the changes"
required: true
- default: 'Update docs'
+ default: "Update quarto docs"
permissions:
contents: write
jobs:
- docs:
+ quarto-docs:
runs-on:
group: ubuntu-vm-large
@@ -36,8 +35,8 @@ jobs:
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
- python-version: '3.11'
- cache: 'poetry'
+ python-version: "3.11"
+ cache: "poetry"
- name: Install Dependencies
run: |
@@ -45,13 +44,16 @@ jobs:
poetry install -E huggingface -E llm
poetry run pip install torch==2.0.1 --extra-index-url https://download.pytorch.org/whl/cpu
poetry run pip install aequitas fairlearn vl-convert-python
+ poetry run pip install griffe mdformat docstring_parser
- - name: Generate Docs
- run: make docs
+ - name: Generate Quarto Docs
+ run: make quarto-docs
- name: Commit changes
uses: EndBug/add-and-commit@v9
with:
default_author: github_actions
- message: 'Generate docs'
- add: 'docs/_build/'
+ message: "Generate quarto docs"
+ add: "docs/"
+ pathspec_error_handling: ignore
+ push: true
diff --git a/.gitignore b/.gitignore
index 0ce6f7c95..0155e5934 100644
--- a/.gitignore
+++ b/.gitignore
@@ -191,6 +191,10 @@ lending_club_loan_data_*.csv
# Pickle files
*.pkl
+# Sample application scorecard model for validation notebook — do not remove!
+!notebooks/code_samples/model_validation/xgb_model_champion.pkl
+# Sample logistic regression model for validation series — do not remove!
+!notebooks/tutorials/model_validation/lr_model_champion.pkl
notebooks/llm/datasets/*.jsonl
@@ -214,3 +218,6 @@ my_tests/
*.sqlite
*.db
*.db-journal
+
+# Quarto docs
+docs/validmind.json
diff --git a/Makefile b/Makefile
index d7494b6f9..f2a8f0652 100644
--- a/Makefile
+++ b/Makefile
@@ -51,6 +51,18 @@ else
poetry run pdoc validmind -d google -t docs/templates --no-show-source --logo https://vmai.s3.us-west-1.amazonaws.com/validmind-logo.svg --favicon https://vmai.s3.us-west-1.amazonaws.com/favicon.ico
endif
+quarto-docs:
+ # Clean old files
+ rm -f docs/validmind.json
+ rm -rf docs/validmind
+ mkdir -p docs/validmind
+
+ # Generate API JSON dump
+ poetry run python -m griffe dump validmind -f -o docs/validmind.json -d google -r -U
+
+ # Generate Quarto docs from templates
+ poetry run python scripts/generate_quarto_docs.py
+
version:
@:$(call check_defined, tag, new semver version tag to use on pyproject.toml)
@poetry version $(tag)
@@ -78,7 +90,7 @@ ensure-clean-notebooks:
# Quick target to run all checks
check: copyright format lint test verify-copyright verify-exposed-credentials ensure-clean-notebooks
-.PHONY: docs
+.PHONY: docs quarto-docs
notebook:
@python notebooks/templates/e2e_template.py
diff --git a/README.md b/README.md
index 271f13032..767c9b947 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ We believe in the power of collaboration and welcome contributions to the ValidM
- Interested in connecting with fellow AI model risk practitioners? Join our [Community Slack](https://docs.validmind.ai/about/contributing/join-community.html)!
-- For more information about ValidMind's open-source tests and Jupyter Notebooks, read the [ValidMind Library docs](https://docs.validmind.ai/developer/get-started-validmind-library.html).
+- For more information about ValidMind's open-source tests and Jupyter Notebooks, read the [ValidMind Library docs](https://docs.validmind.ai/developer/validmind-library.html).
## Getting started
diff --git a/README.pypi.md b/README.pypi.md
index b1553674d..c7e35005b 100644
--- a/README.pypi.md
+++ b/README.pypi.md
@@ -21,7 +21,7 @@ ValidMind helps developers, data scientists and risk and compliance stakeholders
>
> Signing up is FREE — **[Register with ValidMind](https://docs.validmind.ai/guide/configuration/register-with-validmind.html)**
-That's right — you can run tests and log documentation even if you don't have a model available, so go ahead and [**Get started with the ValidMind Library**](https://docs.validmind.ai/developer/get-started-validmind-library.html)!
+That's right — you can run tests and log documentation even if you don't have a model available, so go ahead and get started with the [**ValidMind Library**](https://docs.validmind.ai/developer/validmind-library.html)!
### How do I do more with the ValidMind Library?
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 000000000..b9b133f51
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,104 @@
+# Generating Quarto Markdown for the Python API
+
+This directory includes templates, macros, CSS, and Quarto Markdown output for generating the ValidMind Library (Python API) reference documentation for our docs site.
+
+- `templates/` — Jinja2 templates and macros for generating Quarto Markdown
+- `validmind.css` — CSS for Python API reference styling
+- `_metadata.yml` — Quarto configuration file
+- `_sidebar.yml` — Generated sidebar navigation fragment for Quarto
+- `validmind.qmd`, `validmind/` — Generated API documentation matching the codebase and pdoc structure
+
+## Testing locally
+
+To generate Quarto Markdown locally:
+
+```sh
+make quarto-docs
+```
+
+## How it works
+
+1. Griffe extracts API information from the API codebase and dumps it to a JSON file
+2. A Python script extracts API information from the JSON, processes it, and passes it to Jinja2 templates
+3. Jinja2 templates and shared macros transform this information into Quarto Markdown files
+4. A sidebar navigation fragment is generated based on the output file structure
+5. CI/CD integration tests the Quarto docs generation and commits the output
+6. Documentation repo: Integrates the Quarto files with the docs site source
+
+```mermaid
+flowchart LR
+ make[make quarto-docs] --> clean[Clean old files]
+ clean --> mkdir[Create folder structure]
+ mkdir --> Griffe[Dump API JSON]
+ Griffe --> processJSON[Process API JSON]
+
+ processJSON --> output[Generate QMD files]
+ processJSON --> nav[Generate _sidebar.yml]
+
+ subgraph "Templates"
+ templates[Jinja2 Templates] --> mod_t[module.qmd.jinja2]
+ templates --> class_t[class.qmd.jinja2]
+ templates --> func_t[function.qmd.jinja2]
+ templates --> sidebar_t[sidebar.qmd.jinja2]
+ templates --> version_t[version.qmd.jinja2]
+ templates --> errors_t[errors.qmd.jinja2]
+ templates --> macros[macros/*.jinja2]
+ end
+
+ templates --> processJSON
+
+ output --> test[Integration tests]
+ nav --> test
+
+ subgraph "CI/CD"
+ test --> commit[Commit generated docs]
+ end
+```
+
+### `Makefile`
+
+- `make quarto-docs` — Generates Quarto Markdown from the Python API
+- `make python-docs` — In the documentation repo: Clones this repo, copies the generated Quarto Markdown files over into the docs site source
+
+### GitHub actions
+
+- `.github/integration.yaml` and `.github/python.yaml` — Tests Quarto Markdown generation
+- `.github/quarto-docs.yaml` — Generates and commits Quarto Markdown docs
+
+### Jinja2 Templates
+
+Located in `templates/`, these define how Quarto Markdown is output:
+
+- `module.qmd.jinja2` — Documents Python modules, including functions and classes
+- `version.qmd.jinja2` — Displays library version information
+- `class.qmd.jinja2` — Details class documentation with inheritance and methods
+- `function.qmd.jinja2` — Formats functions, parameters, and return values
+- `errors.qmd.jinja2` — Documents error classes with sorting
+- `sidebar.qmd.jinja2` — Generates navigation structure
+- `macros/docstring.jinja2` — Parses and structures Google-style docstrings
+- `macros/signatures.jinja2` — Formats function signatures and parameters
+- `macros/types.jinja2` — Handles complex type annotations
+- `macros/decorators.jinja2` — Documents function and class decorators
+- `macros/navigation.jinja2` — Generates page linking
+
+### Python script
+
+Located in `scripts/generate_quarto_docs.py`, handles the Quarto Markdown generation:
+
+- Extracts API data using Griffe.
+- Processes data with Jinja2 templates.
+- Lints and writes output to `docs/`
+
+#### Features
+
+- **Private/public filtering** — Controls which members are included
+- **Root module handling** — Special processing for the `validmind` module
+- **Alias resolution** — Maps imported symbols to original definitions
+- **Docstring normalization** — Cleans up formatting inconsistencies
+- **Inherited members** — Documents inherited methods, especially for error classes
+- **Errors module handling** — Sorts and structures error class documentation
+- **Class discovery** — Finds and documents classes across modules
+- **Test suite handling** — Documents test suites and their aliases
+- **VM models handling** — Ensures proper documentation of core model classes
+- **Exclusions** — Omits internal utilities and logging helpers
+- **Sidebar generation** — Builds hierarchical navigation from module structure
diff --git a/docs/_build/index.html b/docs/_build/index.html
deleted file mode 100644
index 59dc58fb5..000000000
--- a/docs/_build/index.html
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-
-
-
-
-
diff --git a/docs/_build/search.js b/docs/_build/search.js
deleted file mode 100644
index 206ef9bc4..000000000
--- a/docs/_build/search.js
+++ /dev/null
@@ -1,46 +0,0 @@
-window.pdocSearch = (function(){
-/** elasticlunr - http://weixsong.github.io * Copyright (C) 2017 Oliver Nightingale * Copyright (C) 2017 Wei Song * MIT Licensed */!function(){function e(e){if(null===e||"object"!=typeof e)return e;var t=e.constructor();for(var n in e)e.hasOwnProperty(n)&&(t[n]=e[n]);return t}var t=function(e){var n=new t.Index;return n.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),e&&e.call(n,n),n};t.version="0.9.5",lunr=t,t.utils={},t.utils.warn=function(e){return function(t){e.console&&console.warn&&console.warn(t)}}(this),t.utils.toString=function(e){return void 0===e||null===e?"":e.toString()},t.EventEmitter=function(){this.events={}},t.EventEmitter.prototype.addListener=function(){var e=Array.prototype.slice.call(arguments),t=e.pop(),n=e;if("function"!=typeof t)throw new TypeError("last argument must be a function");n.forEach(function(e){this.hasHandler(e)||(this.events[e]=[]),this.events[e].push(t)},this)},t.EventEmitter.prototype.removeListener=function(e,t){if(this.hasHandler(e)){var n=this.events[e].indexOf(t);-1!==n&&(this.events[e].splice(n,1),0==this.events[e].length&&delete this.events[e])}},t.EventEmitter.prototype.emit=function(e){if(this.hasHandler(e)){var t=Array.prototype.slice.call(arguments,1);this.events[e].forEach(function(e){e.apply(void 0,t)},this)}},t.EventEmitter.prototype.hasHandler=function(e){return e in this.events},t.tokenizer=function(e){if(!arguments.length||null===e||void 0===e)return[];if(Array.isArray(e)){var n=e.filter(function(e){return null===e||void 0===e?!1:!0});n=n.map(function(e){return t.utils.toString(e).toLowerCase()});var i=[];return n.forEach(function(e){var n=e.split(t.tokenizer.seperator);i=i.concat(n)},this),i}return e.toString().trim().toLowerCase().split(t.tokenizer.seperator)},t.tokenizer.defaultSeperator=/[\s\-]+/,t.tokenizer.seperator=t.tokenizer.defaultSeperator,t.tokenizer.setSeperator=function(e){null!==e&&void 0!==e&&"object"==typeof e&&(t.tokenizer.seperator=e)},t.tokenizer.resetSeperator=function(){t.tokenizer.seperator=t.tokenizer.defaultSeperator},t.tokenizer.getSeperator=function(){return t.tokenizer.seperator},t.Pipeline=function(){this._queue=[]},t.Pipeline.registeredFunctions={},t.Pipeline.registerFunction=function(e,n){n in t.Pipeline.registeredFunctions&&t.utils.warn("Overwriting existing registered function: "+n),e.label=n,t.Pipeline.registeredFunctions[n]=e},t.Pipeline.getRegisteredFunction=function(e){return e in t.Pipeline.registeredFunctions!=!0?null:t.Pipeline.registeredFunctions[e]},t.Pipeline.warnIfFunctionNotRegistered=function(e){var n=e.label&&e.label in this.registeredFunctions;n||t.utils.warn("Function is not registered with pipeline. This may cause problems when serialising the index.\n",e)},t.Pipeline.load=function(e){var n=new t.Pipeline;return e.forEach(function(e){var i=t.Pipeline.getRegisteredFunction(e);if(!i)throw new Error("Cannot load un-registered function: "+e);n.add(i)}),n},t.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach(function(e){t.Pipeline.warnIfFunctionNotRegistered(e),this._queue.push(e)},this)},t.Pipeline.prototype.after=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._queue.indexOf(e);if(-1===i)throw new Error("Cannot find existingFn");this._queue.splice(i+1,0,n)},t.Pipeline.prototype.before=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._queue.indexOf(e);if(-1===i)throw new Error("Cannot find existingFn");this._queue.splice(i,0,n)},t.Pipeline.prototype.remove=function(e){var t=this._queue.indexOf(e);-1!==t&&this._queue.splice(t,1)},t.Pipeline.prototype.run=function(e){for(var t=[],n=e.length,i=this._queue.length,o=0;n>o;o++){for(var r=e[o],s=0;i>s&&(r=this._queue[s](r,o,e),void 0!==r&&null!==r);s++);void 0!==r&&null!==r&&t.push(r)}return t},t.Pipeline.prototype.reset=function(){this._queue=[]},t.Pipeline.prototype.get=function(){return this._queue},t.Pipeline.prototype.toJSON=function(){return this._queue.map(function(e){return t.Pipeline.warnIfFunctionNotRegistered(e),e.label})},t.Index=function(){this._fields=[],this._ref="id",this.pipeline=new t.Pipeline,this.documentStore=new t.DocumentStore,this.index={},this.eventEmitter=new t.EventEmitter,this._idfCache={},this.on("add","remove","update",function(){this._idfCache={}}.bind(this))},t.Index.prototype.on=function(){var e=Array.prototype.slice.call(arguments);return this.eventEmitter.addListener.apply(this.eventEmitter,e)},t.Index.prototype.off=function(e,t){return this.eventEmitter.removeListener(e,t)},t.Index.load=function(e){e.version!==t.version&&t.utils.warn("version mismatch: current "+t.version+" importing "+e.version);var n=new this;n._fields=e.fields,n._ref=e.ref,n.documentStore=t.DocumentStore.load(e.documentStore),n.pipeline=t.Pipeline.load(e.pipeline),n.index={};for(var i in e.index)n.index[i]=t.InvertedIndex.load(e.index[i]);return n},t.Index.prototype.addField=function(e){return this._fields.push(e),this.index[e]=new t.InvertedIndex,this},t.Index.prototype.setRef=function(e){return this._ref=e,this},t.Index.prototype.saveDocument=function(e){return this.documentStore=new t.DocumentStore(e),this},t.Index.prototype.addDoc=function(e,n){if(e){var n=void 0===n?!0:n,i=e[this._ref];this.documentStore.addDoc(i,e),this._fields.forEach(function(n){var o=this.pipeline.run(t.tokenizer(e[n]));this.documentStore.addFieldLength(i,n,o.length);var r={};o.forEach(function(e){e in r?r[e]+=1:r[e]=1},this);for(var s in r){var u=r[s];u=Math.sqrt(u),this.index[n].addToken(s,{ref:i,tf:u})}},this),n&&this.eventEmitter.emit("add",e,this)}},t.Index.prototype.removeDocByRef=function(e){if(e&&this.documentStore.isDocStored()!==!1&&this.documentStore.hasDoc(e)){var t=this.documentStore.getDoc(e);this.removeDoc(t,!1)}},t.Index.prototype.removeDoc=function(e,n){if(e){var n=void 0===n?!0:n,i=e[this._ref];this.documentStore.hasDoc(i)&&(this.documentStore.removeDoc(i),this._fields.forEach(function(n){var o=this.pipeline.run(t.tokenizer(e[n]));o.forEach(function(e){this.index[n].removeToken(e,i)},this)},this),n&&this.eventEmitter.emit("remove",e,this))}},t.Index.prototype.updateDoc=function(e,t){var t=void 0===t?!0:t;this.removeDocByRef(e[this._ref],!1),this.addDoc(e,!1),t&&this.eventEmitter.emit("update",e,this)},t.Index.prototype.idf=function(e,t){var n="@"+t+"/"+e;if(Object.prototype.hasOwnProperty.call(this._idfCache,n))return this._idfCache[n];var i=this.index[t].getDocFreq(e),o=1+Math.log(this.documentStore.length/(i+1));return this._idfCache[n]=o,o},t.Index.prototype.getFields=function(){return this._fields.slice()},t.Index.prototype.search=function(e,n){if(!e)return[];e="string"==typeof e?{any:e}:JSON.parse(JSON.stringify(e));var i=null;null!=n&&(i=JSON.stringify(n));for(var o=new t.Configuration(i,this.getFields()).get(),r={},s=Object.keys(e),u=0;u0&&t.push(e);for(var i in n)"docs"!==i&&"df"!==i&&this.expandToken(e+i,t,n[i]);return t},t.InvertedIndex.prototype.toJSON=function(){return{root:this.root}},t.Configuration=function(e,n){var e=e||"";if(void 0==n||null==n)throw new Error("fields should not be null");this.config={};var i;try{i=JSON.parse(e),this.buildUserConfig(i,n)}catch(o){t.utils.warn("user configuration parse failed, will use default configuration"),this.buildDefaultConfig(n)}},t.Configuration.prototype.buildDefaultConfig=function(e){this.reset(),e.forEach(function(e){this.config[e]={boost:1,bool:"OR",expand:!1}},this)},t.Configuration.prototype.buildUserConfig=function(e,n){var i="OR",o=!1;if(this.reset(),"bool"in e&&(i=e.bool||i),"expand"in e&&(o=e.expand||o),"fields"in e)for(var r in e.fields)if(n.indexOf(r)>-1){var s=e.fields[r],u=o;void 0!=s.expand&&(u=s.expand),this.config[r]={boost:s.boost||0===s.boost?s.boost:1,bool:s.bool||i,expand:u}}else t.utils.warn("field name in user configuration not found in index instance fields");else this.addAllFields2UserConfig(i,o,n)},t.Configuration.prototype.addAllFields2UserConfig=function(e,t,n){n.forEach(function(n){this.config[n]={boost:1,bool:e,expand:t}},this)},t.Configuration.prototype.get=function(){return this.config},t.Configuration.prototype.reset=function(){this.config={}},lunr.SortedSet=function(){this.length=0,this.elements=[]},lunr.SortedSet.load=function(e){var t=new this;return t.elements=e,t.length=e.length,t},lunr.SortedSet.prototype.add=function(){var e,t;for(e=0;e1;){if(r===e)return o;e>r&&(t=o),r>e&&(n=o),i=n-t,o=t+Math.floor(i/2),r=this.elements[o]}return r===e?o:-1},lunr.SortedSet.prototype.locationFor=function(e){for(var t=0,n=this.elements.length,i=n-t,o=t+Math.floor(i/2),r=this.elements[o];i>1;)e>r&&(t=o),r>e&&(n=o),i=n-t,o=t+Math.floor(i/2),r=this.elements[o];return r>e?o:e>r?o+1:void 0},lunr.SortedSet.prototype.intersect=function(e){for(var t=new lunr.SortedSet,n=0,i=0,o=this.length,r=e.length,s=this.elements,u=e.elements;;){if(n>o-1||i>r-1)break;s[n]!==u[i]?s[n]u[i]&&i++:(t.add(s[n]),n++,i++)}return t},lunr.SortedSet.prototype.clone=function(){var e=new lunr.SortedSet;return e.elements=this.toArray(),e.length=e.elements.length,e},lunr.SortedSet.prototype.union=function(e){var t,n,i;this.length>=e.length?(t=this,n=e):(t=e,n=this),i=t.clone();for(var o=0,r=n.toArray();oThe ValidMind Library is a suite of developer tools and methods designed to automate the documentation and validation of your models.\n\n
Designed to be model agnostic, the ValidMind Library provides all the standard functionality without requiring you to rewrite any functions as long as your model is built in Python.
\n\n
With a rich array of documentation tools and test suites, from documenting descriptions of your datasets to testing your models for weak spots and overfit areas, the ValidMind Library helps you automate model documentation by feeding the ValidMind Platform with documentation artifacts and test results.
\n\n
To install the ValidMind Library:
\n\n
\n
pipinstallvalidmind\n
\n
\n\n
To initialize the ValidMind Library, paste the code snippet with the model identifier credentials directly into your development source code, replacing this example with your own:
After you have pasted the code snippet into your development source code and executed the code, the Python Library API will register with ValidMind. You can now use the ValidMind Library to document and test your models, and to upload to the ValidMind Platform.
Initializes the API client instances and calls the /ping endpoint to ensure\nthe provided credentials are valid and we can connect to the ValidMind API.
\n\n
If the API key and secret are not provided, the client will attempt to\nretrieve them from the environment variables VM_API_KEY and VM_API_SECRET.
\n\n
Arguments:
\n\n
\n
project (str, optional): The project CUID. Alias for model. Defaults to None. [DEPRECATED]
\n
model (str, optional): The model CUID. Defaults to None.
\n
api_key (str, optional): The API key. Defaults to None.
\n
api_secret (str, optional): The API secret. Defaults to None.
\n
api_host (str, optional): The API host. Defaults to None.
\n
monitoring (bool): The ongoing monitoring flag. Defaults to False.
\n
generate_descriptions (bool): Whether to use GenAI to generate test result descriptions. Defaults to True.
\n
\n\n
Raises:
\n\n
\n
ValueError: If the API key and secret are not provided
Initializes a VM Dataset, which can then be passed to other functions\nthat can perform additional analysis and tests on the data. This function\nalso ensures we are reading a valid dataset type.
\n\n
The following dataset types are supported:
\n\n
\n
Pandas DataFrame
\n
Polars DataFrame
\n
Numpy ndarray
\n
Torch TensorDataset
\n
\n\n
Arguments:
\n\n
\n
dataset : dataset from various python libraries
\n
model (VMModel): ValidMind model object
\n
targets (vm.vm.DatasetTargets): A list of target variables
\n
target_column (str): The name of the target column in the dataset
\n
feature_columns (list): A list of names of feature columns in the dataset
\n
extra_columns (dictionary): A dictionary containing the names of the
\n
prediction_column and group_by_columns in the dataset
\n
class_labels (dict): A list of class labels for classification problems
\n
type (str): The type of dataset (one of DATASET_TYPES)
\n
input_id (str): The input ID for the dataset (e.g. \"my_dataset\"). By default,\nthis will be set to dataset but if you are passing this dataset as a\ntest input using some other key than dataset, then you should set\nthis to the same key.
Initializes a VM Model, which can then be passed to other functions\nthat can perform additional analysis and tests on the data. This function\nalso ensures we are creating a model supported libraries.
\n\n
Arguments:
\n\n
\n
model: A trained model or VMModel instance
\n
input_id (str): The input ID for the model (e.g. \"my_model\"). By default,\nthis will be set to model but if you are passing this model as a\ntest input using some other key than model, then you should set\nthis to the same key.
\n
attributes (dict): A dictionary of model attributes
\n
predict_fn (callable): A function that takes an input and returns a prediction
\n
**kwargs: Additional arguments to pass to the model
R models must be saved to disk and the filetype depends on the model type...\nCurrently we support the following model types:
\n\n
\n
LogisticRegression glm model in R: saved as an RDS file with saveRDS
\n
LinearRegression lm model in R: saved as an RDS file with saveRDS
\n
XGBClassifier: saved as a .json or .bin file with xgb.save
\n
XGBRegressor: saved as a .json or .bin file with xgb.save
\n
\n\n
LogisticRegression and LinearRegression models are converted to sklearn models by extracting\nthe coefficients and intercept from the R model. XGB models are loaded using the xgboost\nsince xgb models saved in .json or .bin format can be loaded directly with either Python or R
\n\n
Arguments:
\n\n
\n
model_path (str): The path to the R model saved as an RDS or XGB file
\n
model_type (str): The type of the model (one of R_MODEL_TYPES)
Collect and run all the tests associated with a template
\n\n
This function will analyze the current project's documentation template and collect\nall the tests associated with it into a test suite. It will then run the test\nsuite, log the results to the ValidMind API, and display them to the user.
\n\n
Arguments:
\n\n
\n
section (str or list, optional): The section(s) to preview. Defaults to None.
\n
send (bool, optional): Whether to send the results to the ValidMind API. Defaults to True.
\n
fail_fast (bool, optional): Whether to stop running tests after the first failure. Defaults to False.
\n
inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite
\n
config: A dictionary of test parameters to override the defaults
\n
**kwargs: backwards compatibility for passing in test inputs using keyword arguments
\n
\n\n
Returns:
\n\n
\n
TestSuite or dict: The completed TestSuite instance or a dictionary of TestSuites if section is a list.
\n
\n\n
Raises:
\n\n
\n
ValueError: If the project has not been initialized
Unit metrics are key-value pairs where the key is the metric name and the value is\na scalar (int or float). These key-value pairs are associated with the currently\nselected model (inventory model in the ValidMind Platform) and keys can be logged\nto over time to create a history of the metric. On the ValidMind Platform, these metrics\nwill be used to create plots/visualizations for documentation and dashboards etc.
\n\n
Arguments:
\n\n
\n
key (str): The metric key
\n
value (float): The metric value
\n
inputs (list, optional): A list of input IDs that were used to compute the metric.
\n
params (dict, optional): Dictionary of parameters used to compute the metric.
\n
recorded_at (str, optional): The timestamp of the metric. Server will use\ncurrent time if not provided.
\n
thresholds (dict, optional): Dictionary of thresholds for the metric.
Gets a TestSuite object for the current project or a specific test suite
\n\n
This function provides an interface to retrieve the TestSuite instance for the\ncurrent project or a specific TestSuite instance identified by test_suite_id.\nThe project Test Suite will contain sections for every section in the project's\ndocumentation template and these Test Suite Sections will contain all the tests\nassociated with that template section.
\n\n
Arguments:
\n\n
\n
test_suite_id (str, optional): The test suite name. If not passed, then the\nproject's test suite will be returned. Defaults to None.
\n
section (str, optional): The section of the documentation template from which\nto retrieve the test suite. This only applies if test_suite_id is None.\nDefaults to None.
\n
args: Additional arguments to pass to the TestSuite
\n
kwargs: Additional keyword arguments to pass to the TestSuite
This function provides a high level interface for running a test suite. A test suite is\na collection of tests. This function will automatically find the correct test suite\nclass based on the test_suite_id, initialize each of the tests, and run them.
\n\n
Arguments:
\n\n
\n
test_suite_id (str): The test suite name (e.g. 'classifier_full_suite')
\n
config (dict, optional): A dictionary of parameters to pass to the tests in the\ntest suite. Defaults to None.
\n
send (bool, optional): Whether to post the test results to the API. send=False\nis useful for testing. Defaults to True.
\n
fail_fast (bool, optional): Whether to stop running tests after the first failure. Defaults to False.
\n
inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite e.g. model, dataset\nmodels etc. These inputs will be accessible by any test in the test suite. See the test\ndocumentation or vm.describe_test() for more details on the inputs required for each.
\n
**kwargs: backwards compatibility for passing in test inputs using keyword arguments
\n
\n\n
Raises:
\n\n
\n
ValueError: If the test suite name is not found or if there is an error initializing the test suite
Decorator for creating and registering custom tests
\n\n
This decorator registers the function it wraps as a test function within ValidMind\nunder the provided ID. Once decorated, the function can be run using the\nvalidmind.tests.run_test function.
\n\n
The function can take two different types of arguments:
\n\n
\n
Inputs: ValidMind model or dataset (or list of models/datasets). These arguments\nmust use the following names: model, models, dataset, datasets.
\n
Parameters: Any additional keyword arguments of any type (must have a default\nvalue) that can have any name.
\n
\n\n
The function should return one of the following types:
\n\n
\n
Table: Either a list of dictionaries or a pandas DataFrame
\n
Plot: Either a matplotlib figure or a plotly figure
\n
Scalar: A single number (int or float)
\n
Boolean: A single boolean value indicating whether the test passed or failed
\n
\n\n
The function may also include a docstring. This docstring will be used and logged\nas the metric's description.
\n\n
Arguments:
\n\n
\n
func: The function to decorate
\n
test_id: The identifier for the metric. If not provided, the function name is used.
Returns input configuration for the default documentation\ntemplate assigned to this demo model
\n\n
The default documentation template uses the following inputs:
\n\n
\n
raw_dataset
\n
train_dataset
\n
test_dataset
\n
model
\n
\n\n
We assign the following inputs depending on the input config expected\nby each test:
\n\n
\n
When a test expects a \"dataset\" we use the raw_dataset
\n
When a tets expects \"datasets\" we use the train_dataset and test_dataset
\n
When a test expects a \"model\" we use the model
\n
When a test expects \"model\" and \"dataset\" we use the model and test_dataset
\n
The only exception is ClassifierPerformance since that runs twice: once\nwith the train_dataset (in sample) and once with the test_dataset (out of sample)
Split a time series DataFrame into train, validation, and test sets.
\n\n
Arguments:
\n\n
\n
df (pandas.DataFrame): The time series DataFrame to be split.
\n
split_option (str): The split option to choose from: 'train_test_val' (default) or 'train_test'.
\n
train_size (float): The proportion of the dataset to include in the training set. Default is 0.6.
\n
test_size (float): The proportion of the dataset to include in the test set. Default is 0.2.
\n
\n\n
Returns:
\n\n
\n
train_df (pandas.DataFrame): The training set.\n validation_df (pandas.DataFrame): The validation set (only returned if split_option is 'train_test_val').\n test_df (pandas.DataFrame): The test set.
Split a time series DataFrame into train, validation, and test sets.
\n\n
Arguments:
\n\n
\n
df (pandas.DataFrame): The time series DataFrame to be split.
\n
split_option (str): The split option to choose from: 'train_test_val' (default) or 'train_test'.
\n
train_size (float): The proportion of the dataset to include in the training set. Default is 0.6.
\n
test_size (float): The proportion of the dataset to include in the test set. Default is 0.2.
\n
\n\n
Returns:
\n\n
\n
train_df (pandas.DataFrame): The training set.\n validation_df (pandas.DataFrame): The validation set (only returned if split_option is 'train_test_val').\n test_df (pandas.DataFrame): The test set.
This test suite provides a preliminary understanding of the target variable(s)\nused in the time series dataset. It visualizations that present the raw time\nseries data and a histogram of the target variable(s).
\n\n
The raw time series data provides a visual inspection of the target variable's\nbehavior over time. This helps to identify any patterns or trends in the data,\nas well as any potential outliers or anomalies. The histogram of the target\nvariable displays the distribution of values, providing insight into the range\nand frequency of values observed in the data.
This test suite provides a preliminary understanding of the features\nand relationship in multivariate dataset. It presents various\nmultivariate visualizations that can help identify patterns, trends,\nand relationships between pairs of variables. The visualizations are\ndesigned to explore the relationships between multiple features\nsimultaneously. They allow you to quickly identify any patterns or\ntrends in the data, as well as any potential outliers or anomalies.\nThe individual feature distribution can also be explored to provide\ninsight into the range and frequency of values observed in the data.\nThis multivariate analysis test suite aims to provide an overview of\nthe data structure and guide further exploration and modeling.
Test IDs are in the format namespace.path_to_module.TestClassOrFuncName[:tag].\nThe tag is optional and is used to distinguish between multiple results from the\nsame test.
\n\n
Arguments:
\n\n
\n
test_id (str): The test ID in the format namespace.path_to_module.TestName[:tag]
\n
test_func (callable, optional): The test function to load. If not provided, the\ntest will be loaded from the test provider. Defaults to None.
This function can be used to see test details including the test name, description,\nrequired inputs and default params. It can also be used to get a dictionary of the\nabove information for programmatic use.
\n\n
Arguments:
\n\n
\n
test_id (str, optional): The test ID. Defaults to None.
\n
raw (bool, optional): If True, returns a dictionary with the test details.\nDefaults to False.
This function is the main entry point for running tests. It can run simple unit metrics,\nValidMind and custom tests, composite tests made up of multiple unit metrics and comparison\ntests made up of multiple tests.
\n\n
Arguments:
\n\n
\n
test_id (TestID, optional): Test ID to run. Not required if name and unit_metrics provided.
\n
params (dict, optional): Parameters to customize test behavior. See test details for available parameters.
\n
param_grid (Union[Dict[str, List[Any]], List[Dict[str, Any]]], optional): For comparison tests, either:\n
\n
Dict mapping parameter names to lists of values (creates Cartesian product)
\n
List of parameter dictionaries to test
\n
\n
inputs (Dict[str, Any], optional): Test inputs (models/datasets initialized with vm.init_model/dataset)
\n
input_grid (Union[Dict[str, List[Any]], List[Dict[str, Any]]], optional): For comparison tests, either:\n
\n
Dict mapping input names to lists of values (creates Cartesian product)
\n
List of input dictionaries to test
\n
\n
name (str, optional): Test name (required for composite metrics)
\n
unit_metrics (list, optional): Unit metric IDs to run as composite metric
\n
show (bool, optional): Whether to display results. Defaults to True.
\n
generate_description (bool, optional): Whether to generate a description. Defaults to True.
\n
title (str, optional): Custom title for the test result
\n
post_process_fn (Callable[[TestResult], None], optional): Function to post-process the test result
\n
\n\n
Returns:
\n\n
\n
TestResult: A TestResult object containing the test results
Test providers in ValidMind are responsible for loading tests from different sources,\nsuch as local files, databases, or remote services. The LocalTestProvider specifically\nloads tests from the local file system.
\n\n
To use the LocalTestProvider, you need to provide the root_folder, which is the\nroot directory for local tests. The test_id is a combination of the namespace (set\nwhen registering the test provider) and the path to the test class module, where\nslashes are replaced by dots and the .py extension is left out.
\n\n
Example usage:
\n\n
# Create an instance of LocalTestProvider with the root folder\ntest_provider = LocalTestProvider(\"/path/to/tests/folder\")\n\n# Register the test provider with a namespace\nregister_test_provider(\"my_namespace\", test_provider)\n\n# List all tests in the namespace (returns a list of test IDs)\ntest_provider.list_tests()\n# this is used by the validmind.tests.list_tests() function to aggregate all tests\n# from all test providers\n\n# Load a test using the test_id (namespace + path to test class module)\ntest = test_provider.load_test(\"my_namespace.my_test_class\")\n# full path to the test class module is /path/to/tests/folder/my_test_class.py\n
\n\n
Attributes:
\n\n
\n
root_folder (str): The root directory for local tests.
Decorator for creating and registering custom tests
\n\n
This decorator registers the function it wraps as a test function within ValidMind\nunder the provided ID. Once decorated, the function can be run using the\nvalidmind.tests.run_test function.
\n\n
The function can take two different types of arguments:
\n\n
\n
Inputs: ValidMind model or dataset (or list of models/datasets). These arguments\nmust use the following names: model, models, dataset, datasets.
\n
Parameters: Any additional keyword arguments of any type (must have a default\nvalue) that can have any name.
\n
\n\n
The function should return one of the following types:
\n\n
\n
Table: Either a list of dictionaries or a pandas DataFrame
\n
Plot: Either a matplotlib figure or a plotly figure
\n
Scalar: A single number (int or float)
\n
Boolean: A single boolean value indicating whether the test passed or failed
\n
\n\n
The function may also include a docstring. This docstring will be used and logged\nas the metric's description.
\n\n
Arguments:
\n\n
\n
func: The function to decorate
\n
test_id: The identifier for the metric. If not provided, the function name is used.
Analyzes time series data using Autocorrelation Function (ACF) and Partial Autocorrelation Function (PACF) plots to\nreveal trends and correlations.
\n\n
Purpose
\n\n
The ACF (Autocorrelation Function) and PACF (Partial Autocorrelation Function) plot test is employed to analyze\ntime series data in machine learning models. It illuminates the correlation of the data over time by plotting the\ncorrelation of the series with its own lags (ACF), and the correlations after removing effects already accounted\nfor by earlier lags (PACF). This information can identify trends, such as seasonality, degrees of autocorrelation,\nand inform the selection of order parameters for AutoRegressive Integrated Moving Average (ARIMA) models.
\n\n
Test Mechanism
\n\n
The ACFandPACFPlot test accepts a dataset with a time-based index. It first confirms the index is of a datetime\ntype, then handles any NaN values. The test subsequently generates ACF and PACF plots for each column in the\ndataset, producing a subplot for each. If the dataset doesn't include key columns, an error is returned.
\n\n
Signs of High Risk
\n\n
\n
Sudden drops in the correlation at a specific lag might signal a model at high risk.
\n
Consistent high correlation across multiple lags could also indicate non-stationarity in the data, which may\nsuggest that a model estimated on this data won't generalize well to future, unknown data.
\n
\n\n
Strengths
\n\n
\n
ACF and PACF plots offer clear graphical representations of the correlations in time series data.
\n
These plots are effective at revealing important data characteristics such as seasonality, trends, and\ncorrelation patterns.
\n
The insights from these plots aid in better model configuration, particularly in the selection of ARIMA model\nparameters.
\n
\n\n
Limitations
\n\n
\n
ACF and PACF plots are exclusively for time series data and hence, can't be applied to all ML models.
\n
These plots require large, consistent datasets as gaps could lead to misleading results.
\n
The plots can only represent linear correlations and fail to capture any non-linear relationships within the data.
\n
The plots might be difficult for non-experts to interpret and should not replace more advanced analyses.
Assesses the stationarity of a time series dataset using the Augmented Dickey-Fuller (ADF) test.
\n\n
Purpose
\n\n
The Augmented Dickey-Fuller (ADF) test metric is used to determine the order of integration, i.e., the stationarity\nof a given time series dataset. The stationary property of data is pivotal in many machine learning models as it\nimpacts the reliability and effectiveness of predictions and forecasts.
\n\n
Test Mechanism
\n\n
The ADF test is executed using the adfuller function from the statsmodels library on each feature of the\ndataset. Multiple outputs are generated for each run, including the ADF test statistic and p-value, count of lags\nused, the number of observations considered in the test, critical values at various confidence levels, and the\ninformation criterion. These results are stored for each feature for subsequent analysis.
\n\n
Signs of High Risk
\n\n
\n
An inflated ADF statistic and high p-value (generally above 0.05) indicate a high risk to the model's performance\ndue to the presence of a unit root indicating non-stationarity.
\n
Non-stationarity might result in untrustworthy or insufficient forecasts.
\n
\n\n
Strengths
\n\n
\n
The ADF test is robust to sophisticated correlations within the data, making it suitable for settings where data\ndisplays complex stochastic behavior.
\n
It provides explicit outputs like test statistics, critical values, and information criterion, enhancing\nunderstanding and transparency in the model validation process.
\n
\n\n
Limitations
\n\n
\n
The ADF test might demonstrate low statistical power, making it challenging to differentiate between a unit root\nand near-unit-root processes, potentially causing false negatives.
\n
It assumes the data follows an autoregressive process, which might not always be the case.
\n
The test struggles with time series data that have structural breaks.
Automatically identifies the optimal Autoregressive (AR) order for a time series using BIC and AIC criteria.
\n\n
Purpose
\n\n
The AutoAR test is intended to automatically identify the Autoregressive (AR) order of a time series by utilizing\nthe Bayesian Information Criterion (BIC) and Akaike Information Criterion (AIC). AR order is crucial in forecasting\ntasks as it dictates the quantity of prior terms in the sequence to use for predicting the current term. The\nobjective is to select the most fitting AR model that encapsulates the trend and seasonality in the time series\ndata.
\n\n
Test Mechanism
\n\n
The test mechanism operates by iterating through a possible range of AR orders up to a defined maximum. An AR model\nis fitted for each order, and the corresponding BIC and AIC are computed. BIC and AIC statistical measures are\ndesigned to penalize models for complexity, preferring simpler models that fit the data proficiently. To verify the\nstationarity of the time series, the Augmented Dickey-Fuller test is executed. The AR order, BIC, and AIC findings\nare compiled into a dataframe for effortless comparison. Then, the AR order with the smallest BIC is established as\nthe desirable order for each variable.
\n\n
Signs of High Risk
\n\n
\n
An augmented Dickey Fuller test p-value > 0.05, indicating the time series isn't stationary, may lead to\ninaccurate results.
\n
Problems with the model fitting procedure, such as computational or convergence issues.
\n
Continuous selection of the maximum specified AR order may suggest an insufficient set limit.
\n
\n\n
Strengths
\n\n
\n
The test independently pinpoints the optimal AR order, thereby reducing potential human bias.
\n
It strikes a balance between model simplicity and goodness-of-fit to avoid overfitting.
\n
Has the capability to account for stationarity in a time series, an essential aspect for dependable AR modeling.
\n
The results are aggregated into a comprehensive table, enabling an easy interpretation.
\n
\n\n
Limitations
\n\n
\n
The tests need a stationary time series input.
\n
They presume a linear relationship between the series and its lags.
\n
The search for the best model is constrained by the maximum AR order supplied in the parameters. Therefore, a low\nmax_ar_order could result in subpar outcomes.
\n
AIC and BIC may not always agree on the selection of the best model. This potentially requires the user to juggle\ninterpretational choices.
Automatically selects the optimal Moving Average (MA) order for each variable in a time series dataset based on\nminimal BIC and AIC values.
\n\n
Purpose
\n\n
The AutoMA metric serves an essential role of automated decision-making for selecting the optimal Moving Average\n(MA) order for every variable in a given time series dataset. The selection is dependent on the minimalization of\nBIC (Bayesian Information Criterion) and AIC (Akaike Information Criterion); these are established statistical\ntools used for model selection. Furthermore, prior to the commencement of the model fitting process, the algorithm\nconducts a stationarity test (Augmented Dickey-Fuller test) on each series.
\n\n
Test Mechanism
\n\n
Starting off, the AutoMA algorithm checks whether the max_ma_order parameter has been provided. It consequently\nloops through all variables in the dataset, carrying out the Dickey-Fuller test for stationarity. For each\nstationary variable, it fits an ARIMA model for orders running from 0 to max_ma_order. The result is a list\nshowcasing the BIC and AIC values of the ARIMA models based on different orders. The MA order, which yields the\nsmallest BIC, is chosen as the 'best MA order' for every single variable. The final results include a table\nsummarizing the auto MA analysis and another table listing the best MA order for each variable.
\n\n
Signs of High Risk
\n\n
\n
When a series is non-stationary (p-value>0.05 in the Dickey-Fuller test), the produced result could be inaccurate.
\n
Any error that arises in the process of fitting the ARIMA models, especially with a higher MA order, can\npotentially indicate risks and might need further investigation.
\n
\n\n
Strengths
\n\n
\n
The metric facilitates automation in the process of selecting the MA order for time series forecasting. This\nsignificantly saves time and reduces efforts conventionally necessary for manual hyperparameter tuning.
\n
The use of both BIC and AIC enhances the likelihood of selecting the most suitable model.
\n
The metric ascertains the stationarity of the series prior to model fitting, thus ensuring that the underlying\nassumptions of the MA model are fulfilled.
\n
\n\n
Limitations
\n\n
\n
If the time series fails to be stationary, the metric may yield inaccurate results. Consequently, it necessitates\npre-processing steps to stabilize the series before fitting the ARIMA model.
\n
The metric adopts a rudimentary model selection process based on BIC and doesn't consider other potential model\nselection strategies. Depending on the specific dataset, other strategies could be more appropriate.
\n
The 'max_ma_order' parameter must be manually input which doesn't always guarantee optimal performance,\nespecially when configured too low.
\n
The computation time increases with the rise in max_ma_order, hence, the metric may become computationally\ncostly for larger values.
Automates Augmented Dickey-Fuller test to assess stationarity across multiple time series in a DataFrame.
\n\n
Purpose
\n\n
The AutoStationarity metric is intended to automatically detect and evaluate the stationary nature of each time\nseries in a DataFrame. It incorporates the Augmented Dickey-Fuller (ADF) test, a statistical approach used to\nassess stationarity. Stationarity is a fundamental property suggesting that statistic features like mean and\nvariance remain unchanged over time. This is necessary for many time-series models.
\n\n
Test Mechanism
\n\n
The mechanism for the AutoStationarity test involves applying the Augmented Dicky-Fuller test to each time series\nwithin the given dataframe to assess if they are stationary. Every series in the dataframe is looped, using the ADF\ntest up to a defined maximum order (configurable and by default set to 5). The p-value resulting from the ADF test\nis compared against a predetermined threshold (also configurable and by default set to 0.05). The time series is\ndeemed stationary at its current differencing order if the p-value is less than the threshold.
\n\n
Signs of High Risk
\n\n
\n
A significant number of series not achieving stationarity even at the maximum order of differencing can indicate\nhigh risk or potential failure in the model.
\n
This could suggest the series may not be appropriately modeled by a stationary process, hence other modeling\napproaches might be required.
\n
\n\n
Strengths
\n\n
\n
The key strength in this metric lies in the automation of the ADF test, enabling mass stationarity analysis\nacross various time series and boosting the efficiency and credibility of the analysis.
\n
The utilization of the ADF test, a widely accepted method for testing stationarity, lends authenticity to the\nresults derived.
\n
The introduction of the max order and threshold parameters give users the autonomy to determine their preferred\nlevels of stringency in the tests.
\n
\n\n
Limitations
\n\n
\n
The Augmented Dickey-Fuller test and the stationarity test are not without their limitations. These tests are\npremised on the assumption that the series can be modeled by an autoregressive process, which may not always hold\ntrue.
\n
The stationarity check is highly sensitive to the choice of threshold for the significance level; an extremely\nhigh or low threshold could lead to incorrect results regarding the stationarity properties.
\n
There's also a risk of over-differencing if the maximum order is set too high, which could induce unnecessary\ncycles.
Generates bivariate scatterplots to visually inspect relationships between pairs of numerical predictor variables\nin machine learning classification tasks.
\n\n
Purpose
\n\n
This function is intended for visual inspection and monitoring of relationships between pairs of numerical\nvariables in a machine learning model targeting classification tasks. It helps in understanding how predictor\nvariables (features) interact with each other, which can inform feature selection, model-building strategies, and\nidentify potential biases or irregularities in the data.
\n\n
Test Mechanism
\n\n
The function creates scatter plots for each pair of numerical features in the dataset. It first filters out\nnon-numerical and binary features, ensuring the plots focus on meaningful numerical relationships. The resulting\nscatterplots are color-coded uniformly to avoid visual distraction, and the function returns a tuple of Plotly\nfigure objects, each representing a scatter plot for a pair of features.
\n\n
Signs of High Risk
\n\n
\n
Visual patterns suggesting non-linear relationships, multicollinearity, clustering, or outlier points in the\nscatter plots.
\n
Such issues could affect the assumptions and performance of certain models, especially those assuming linearity,\nlike logistic regression.
\n
\n\n
Strengths
\n\n
\n
Scatterplots provide an intuitive and visual tool to explore relationships between two variables.
\n
They are useful for identifying outliers, variable associations, and trends, including non-linear patterns.
\n
Supports visualization of binary or multi-class classification datasets, focusing on numerical features.
\n
\n\n
Limitations
\n\n
\n
Scatterplots are limited to bivariate analysis, showing relationships between only two variables at a time.
\n
Not ideal for very large datasets where overlapping points can reduce the clarity of the visualization.
\n
Scatterplots are exploratory tools and do not provide quantitative measures of model quality or performance.
\n
Interpretation is subjective and relies on the domain knowledge and judgment of the viewer.
Detects autocorrelation in time-series data through the Box-Pierce test to validate model performance.
\n\n
Purpose
\n\n
The Box-Pierce test is utilized to detect the presence of autocorrelation in a time-series dataset.\nAutocorrelation, or serial correlation, refers to the degree of similarity between observations based on the\ntemporal spacing between them. This test is essential for affirming the quality of a time-series model by ensuring\nthat the error terms in the model are random and do not adhere to a specific pattern.
\n\n
Test Mechanism
\n\n
The implementation of the Box-Pierce test involves calculating a test statistic along with a corresponding p-value\nderived from the dataset features. These quantities are used to test the null hypothesis that posits the data to be\nindependently distributed. This is achieved by iterating over every feature column in the time-series data and\napplying the acorr_ljungbox function of the statsmodels library. The function yields the Box-Pierce test\nstatistic as well as the respective p-value, all of which are cached as test results.
\n\n
Signs of High Risk
\n\n
\n
A low p-value, typically under 0.05 as per statistical convention, throws the null hypothesis of independence\ninto question. This implies that the dataset potentially houses autocorrelations, thus indicating a high-risk\nscenario concerning model performance.
\n
Large Box-Pierce test statistic values may indicate the presence of autocorrelation.
\n
\n\n
Strengths
\n\n
\n
Detects patterns in data that are supposed to be random, thereby ensuring no underlying autocorrelation.
\n
Can be computed efficiently given its low computational complexity.
\n
Can be widely applied to most regression problems, making it very versatile.
\n
\n\n
Limitations
\n\n
\n
Assumes homoscedasticity (constant variance) and normality of residuals, which may not always be the case in\nreal-world datasets.
\n
May exhibit reduced power for detecting complex autocorrelation schemes such as higher-order or negative\ncorrelations.
\n
It only provides a general indication of the existence of autocorrelation, without providing specific insights\ninto the nature or patterns of the detected autocorrelation.
\n
In the presence of trends or seasonal patterns, the Box-Pierce test may yield misleading results.
\n
Applicability is limited to time-series data, which limits its overall utility.
Assesses the statistical association between categorical features and a target variable using the Chi-Squared test.
\n\n
Purpose
\n\n
The ChiSquaredFeaturesTable function is designed to evaluate the relationship between categorical features and a\ntarget variable in a dataset. It performs a Chi-Squared test of independence for each categorical feature to\ndetermine whether a statistically significant association exists with the target variable. This is particularly\nuseful in Model Risk Management for understanding the relevance of features and identifying potential biases in a\nclassification model.
\n\n
Test Mechanism
\n\n
The function creates a contingency table for each categorical feature and the target variable, then applies the\nChi-Squared test to compute the Chi-squared statistic and the p-value. The results for each feature include the\nvariable name, Chi-squared statistic, p-value, p-value threshold, and a pass/fail status based on whether the\np-value is below the specified threshold. The output is a DataFrame summarizing these results, sorted by p-value to\nhighlight the most statistically significant associations.
\n\n
Signs of High Risk
\n\n
\n
High p-values (greater than the set threshold) indicate a lack of significant association between a feature and\nthe target variable, resulting in a 'Fail' status.
\n
Features with a 'Fail' status might not be relevant for the model, which could negatively impact model\nperformance.
\n
\n\n
Strengths
\n\n
\n
Provides a clear, statistical assessment of the relationship between categorical features and the target variable.
\n
Produces an easily interpretable summary with a 'Pass/Fail' outcome for each feature, helping in feature\nselection.
\n
The p-value threshold is adjustable, allowing for flexibility in statistical rigor.
\n
\n\n
Limitations
\n\n
\n
Assumes the dataset is tabular and consists of categorical variables, which may not be suitable for all datasets.
\n
The test is designed for classification tasks and is not applicable to regression problems.
\n
As with all hypothesis tests, the Chi-Squared test can only detect associations, not causal relationships.
\n
The choice of p-value threshold can affect the interpretation of feature relevance, and different thresholds may\nlead to different conclusions.
Evaluates and quantifies class distribution imbalance in a dataset used by a machine learning model.
\n\n
Purpose
\n\n
The Class Imbalance test is designed to evaluate the distribution of target classes in a dataset that's utilized by\na machine learning model. Specifically, it aims to ensure that the classes aren't overly skewed, which could lead\nto bias in the model's predictions. It's crucial to have a balanced training dataset to avoid creating a model\nthat's biased with high accuracy for the majority class and low accuracy for the minority class.
\n\n
Test Mechanism
\n\n
This Class Imbalance test operates by calculating the frequency (expressed as a percentage) of each class in the\ntarget column of the dataset. It then checks whether each class appears in at least a set minimum percentage of the\ntotal records. This minimum percentage is a modifiable parameter, but the default value is set to 10%.
\n\n
Signs of High Risk
\n\n
\n
Any class that represents less than the pre-set minimum percentage threshold is marked as high risk, implying a\npotential class imbalance.
\n
The function provides a pass/fail outcome for each class based on this criterion.
\n
Fundamentally, if any class fails this test, it's highly likely that the dataset possesses imbalanced class\ndistribution.
\n
\n\n
Strengths
\n\n
\n
The test can spot under-represented classes that could affect the efficiency of a machine learning model.
\n
The calculation is straightforward and swift.
\n
The test is highly informative because it not only spots imbalance, but it also quantifies the degree of\nimbalance.
\n
The adjustable threshold enables flexibility and adaptation to differing use-cases or domain-specific needs.
\n
The test creates a visually insightful plot showing the classes and their corresponding proportions, enhancing\ninterpretability and comprehension of the data.
\n
\n\n
Limitations
\n\n
\n
The test might struggle to perform well or provide vital insights for datasets with a high number of classes. In\nsuch cases, the imbalance could be inevitable due to the inherent class distribution.
\n
Sensitivity to the threshold value might result in faulty detection of imbalance if the threshold is set\nexcessively high.
\n
Regardless of the percentage threshold, it doesn't account for varying costs or impacts of misclassifying\ndifferent classes, which might fluctuate based on specific applications or domains.
\n
While it can identify imbalances in class distribution, it doesn't provide direct methods to address or correct\nthese imbalances.
\n
The test is only applicable for classification operations and unsuitable for regression or clustering tasks.
Returns a collection of histograms for a numerical or categorical column.\nWe store different combinations of bin sizes to allow analyzing the data better
\n\n
Will be used in favor of _get_histogram in the future
Provides comprehensive analysis and statistical summaries of each column in a machine learning model's dataset.
\n\n
Purpose
\n\n
The test depicted in the script is meant to run a comprehensive analysis on a Machine Learning model's datasets.\nThe test or metric is implemented to obtain a complete summary of the columns in the dataset, including vital\nstatistics of each column such as count, distinct values, missing values, histograms for numerical, categorical,\nboolean, and text columns. This summary gives a comprehensive overview of the dataset to better understand the\ncharacteristics of the data that the model is trained on or evaluates.
\n\n
Test Mechanism
\n\n
The DatasetDescription class accomplishes the purpose as follows: firstly, the test method \"run\" infers the data\ntype of each column in the dataset and stores the details (id, column type). For each column, the\n\"describe_column\" method is invoked to collect statistical information about the column, including count,\nmissing value count and its proportion to the total, unique value count, and its proportion to the total. Depending\non the data type of a column, histograms are generated that reflect the distribution of data within the column.\nNumerical columns use the \"get_numerical_histograms\" method to calculate histogram distribution, whereas for\ncategorical, boolean and text columns, a histogram is computed with frequencies of each unique value in the\ndatasets. For unsupported types, an error is raised. Lastly, a summary table is built to aggregate all the\nstatistical insights and histograms of the columns in a dataset.
\n\n
Signs of High Risk
\n\n
\n
High ratio of missing values to total values in one or more columns which may impact the quality of the\npredictions.
\n
Unsupported data types in dataset columns.
\n
Large number of unique values in the dataset's columns which might make it harder for the model to establish\npatterns.
\n
Extreme skewness or irregular distribution of data as reflected in the histograms.
\n
\n\n
Strengths
\n\n
\n
Provides a detailed analysis of the dataset with versatile summaries like count, unique values, histograms, etc.
\n
Flexibility in handling different types of data: numerical, categorical, boolean, and text.
\n
Useful in detecting problems in the dataset like missing values, unsupported data types, irregular data\ndistribution, etc.
\n
The summary gives a comprehensive understanding of dataset features allowing developers to make informed\ndecisions.
\n
\n\n
Limitations
\n\n
\n
The computation can be expensive from a resource standpoint, particularly for large datasets with numerous columns.
\n
The histograms use an arbitrary number of bins which may not be the optimal number of bins for specific data\ndistribution.
\n
Unsupported data types for columns will raise an error which may limit evaluating the dataset.
\n
Columns with all null or missing values are not included in histogram computation.
\n
This test only validates the quality of the dataset but doesn't address the model's performance directly.
Evaluates and visualizes the distribution proportions among training, testing, and validation datasets of an ML\nmodel.
\n\n
Purpose
\n\n
The DatasetSplit test is designed to evaluate and visualize the distribution of data among training, testing, and\nvalidation datasets, if available, within a given machine learning model. The main purpose is to assess whether the\nmodel's datasets are split appropriately, as an imbalanced split might affect the model's ability to learn from the\ndata and generalize to unseen data.
\n\n
Test Mechanism
\n\n
The DatasetSplit test first calculates the total size of all available datasets in the model. Then, for each\nindividual dataset, the methodology involves determining the size of the dataset and its proportion relative to the\ntotal size. The results are then conveniently summarized in a table that shows dataset names, sizes, and\nproportions. Absolute size and proportion of the total dataset size are displayed for each individual dataset.
\n\n
Signs of High Risk
\n\n
\n
A very small training dataset, which may result in the model not learning enough from the data.
\n
A very large training dataset and a small test dataset, which may lead to model overfitting and poor\ngeneralization to unseen data.
\n
A small or non-existent validation dataset, which might complicate the model's performance assessment.
\n
\n\n
Strengths
\n\n
\n
The DatasetSplit test provides a clear, understandable visualization of dataset split proportions, which can\nhighlight any potential imbalance in dataset splits quickly.
\n
It covers a wide range of task types including classification, regression, and text-related tasks.
\n
The metric is not tied to any specific data type and is applicable to tabular data, time series data, or text\ndata.
\n
\n\n
Limitations
\n\n
\n
The DatasetSplit test does not provide any insight into the quality or diversity of the data within each split,\njust the size and proportion.
\n
The test does not give any recommendations or adjustments for imbalanced datasets.
\n
Potential lack of compatibility with more complex modes of data splitting (for example, stratified or time-based\nsplits) could limit the applicability of this test.
Performs a detailed descriptive statistical analysis of both numerical and categorical data within a model's\ndataset.
\n\n
Purpose
\n\n
The purpose of the Descriptive Statistics metric is to provide a comprehensive summary of both numerical and\ncategorical data within a dataset. This involves statistics such as count, mean, standard deviation, minimum and\nmaximum values for numerical data. For categorical data, it calculates the count, number of unique values, most\ncommon value and its frequency, and the proportion of the most frequent value relative to the total. The goal is to\nvisualize the overall distribution of the variables in the dataset, aiding in understanding the model's behavior\nand predicting its performance.
\n\n
Test Mechanism
\n\n
The testing mechanism utilizes two in-built functions of pandas dataframes: describe() for numerical fields and\nvalue_counts() for categorical fields. The describe() function pulls out several summary statistics, while\nvalue_counts() accounts for unique values. The resulting data is formatted into two distinct tables, one for\nnumerical and another for categorical variable summaries. These tables provide a clear summary of the main\ncharacteristics of the variables, which can be instrumental in assessing the model's performance.
\n\n
Signs of High Risk
\n\n
\n
Skewed data or significant outliers can represent high risk. For numerical data, this may be reflected via a\nsignificant difference between the mean and median (50% percentile).
\n
For categorical data, a lack of diversity (low count of unique values), or overdominance of a single category\n(high frequency of the top value) can indicate high risk.
\n
\n\n
Strengths
\n\n
\n
Provides a comprehensive summary of the dataset, shedding light on the distribution and characteristics of the\nvariables under consideration.
\n
It is a versatile and robust method, applicable to both numerical and categorical data.
\n
Helps highlight crucial anomalies such as outliers, extreme skewness, or lack of diversity, which are vital in\nunderstanding model behavior during testing and validation.
\n
\n\n
Limitations
\n\n
\n
While this metric offers a high-level overview of the data, it may fail to detect subtle correlations or complex\npatterns.
\n
Does not offer any insights on the relationship between variables.
\n
Alone, descriptive statistics cannot be used to infer properties about future unseen data.
\n
Should be used in conjunction with other statistical tests to provide a comprehensive understanding of the\nmodel's data.
Assesses stationarity in time series data using the Dickey-Fuller GLS test to determine the order of integration.
\n\n
Purpose
\n\n
The Dickey-Fuller GLS (DFGLS) test is utilized to determine the order of integration in time series data. For\nmachine learning models dealing with time series and forecasting, this metric evaluates the existence of a unit\nroot, thereby checking whether a time series is non-stationary. This analysis is a crucial initial step when\ndealing with time series data.
\n\n
Test Mechanism
\n\n
This code implements the Dickey-Fuller GLS unit root test on each attribute of the dataset. This process involves\niterating through every column of the dataset and applying the DFGLS test to assess the presence of a unit root.\nThe resulting information, including the test statistic ('stat'), the p-value ('pvalue'), the quantity of lagged\ndifferences utilized in the regression ('usedlag'), and the number of observations ('nobs'), is subsequently stored.
\n\n
Signs of High Risk
\n\n
\n
A high p-value for the DFGLS test represents a high risk. Specifically, a p-value above a typical threshold of\n0.05 suggests that the time series data is quite likely to be non-stationary, thus presenting a high risk for\ngenerating unreliable forecasts.
\n
\n\n
Strengths
\n\n
\n
The Dickey-Fuller GLS test is a potent tool for checking the stationarity of time series data.
\n
It helps to verify the assumptions of the models before the actual construction of the machine learning models\nproceeds.
\n
The results produced by this metric offer a clear insight into whether the data is appropriate for specific\nmachine learning models, especially those demanding the stationarity of time series data.
\n
\n\n
Limitations
\n\n
\n
Despite its benefits, the DFGLS test does present some drawbacks. It can potentially lead to inaccurate\nconclusions if the time series data incorporates a structural break.
\n
If the time series tends to follow a trend while still being stationary, the test might misinterpret it,\nnecessitating further detrending.
\n
The test also presents challenges when dealing with shorter time series data or volatile data, not producing\nreliable results in these cases.
Tests dataset for duplicate entries, ensuring model reliability via data quality verification.
\n\n
Purpose
\n\n
The 'Duplicates' test is designed to check for duplicate rows within the dataset provided to the model. It serves\nas a measure of data quality, ensuring that the model isn't merely memorizing duplicate entries or being swayed by\nredundant information. This is an important step in the pre-processing of data for both classification and\nregression tasks.
\n\n
Test Mechanism
\n\n
This test operates by checking each row for duplicates in the dataset. If a text column is specified in the\ndataset, the test is conducted on this column; if not, the test is run on all feature columns. The number and\npercentage of duplicates are calculated and returned in a DataFrame. Additionally, a test is passed if the total\ncount of duplicates falls below a specified minimum threshold.
\n\n
Signs of High Risk
\n\n
\n
A high number of duplicate rows in the dataset, which can lead to overfitting where the model performs well on\nthe training data but poorly on unseen data.
\n
A high percentage of duplicate rows in the dataset, indicating potential problems with data collection or\nprocessing.
\n
\n\n
Strengths
\n\n
\n
Assists in improving the reliability of the model's training process by ensuring the training data is not\ncontaminated with duplicate entries, which can distort statistical analyses.
\n
Provides both absolute numbers and percentage values of duplicate rows, giving a thorough overview of data\nquality.
\n
Highly customizable as it allows for setting a user-defined minimum threshold to determine if the test has been\npassed.
\n
\n\n
Limitations
\n\n
\n
Does not distinguish between benign duplicates (i.e., coincidental identical entries in different rows) and\nproblematic duplicates originating from data collection or processing errors.
\n
The test becomes more computationally intensive as the size of the dataset increases, which might not be suitable\nfor very large datasets.
\n
Can only check for exact duplicates and may miss semantically similar information packaged differently.
Assesses the degree of co-movement between pairs of time series data using the Engle-Granger cointegration test.
\n\n
Purpose
\n\n
The intent of this Engle-Granger cointegration test is to explore and quantify the degree of co-movement between\npairs of time series variables in a dataset. This is particularly useful in enhancing the accuracy of predictive\nregressions whenever the underlying variables are co-integrated, i.e., they move together over time.
\n\n
Test Mechanism
\n\n
The test first drops any non-applicable values from the input dataset and then iterates over each pair of variables\nto apply the Engle-Granger cointegration test. The test generates a 'p' value, which is then compared against a\npre-specified threshold (0.05 by default). The pair is labeled as 'Cointegrated' if the 'p' value is less than or\nequal to the threshold or 'Not cointegrated' otherwise. A summary table is returned by the metric showing\ncointegration results for each variable pair.
\n\n
Signs of High Risk
\n\n
\n
A significant number of hypothesized cointegrated variables do not pass the test.
\n
A considerable number of 'p' values are close to the threshold, indicating minor data fluctuations can switch the\ndecision between 'Cointegrated' and 'Not cointegrated'.
\n
\n\n
Strengths
\n\n
\n
Provides an effective way to analyze relationships between time series, particularly in contexts where it's\nessential to check if variables move together in a statistically significant manner.
\n
Useful in various domains, especially finance or economics, where predictive models often hinge on understanding\nhow different variables move together over time.
\n
\n\n
Limitations
\n\n
\n
Assumes that the time series are integrated of the same order, which isn't always true in multivariate time\nseries datasets.
\n
The presence of non-stationary characteristics in the series or structural breaks can result in falsely positive\nor negative cointegration results.
\n
May not perform well for small sample sizes due to lack of statistical power and should be supplemented with\nother predictive indicators for a more robust model evaluation.
Visualizes the correlation between input features and the model's target output in a color-coded horizontal bar\nplot.
\n\n
Purpose
\n\n
This test is designed to graphically illustrate the correlations between distinct input features and the target\noutput of a Machine Learning model. Understanding how each feature influences the model's predictions is crucial\u2014a\nhigher correlation indicates a stronger influence of the feature on the target variable. This correlation study is\nespecially advantageous during feature selection and for comprehending the model's operation.
\n\n
Test Mechanism
\n\n
This FeatureTargetCorrelationPlot test computes and presents the correlations between the features and the target\nvariable using a specific dataset. These correlations are calculated and are then graphically represented in a\nhorizontal bar plot, color-coded based on the strength of the correlation. A hovering template can also be utilized\nfor informative tooltips. It is possible to specify the features to be analyzed and adjust the graph's height\naccording to need.
\n\n
Signs of High Risk
\n\n
\n
There are no strong correlations (either positive or negative) between features and the target variable. This\ncould suggest high risk as the supplied features do not appear to significantly impact the prediction output.
\n
The presence of duplicated correlation values might hint at redundancy in the feature set.
\n
\n\n
Strengths
\n\n
\n
Provides visual assistance to interpreting correlations more effectively.
\n
Gives a clear and simple tour of how each feature affects the model's target variable.
\n
Beneficial for feature selection and grasping the model's prediction nature.
\n
Precise correlation values for each feature are offered by the hover template, contributing to a granular-level\ncomprehension.
\n
\n\n
Limitations
\n\n
\n
The test only accepts numerical data, meaning variables of other types need to be prepared beforehand.
\n
The plot assumes all correlations to be linear, thus non-linear relationships might not be captured effectively.
\n
Not apt for models that employ complex feature interactions, like Decision Trees or Neural Networks, as the test\nmay not accurately reflect their importance.
Assesses the number of unique values in categorical columns to detect high cardinality and potential overfitting.
\n\n
Purpose
\n\n
The \u201cHigh Cardinality\u201d test is used to evaluate the number of unique values present in the categorical columns of a\ndataset. In this context, high cardinality implies the presence of a large number of unique, non-repetitive values\nin the dataset.
\n\n
Test Mechanism
\n\n
The test first infers the dataset's type and then calculates an initial numeric threshold based on the test\nparameters. It only considers columns classified as \"Categorical\". For each of these columns, the number of\ndistinct values (n_distinct) and the percentage of distinct values (p_distinct) are calculated. The test will pass\nif n_distinct is less than the calculated numeric threshold. Lastly, the results, which include details such as\ncolumn name, number of distinct values, and pass/fail status, are compiled into a table.
\n\n
Signs of High Risk
\n\n
\n
A large number of distinct values (high cardinality) in one or more categorical columns implies a high risk.
\n
A column failing the test (n_distinct >= num_threshold) is another indicator of high risk.
\n
\n\n
Strengths
\n\n
\n
The High Cardinality test is effective in early detection of potential overfitting and unwanted noise.
\n
It aids in identifying potential outliers and inconsistencies, thereby improving data quality.
\n
The test can be applied to both classification and regression task types, demonstrating its versatility.
\n
\n\n
Limitations
\n\n
\n
The test is restricted to only \"Categorical\" data types and is thus not suitable for numerical or continuous\nfeatures, limiting its scope.
\n
The test does not consider the relevance or importance of unique values in categorical features, potentially\ncausing it to overlook critical data points.
\n
The threshold (both number and percent) used for the test is static and may not be optimal for diverse datasets\nand varied applications. Further mechanisms to adjust and refine this threshold could enhance its effectiveness.
Identifies highly correlated feature pairs in a dataset suggesting feature redundancy or multicollinearity.
\n\n
Purpose
\n\n
The High Pearson Correlation test measures the linear relationship between features in a dataset, with the main\ngoal of identifying high correlations that might indicate feature redundancy or multicollinearity. Identification\nof such issues allows developers and risk management teams to properly deal with potential impacts on the machine\nlearning model's performance and interpretability.
\n\n
Test Mechanism
\n\n
The test works by generating pairwise Pearson correlations for all features in the dataset, then sorting and\neliminating duplicate and self-correlations. It assigns a Pass or Fail based on whether the absolute value of the\ncorrelation coefficient surpasses a pre-set threshold (defaulted at 0.3). It lastly returns the top n strongest\ncorrelations regardless of passing or failing status (where n is 10 by default but can be configured by passing the\ntop_n_correlations parameter).
\n\n
Signs of High Risk
\n\n
\n
A high risk indication would be the presence of correlation coefficients exceeding the threshold.
\n
If the features share a strong linear relationship, this could lead to potential multicollinearity and model\noverfitting.
\n
Redundancy of variables can undermine the interpretability of the model due to uncertainty over the authenticity\nof individual variable's predictive power.
\n
\n\n
Strengths
\n\n
\n
Provides a quick and simple means of identifying relationships between feature pairs.
\n
Generates a transparent output that displays pairs of correlated variables, the Pearson correlation coefficient,\nand a Pass or Fail status for each.
\n
Aids in early identification of potential multicollinearity issues that may disrupt model training.
\n
\n\n
Limitations
\n\n
\n
Can only delineate linear relationships, failing to shed light on nonlinear relationships or dependencies.
\n
Sensitive to outliers where a few outliers could notably affect the correlation coefficient.
\n
Limited to identifying redundancy only within feature pairs; may fail to spot more complex relationships among\nthree or more variables.
Visualizes outlier distribution across percentiles in numerical data using the Interquartile Range (IQR) method.
\n\n
Purpose
\n\n
The InterQuartile Range Outliers Bar Plot (IQROutliersBarPlot) metric aims to visually analyze and evaluate the\nextent of outliers in numeric variables based on percentiles. Its primary purpose is to clarify the dataset's\ndistribution, flag possible abnormalities in it, and gauge potential risks associated with processing potentially\nskewed data, which can affect the machine learning model's predictive prowess.
\n\n
Test Mechanism
\n\n
The examination invokes a series of steps:
\n\n\n
For every numeric feature in the dataset, the 25th percentile (Q1) and 75th percentile (Q3) are calculated\nbefore deriving the Interquartile Range (IQR), the difference between Q1 and Q3.
\n
Subsequently, the metric calculates the lower and upper thresholds by subtracting Q1 from the threshold times\nIQR and adding Q3 to threshold times IQR, respectively. The default threshold is set at 1.5.
\n
Any value in the feature that falls below the lower threshold or exceeds the upper threshold is labeled as an\noutlier.
\n
The number of outliers are tallied for different percentiles, such as [0-25], [25-50], [50-75], and [75-100].
\n
These counts are employed to construct a bar plot for the feature, showcasing the distribution of outliers\nacross different percentiles.
\n\n\n
Signs of High Risk
\n\n
\n
A prevalence of outliers in the data, potentially skewing its distribution.
\n
Outliers dominating higher percentiles (75-100) which implies the presence of extreme values, capable of severely\ninfluencing the model's performance.
\n
Certain features harboring most of their values as outliers, which signifies that these features might not\ncontribute positively to the model's forecasting ability.
\n
\n\n
Strengths
\n\n
\n
Effectively identifies outliers in the data through visual means, facilitating easier comprehension and offering\ninsights into the outliers' possible impact on the model.
\n
Provides flexibility by accommodating all numeric features or a chosen subset.
\n
Task-agnostic in nature; it is viable for both classification and regression tasks.
\n
Can handle large datasets as its operation does not hinge on computationally heavy operations.
\n
\n\n
Limitations
\n\n
\n
Its application is limited to numerical variables and does not extend to categorical ones.
\n
Only reveals the presence and distribution of outliers and does not provide insights into how these outliers\nmight affect the model's predictive performance.
\n
The assumption that data is unimodal and symmetric may not always hold true. In cases with non-normal\ndistributions, the results can be misleading.
Determines and summarizes outliers in numerical features using the Interquartile Range method.
\n\n
Purpose
\n\n
The \"Interquartile Range Outliers Table\" (IQROutliersTable) metric is designed to identify and summarize outliers\nwithin numerical features of a dataset using the Interquartile Range (IQR) method. This exercise is crucial in the\npre-processing of data because outliers can substantially distort statistical analysis and impact the performance\nof machine learning models.
\n\n
Test Mechanism
\n\n
The IQR, which is the range separating the first quartile (25th percentile) from the third quartile (75th\npercentile), is calculated for each numerical feature within the dataset. An outlier is defined as a data point\nfalling below the \"Q1 - 1.5 * IQR\" or above \"Q3 + 1.5 * IQR\" range. The test computes the number of outliers and\ntheir summary statistics (minimum, 25th percentile, median, 75th percentile, and maximum values) for each numerical\nfeature. If no specific features are chosen, the test applies to all numerical features in the dataset. The default\noutlier threshold is set to 1.5 but can be customized by the user.
\n\n
Signs of High Risk
\n\n
\n
A large number of outliers in multiple features.
\n
Outliers significantly distanced from the mean value of variables.
\n
Extremely high or low outlier values indicative of data entry errors or other data quality issues.
\n
\n\n
Strengths
\n\n
\n
Provides a comprehensive summary of outliers for each numerical feature, helping pinpoint features with potential\nquality issues.
\n
The IQR method is robust to extremely high or low outlier values as it is based on quartile calculations.
\n
Can be customized to work on selected features and set thresholds for outliers.
\n
\n\n
Limitations
\n\n
\n
Might cause false positives if the variable deviates from a normal or near-normal distribution, especially for\nskewed distributions.
\n
Does not provide interpretation or recommendations for addressing outliers, relying on further analysis by users\nor data scientists.
\n
Only applicable to numerical features, not categorical data.
\n
Default thresholds may not be optimal for data with heavy pre-processing, manipulation, or inherently high\nkurtosis (heavy tails).
Detects outliers in a dataset using the Isolation Forest algorithm and visualizes results through scatter plots.
\n\n
Purpose
\n\n
The IsolationForestOutliers test is designed to identify anomalies or outliers in the model's dataset using the\nisolation forest algorithm. This algorithm assumes that anomalous data points can be isolated more quickly due to\ntheir distinctive properties. By creating isolation trees and identifying instances with shorter average path\nlengths, the test is able to pick out data points that differ from the majority.
\n\n
Test Mechanism
\n\n
The test uses the isolation forest algorithm, which builds an ensemble of isolation trees by randomly selecting\nfeatures and splitting the data based on random thresholds. It isolates anomalies rather than focusing on normal\ndata points. For each pair of variables, a scatter plot is generated which distinguishes the identified outliers\nfrom the inliers. The results of the test can be visualized using these scatter plots, illustrating the distinction\nbetween outliers and inliers.
\n\n
Signs of High Risk
\n\n
\n
The presence of high contamination, indicating a large number of anomalies
\n
Inability to detect clusters of anomalies that are close in the feature space
\n
Misclassifying normal instances as anomalies
\n
Failure to detect actual anomalies
\n
\n\n
Strengths
\n\n
\n
Ability to handle large, high-dimensional datasets
\n
Efficiency in isolating anomalies instead of normal instances
\n
Insensitivity to the underlying distribution of data
\n
Ability to recognize anomalies even when they are not separated from the main data cloud through identifying\ndistinctive properties
\n
Visually presents the test results for better understanding and interpretability
\n
\n\n
Limitations
\n\n
\n
Difficult to detect anomalies that are close to each other or prevalent in datasets
\n
Dependency on the contamination parameter which may need fine-tuning to be effective
\n
Potential failure in detecting collective anomalies if they behave similarly to normal data
\n
Potential lack of precision in identifying which features contribute most to the anomalous behavior
Assesses normality of dataset features in an ML model using the Jarque-Bera test.
\n\n
Purpose
\n\n
The purpose of the Jarque-Bera test as implemented in this metric is to determine if the features in the dataset of\na given Machine Learning model follow a normal distribution. This is crucial for understanding the distribution and\nbehavior of the model's features, as numerous statistical methods assume normal distribution of the data.
\n\n
Test Mechanism
\n\n
The test mechanism involves computing the Jarque-Bera statistic, p-value, skew, and kurtosis for each feature in\nthe dataset. It utilizes the 'jarque_bera' function from the 'statsmodels' library in Python, storing the results\nin a dictionary. The test evaluates the skewness and kurtosis to ascertain whether the dataset follows a normal\ndistribution. A significant p-value (typically less than 0.05) implies that the data does not possess normal\ndistribution.
\n\n
Signs of High Risk
\n\n
\n
A high Jarque-Bera statistic and a low p-value (usually less than 0.05) indicate high-risk conditions.
\n
Such results suggest the data significantly deviates from a normal distribution. If a machine learning model\nexpects feature data to be normally distributed, these findings imply that it may not function as intended.
\n
\n\n
Strengths
\n\n
\n
Provides insights into the shape of the data distribution, helping determine whether a given set of data follows\na normal distribution.
\n
Particularly useful for risk assessment for models that assume a normal distribution of data.
\n
By measuring skewness and kurtosis, it provides additional insights into the nature and magnitude of a\ndistribution's deviation.
\n
\n\n
Limitations
\n\n
\n
Only checks for normality in the data distribution. It cannot provide insights into other types of distributions.
\n
Datasets that aren't normally distributed but follow some other distribution might lead to inaccurate risk\nassessments.
\n
Highly sensitive to large sample sizes, often rejecting the null hypothesis (that data is normally distributed)\neven for minor deviations in larger datasets.
Assesses the stationarity of time-series data in a machine learning model using the KPSS unit root test.
\n\n
Purpose
\n\n
The KPSS (Kwiatkowski-Phillips-Schmidt-Shin) unit root test is utilized to ensure the stationarity of data within a\nmachine learning model. It specifically works on time-series data to establish the order of integration, which is\nessential for accurate forecasting. A fundamental requirement for any time series model is that the series should\nbe stationary.
\n\n
Test Mechanism
\n\n
This test calculates the KPSS score for each feature in the dataset. The KPSS score includes a statistic, a\np-value, a used lag, and critical values. The core principle behind the KPSS test is to evaluate the hypothesis\nthat an observable time series is stationary around a deterministic trend. If the computed statistic exceeds the\ncritical value, the null hypothesis (that the series is stationary) is rejected, indicating that the series is\nnon-stationary.
\n\n
Signs of High Risk
\n\n
\n
High KPSS score, particularly if the calculated statistic is higher than the critical value.
\n
Rejection of the null hypothesis, indicating that the series is recognized as non-stationary, can severely affect\nthe model's forecasting capability.
\n
\n\n
Strengths
\n\n
\n
Directly measures the stationarity of a series, fulfilling a key prerequisite for many time-series models.
\n
The underlying logic of the test is intuitive and simple, making it easy to understand and accessible for both\ndevelopers and risk management teams.
\n
\n\n
Limitations
\n\n
\n
Assumes the absence of a unit root in the series and doesn't differentiate between series that are stationary and\nthose border-lining stationarity.
\n
The test may have restricted power against certain alternatives.
\n
The reliability of the test is contingent on the number of lags selected, which introduces potential bias in the\nmeasurement.
Assesses autocorrelations in dataset features by performing a Ljung-Box test on each feature.
\n\n
Purpose
\n\n
The Ljung-Box test is a type of statistical test utilized to ascertain whether there are autocorrelations within a\ngiven dataset that differ significantly from zero. In the context of a machine learning model, this test is\nprimarily used to evaluate data utilized in regression tasks, especially those involving time series and\nforecasting.
\n\n
Test Mechanism
\n\n
The test operates by iterating over each feature within the dataset and applying the acorr_ljungbox\nfunction from the statsmodels.stats.diagnostic library. This function calculates the Ljung-Box statistic and\np-value for each feature. These results are then stored in a pandas DataFrame where the columns are the feature names,\nstatistic, and p-value respectively. Generally, a lower p-value indicates a higher likelihood of significant\nautocorrelations within the feature.
\n\n
Signs of High Risk
\n\n
\n
High Ljung-Box statistic values or low p-values.
\n
Presence of significant autocorrelations in the respective features.
\n
Potential for negative impact on model performance or bias if autocorrelations are not properly handled.
\n
\n\n
Strengths
\n\n
\n
Powerful tool for detecting autocorrelations within datasets, especially in time series data.
\n
Provides quantitative measures (statistic and p-value) for precise evaluation.
\n
Helps avoid issues related to autoregressive residuals and other challenges in regression models.
\n
\n\n
Limitations
\n\n
\n
Cannot detect all types of non-linearity or complex interrelationships among variables.
\n
Testing individual features may not fully encapsulate the dynamics of the data if features interact with each other.
\n
Designed more for traditional statistical models and may not be fully compatible with certain types of complex\nmachine learning models.
Assesses and visualizes correlation between target variable and lagged independent variables in a time-series\ndataset.
\n\n
Purpose
\n\n
The LaggedCorrelationHeatmap metric is utilized to appraise and illustrate the correlation between the target\nvariable and delayed copies (lags) of independent variables in a time-series dataset. It assists in revealing\nrelationships in time-series data where the influence of an independent variable on the dependent variable is not\nimmediate but occurs after a period (lags).
\n\n
Test Mechanism
\n\n
To execute this test, Python's Pandas library pairs with Plotly to perform computations and present the\nvisualization in the form of a heatmap. The test begins by extracting the target variable and corresponding\nindependent variables from the dataset. Then, generation of lags of independent variables takes place, followed by\nthe calculation of correlation between these lagged variables and the target variable. The outcome is a correlation\nmatrix that gets recorded and illustrated as a heatmap, where different color intensities represent the strength of\nthe correlation, making patterns easier to identify.
\n\n
Signs of High Risk
\n\n
\n
Insignificant correlations across the heatmap, indicating a lack of noteworthy relationships between variables.
\n
Correlations that break intuition or previous understanding, suggesting potential issues with the dataset or the\nmodel.
\n
\n\n
Strengths
\n\n
\n
This metric serves as an exceptional tool for exploring and visualizing time-dependent relationships between\nfeatures and the target variable in a time-series dataset.
\n
It aids in identifying delayed effects that might go unnoticed with other correlation measures.
\n
The heatmap offers an intuitive visual representation of time-dependent correlations and influences.
\n
\n\n
Limitations
\n\n
\n
The metric presumes linear relationships between variables, potentially ignoring non-linear relationships.
\n
The correlation considered is linear; therefore, intricate non-linear interactions might be overlooked.
\n
The metric is only applicable for time-series data, limiting its utility outside of this context.
\n
The number of lags chosen can significantly influence the results; too many lags can render the heatmap difficult\nto interpret, while too few might overlook delayed effects.
\n
This metric does not take into account any causal relationships, but merely demonstrates correlation.
Evaluates dataset quality by ensuring missing value ratio across all features does not exceed a set threshold.
\n\n
Purpose
\n\n
The Missing Values test is designed to evaluate the quality of a dataset by measuring the number of missing values\nacross all features. The objective is to ensure that the ratio of missing data to total data is less than a\npredefined threshold, defaulting to 1, in order to maintain the data quality necessary for reliable predictive\nstrength in a machine learning model.
\n\n
Test Mechanism
\n\n
The mechanism for this test involves iterating through each column of the dataset, counting missing values\n(represented as NaNs), and calculating the percentage they represent against the total number of rows. The test\nthen checks if these missing value counts are less than the predefined min_threshold. The results are shown in a\ntable summarizing each column, the number of missing values, the percentage of missing values in each column, and a\nPass/Fail status based on the threshold comparison.
\n\n
Signs of High Risk
\n\n
\n
When the number of missing values in any column exceeds the min_threshold value.
\n
Presence of missing values across many columns, leading to multiple instances of failing the threshold.
\n
\n\n
Strengths
\n\n
\n
Quick and granular identification of missing data across each feature in the dataset.
\n
Provides an effective and straightforward means of maintaining data quality, essential for constructing efficient\nmachine learning models.
\n
\n\n
Limitations
\n\n
\n
Does not suggest the root causes of the missing values or recommend ways to impute or handle them.
\n
May overlook features with significant missing data but still less than the min_threshold, potentially\nimpacting the model.
\n
Does not account for data encoded as values like \"-999\" or \"None,\" which might not technically classify as\nmissing but could bear similar implications.
Assesses the percentage and distribution of missing values in the dataset via a bar plot, with emphasis on\nidentifying high-risk columns based on a user-defined threshold.
\n\n
Purpose
\n\n
The 'MissingValuesBarPlot' metric provides a color-coded visual representation of the percentage of missing values\nfor each column in an ML model's dataset. The primary purpose of this metric is to easily identify and quantify\nmissing data, which are essential steps in data preprocessing. The presence of missing data can potentially skew\nthe model's predictions and decrease its accuracy. Additionally, this metric uses a pre-set threshold to categorize\nvarious columns into ones that contain missing data above the threshold (high risk) and below the threshold (less\nrisky).
\n\n
Test Mechanism
\n\n
The test mechanism involves scanning each column in the input dataset and calculating the percentage of missing\nvalues. It then compares each column's missing data percentage with the predefined threshold, categorizing columns\nwith missing data above the threshold as high-risk. The test generates a bar plot in which columns with missing\ndata are represented on the y-axis and their corresponding missing data percentages are displayed on the x-axis.\nThe color of each bar reflects the missing data percentage in relation to the threshold: grey for values below the\nthreshold and light coral for those exceeding it. The user-defined threshold is represented by a red dashed line on\nthe plot.
\n\n
Signs of High Risk
\n\n
\n
Columns with higher percentages of missing values beyond the threshold are high-risk. These are visually\nrepresented by light coral bars on the bar plot.
\n
\n\n
Strengths
\n\n
\n
Helps in quickly identifying and quantifying missing data across all columns of the dataset.
\n
Facilitates pattern recognition through visual representation.
\n
Enables customization of the level of risk tolerance via a user-defined threshold.
\n
Supports both classification and regression tasks, sharing its versatility.
\n
\n\n
Limitations
\n\n
\n
It only considers the quantity of missing values, not differentiating between different types of missingness\n(Missing completely at random - MCAR, Missing at random - MAR, Not Missing at random - NMAR).
\n
It doesn't offer insights into potential approaches for handling missing entries, such as various imputation\nstrategies.
\n
The metric does not consider possible impacts of the missing data on the model's accuracy or precision.
\n
Interpretation of the findings and the next steps might require an expert understanding of the field.
Calculates mutual information scores between features and target variable to evaluate feature relevance.
\n\n
Purpose
\n\n
The Mutual Information test quantifies the predictive power of each feature by measuring its statistical\ndependency with the target variable. This helps identify relevant features for model training and\ndetect potential redundant or irrelevant variables, supporting feature selection decisions and model\ninterpretability.
\n\n
Test Mechanism
\n\n
The test employs sklearn's mutual_info_classif/mutual_info_regression functions to compute mutual\ninformation between each feature and the target. It produces a normalized score (0 to 1) for each\nfeature, where higher scores indicate stronger relationships. Results are presented in both tabular\nformat and visualized through a bar plot with a configurable threshold line.
\n\n
Signs of High Risk
\n\n
\n
Many features showing very low mutual information scores
\n
Key business features exhibiting unexpectedly low scores
\n
All features showing similar, low information content
\n
Large discrepancy between business importance and MI scores
\n
Highly skewed distribution of MI scores
\n
Critical features below the minimum threshold
\n
Unexpected zero or near-zero scores for known important features
\n
Inconsistent scores across different data samples
\n
\n\n
Strengths
\n\n
\n
Captures non-linear relationships between features and target
\n
Scale-invariant measurement of feature relevance
\n
Works for both classification and regression tasks
\n
Provides interpretable scores (0 to 1 scale)
\n
Supports automated feature selection
\n
No assumptions about data distribution
\n
Handles numerical and categorical features
\n
Computationally efficient for most datasets
\n
\n\n
Limitations
\n\n
\n
Requires sufficient data for reliable estimates
\n
May be computationally intensive for very large datasets
\n
Cannot detect redundant features (pairwise relationships)
\n
Sensitive to feature discretization for continuous variables
\n
Does not account for feature interactions
\n
May underestimate importance of rare but crucial events
Evaluates linear dependency between numerical variables in a dataset via a Pearson Correlation coefficient heat map.
\n\n
Purpose
\n\n
This test is intended to evaluate the extent of linear dependency between all pairs of numerical variables in the\ngiven dataset. It provides the Pearson Correlation coefficient, which reveals any high correlations present. The\npurpose of doing this is to identify potential redundancy, as variables that are highly correlated can often be\nremoved to reduce the dimensionality of the dataset without significantly impacting the model's performance.
\n\n
Test Mechanism
\n\n
This metric test generates a correlation matrix for all numerical variables in the dataset using the Pearson\ncorrelation formula. A heat map is subsequently created to visualize this matrix effectively. The color of each\npoint on the heat map corresponds to the magnitude and direction (positive or negative) of the correlation, with a\nrange from -1 (perfect negative correlation) to 1 (perfect positive correlation). Any correlation coefficients\nhigher than 0.7 (in absolute terms) are indicated in white in the heat map, suggesting a high degree of correlation.
\n\n
Signs of High Risk
\n\n
\n
A large number of variables in the dataset showing a high degree of correlation (coefficients approaching \u00b11).\nThis indicates redundancy within the dataset, suggesting that some variables may not be contributing new\ninformation to the model.
\n
Potential risk of overfitting.
\n
\n\n
Strengths
\n\n
\n
Detects and quantifies the linearity of relationships between variables, aiding in identifying redundant\nvariables to simplify models and potentially improve performance.
\n
The heatmap visualization provides an easy-to-understand overview of correlations, beneficial for users not\ncomfortable with numerical matrices.
\n
\n\n
Limitations
\n\n
\n
Limited to detecting linear relationships, potentially missing non-linear relationships which impede\nopportunities for dimensionality reduction.
\n
Measures only the degree of linear relationship, not the strength of one variable's effect on another.
\n
The 0.7 correlation threshold is arbitrary and might exclude valid dependencies with lower coefficients.
Assesses the stationarity of time series data in each feature of the ML model using the Phillips-Perron test.
\n\n
Purpose
\n\n
The Phillips-Perron (PP) test is used to determine the stationarity of time series data for each feature in a\ndataset, which is crucial for forecasting tasks. It tests the null hypothesis that a time series is unit-root\nnon-stationary. This is vital for understanding the stochastic behavior of the data and ensuring the robustness and\nvalidity of predictions generated by regression analysis models.
\n\n
Test Mechanism
\n\n
The PP test is conducted for each feature in the dataset as follows:
\n\n
\n
A data frame is created from the dataset.
\n
For each column, the Phillips-Perron method calculates the test statistic, p-value, lags used, and number of\nobservations.
\n
The results are then stored for each feature, providing a metric that indicates the stationarity of the time\nseries data.
\n
\n\n
Signs of High Risk
\n\n
\n
A high p-value, indicating that the series has a unit root and is non-stationary.
\n
Test statistic values exceeding critical values, suggesting non-stationarity.
\n
High 'usedlag' value, pointing towards autocorrelation issues that may degrade model performance.
\n
\n\n
Strengths
\n\n
\n
Resilience against heteroskedasticity in the error term.
\n
Effective for long time series data.
\n
Helps in determining whether the time series is stationary, aiding in the selection of suitable forecasting\nmodels.
\n
\n\n
Limitations
\n\n
\n
Applicable only within a univariate time series framework.
\n
Relies on asymptotic theory, which may reduce the test\u2019s power for small sample sizes.
\n
Non-stationary time series must be converted to stationary series through differencing, potentially leading to\nloss of important data points.
Visualizes combinations of protected classes and their corresponding error metric differences.
\n\n
Purpose
\n\n
This test aims to provide insights into how different combinations of protected classes affect various error metrics,\nparticularly the false negative rate (FNR) and false positive rate (FPR). By visualizing these combinations,\nit helps identify potential biases or disparities in model performance across different intersectional groups.
\n\n
Test Mechanism
\n\n
The test performs the following steps:
\n\n\n
Combines the specified protected class columns to create a single multi-class category.
\n
Calculates error metrics (FNR, FPR, etc.) for each combination of protected classes.
\n
Generates visualizations showing the distribution of these metrics across all class combinations.
\n\n\n
Signs of High Risk
\n\n
\n
Large disparities in FNR or FPR across different protected class combinations.
\n
Consistent patterns of higher error rates for specific combinations of protected attributes.
\n
Unexpected or unexplainable variations in error metrics between similar group combinations.
\n
\n\n
Strengths
\n\n
\n
Provides a comprehensive view of intersectional fairness across multiple protected attributes.
\n
Allows for easy identification of potentially problematic combinations of protected classes.
\n
Visualizations make it easier to spot patterns or outliers in model performance across groups.
\n
\n\n
Limitations
\n\n
\n
May become complex and difficult to interpret with a large number of protected classes or combinations.
\n
Does not provide statistical significance of observed differences.
\n
Visualization alone may not capture all nuances of intersectional fairness.
Visualizes the distribution of protected classes in the dataset relative to the target variable\nand provides descriptive statistics.
\n\n
Purpose
\n\n
The ProtectedClassesDescription test aims to identify potential biases or significant differences in the\ndistribution of target outcomes across different protected classes. This visualization and statistical summary\nhelp in understanding the relationship between protected attributes and the target variable, which is crucial\nfor assessing fairness in machine learning models.
\n\n
Test Mechanism
\n\n
The function creates interactive stacked bar charts for each specified protected class using Plotly.\nAdditionally, it generates a single table of descriptive statistics for all protected classes, including:
\n\n
\n
Protected class and category
\n
Count and percentage of each category within the protected class
\n
Mean, median, and mode of the target variable for each category
\n
Standard deviation of the target variable for each category
\n
Minimum and maximum values of the target variable for each category
\n
\n\n
Signs of High Risk
\n\n
\n
Significant imbalances in the distribution of target outcomes across different categories of a protected class.
\n
Large disparities in mean, median, or mode of the target variable across categories.
\n
Underrepresentation or overrepresentation of certain groups within protected classes.
\n
High standard deviations in certain categories, indicating potential volatility or outliers.
\n
\n\n
Strengths
\n\n
\n
Provides both visual and statistical representation of potential biases in the dataset.
\n
Allows for easy identification of imbalances in target variable distribution across protected classes.
\n
Interactive plots enable detailed exploration of the data.
\n
Consolidated statistical summary provides quantitative measures to complement visual analysis.
\n
Applicable to both classification and regression tasks.
\n
\n\n
Limitations
\n\n
\n
Does not provide advanced statistical measures of bias or fairness.
\n
May become cluttered if there are many categories within a protected class or many unique target values.
\n
Interpretation may require domain expertise to understand the implications of observed disparities.
\n
Does not account for intersectionality or complex interactions between multiple protected attributes.
Investigates disparities in model performance across different protected class segments.
\n\n
Purpose
\n\n
This test aims to identify and quantify potential biases in model outcomes by comparing various performance metrics\nacross different segments of protected classes. It helps in assessing whether the model produces discriminatory\noutcomes for certain groups, which is crucial for ensuring fairness in machine learning models.
\n\n
Test Mechanism
\n\n
The test performs the following steps:
\n\n\n
Calculates performance metrics (e.g., false negative rate, false positive rate, true positive rate) for each segment\nof the specified protected classes.
\n
Computes disparity ratios by comparing these metrics between different segments and a reference group.
\n
Generates visualizations showing the disparities and their relation to a user-defined disparity tolerance threshold.
\n
Produces a comprehensive table with various disparity metrics for detailed analysis.
\n\n\n
Signs of High Risk
\n\n
\n
Disparity ratios exceeding the specified disparity tolerance threshold.
\n
Consistent patterns of higher error rates or lower performance for specific protected class segments.
\n
Statistically significant differences in performance metrics across segments.
\n
\n\n
Strengths
\n\n
\n
Provides a comprehensive view of model fairness across multiple protected attributes and metrics.
\n
Allows for easy identification of problematic disparities through visual and tabular representations.
\n
Customizable disparity tolerance threshold to align with specific use-case requirements.
\n
Applicable to various performance metrics, offering a multi-faceted analysis of model fairness.
\n
\n\n
Limitations
\n\n
\n
Relies on a predefined reference group for each protected class, which may not always be the most appropriate choice.
\n
Does not account for intersectionality between different protected attributes.
\n
The interpretation of results may require domain expertise to understand the implications of observed disparities.
Obtains a classifier by applying group-specific thresholds to the provided estimator.
\n\n
Purpose
\n\n
This test aims to optimize the fairness of a machine learning model by applying different\nclassification thresholds for different protected groups. It helps in mitigating bias and\nachieving more equitable outcomes across different demographic groups.
\n\n
Test Mechanism
\n\n
The test uses Fairlearn's ThresholdOptimizer to:
\n\n\n
Fit an optimizer on the training data, considering protected classes.
\n
Apply optimized thresholds to make predictions on the test data.
\n
Calculate and report various fairness metrics.
\n
Visualize the optimized thresholds.
\n\n\n
Signs of High Risk
\n\n
\n
Large disparities in fairness metrics (e.g., Demographic Parity Ratio, Equalized Odds Ratio)\nacross different protected groups.
\n
Significant differences in False Positive Rates (FPR) or True Positive Rates (TPR) between groups.
\n
Thresholds that vary widely across different protected groups.
\n
\n\n
Strengths
\n\n
\n
Provides a post-processing method to improve model fairness without modifying the original model.
\n
Allows for balancing multiple fairness criteria simultaneously.
\n
Offers visual insights into the threshold optimization process.
\n
\n\n
Limitations
\n\n
\n
May lead to a decrease in overall model performance while improving fairness.
\n
Requires access to protected attribute information at prediction time.
\n
The effectiveness can vary depending on the chosen fairness constraint and objective.
Evaluates the stationarity of time series data by plotting its rolling mean and standard deviation over a specified\nwindow.
\n\n
Purpose
\n\n
The RollingStatsPlot metric is employed to gauge the stationarity of time series data in a given dataset. This\nmetric specifically evaluates the rolling mean and rolling standard deviation of the dataset over a pre-specified\nwindow size. The rolling mean provides an understanding of the average trend in the data, while the rolling\nstandard deviation gauges the volatility of the data within the window. It is critical in preparing time series\ndata for modeling as it reveals key insights into data behavior across time.
\n\n
Test Mechanism
\n\n
This mechanism is comprised of two steps. Initially, the rolling mean and standard deviation for each of the\ndataset's columns are calculated over a window size, which can be user-specified or by default set to 12 data\npoints. Then, the calculated rolling mean and standard deviation are visualized via separate plots, illustrating\nthe trends and volatility in the dataset. A straightforward check is conducted to ensure the existence of columns\nin the dataset, and to verify that the given dataset has been indexed by its date and time\u2014a necessary prerequisite\nfor time series analysis.
\n\n
Signs of High Risk
\n\n
\n
The presence of non-stationary patterns in either the rolling mean or the rolling standard deviation plots, which\ncould indicate trends or seasonality in the data that may affect the performance of time series models.
\n
Missing columns in the dataset, which would prevent the execution of this metric correctly.
\n
The detection of NaN values in the dataset, which may need to be addressed before the metric can proceed\nsuccessfully.
\n
\n\n
Strengths
\n\n
\n
Offers visualizations of trending behavior and volatility within the data, facilitating a broader understanding\nof the dataset's inherent characteristics.
\n
Checks of the dataset's integrity, such as the existence of all required columns and the availability of a\ndatetime index.
\n
Adjusts to accommodate various window sizes, thus allowing accurate analysis of data with differing temporal\ngranularities.
\n
Considers each column of the data individually, thereby accommodating multi-feature datasets.
\n
\n\n
Limitations
\n\n
\n
For all columns, a fixed-size window is utilized. This may not accurately capture patterns in datasets where\ndifferent features may require different optimal window sizes.
\n
Requires the dataset to be indexed by date and time, hence it may not be usable for datasets without a timestamp\nindex.
\n
Primarily serves for data visualization as it does not facilitate any quantitative measures for stationarity,\nsuch as through statistical tests. Therefore, the interpretation is subjective and depends heavily on modeler\ndiscretion.
Executes Runs Test on ML model to detect non-random patterns in output data sequence.
\n\n
Purpose
\n\n
The Runs Test is a statistical procedure used to determine whether the sequence of data extracted from the ML model\nbehaves randomly or not. Specifically, it analyzes runs, sequences of consecutive positives or negatives, in the\ndata to check if there are more or fewer runs than expected under the assumption of randomness. This can be an\nindication of some pattern, trend, or cycle in the model's output which may need attention.
\n\n
Test Mechanism
\n\n
The testing mechanism applies the Runs Test from the statsmodels module on each column of the training dataset. For\nevery feature in the dataset, a Runs Test is executed, whose output includes a Runs Statistic and P-value. A low\nP-value suggests that data arrangement in the feature is not likely to be random. The results are stored in a\ndictionary where the keys are the feature names, and the values are another dictionary storing the test statistic\nand the P-value for each feature.
\n\n
Signs of High Risk
\n\n
\n
High risk is indicated when the P-value is close to zero.
\n
If the P-value is less than a predefined significance level (like 0.05), it suggests that the runs (series of\npositive or negative values) in the model's output are not random and are longer or shorter than what is expected\nunder a random scenario.
\n
This would mean there's a high risk of non-random distribution of errors or model outcomes, suggesting potential\nissues with the model.
\n
\n\n
Strengths
\n\n
\n
Straightforward and fast for detecting non-random patterns in data sequence.
\n
Validates assumptions of randomness, which is valuable for checking error distributions in regression models,\ntrendless time series data, and ensuring a classifier doesn't favor one class over another.
\n
Can be applied to both classification and regression tasks, making it versatile.
\n
\n\n
Limitations
\n\n
\n
Assumes that the data is independently and identically distributed (i.i.d.), which might not be the case for many\nreal-world datasets.
\n
The conclusion drawn from the low P-value indicating non-randomness does not provide information about the type\nor the source of the detected pattern.
\n
Sensitive to extreme values (outliers), and overly large or small run sequences can influence the results.
\n
Does not provide model performance evaluation; it is used to detect patterns in the sequence of outputs only.
Assesses visual relationships, patterns, and outliers among features in a dataset through scatter plot matrices.
\n\n
Purpose
\n\n
The ScatterPlot test aims to visually analyze a given dataset by constructing a scatter plot matrix of its\nnumerical features. The primary goal is to uncover relationships, patterns, and outliers across different features\nto provide both quantitative and qualitative insights into multidimensional relationships within the dataset. This\nvisual assessment aids in understanding the efficacy of the chosen features for model training and their\nsuitability.
\n\n
Test Mechanism
\n\n
Using the Seaborn library, the ScatterPlot function creates the scatter plot matrix. The process involves\nretrieving all numerical columns from the dataset and generating a scatter matrix for these columns. The resulting\nscatter plot provides visual representations of feature relationships. The function also adjusts axis labels for\nreadability and returns the final plot as a Matplotlib Figure object for further analysis and visualization.
\n\n
Signs of High Risk
\n\n
\n
The emergence of non-linear or random patterns across different feature pairs, suggesting complex relationships\nunsuitable for linear assumptions.
\n
Lack of clear patterns or clusters, indicating weak or non-existent correlations among features, which could\nchallenge certain model types.
\n
Presence of outliers, as visual outliers can adversely influence the model's performance.
\n
\n\n
Strengths
\n\n
\n
Provides insight into the multidimensional relationships among multiple features.
\n
Assists in identifying trends, correlations, and outliers that could affect model performance.
\n
Validates assumptions made during model creation, such as linearity.
\n
Versatile for application in both regression and classification tasks.
\n
Using Seaborn facilitates an intuitive and detailed visual exploration of data.
\n
\n\n
Limitations
\n\n
\n
Scatter plot matrices may become cluttered and hard to decipher as the number of features increases.
\n
Primarily reveals pairwise relationships and may fail to illuminate complex interactions involving three or more\nfeatures.
\n
Being a visual tool, precision in quantitative analysis might be compromised.
\n
Outliers not clearly visible in plots can be missed, affecting model performance.
\n
Assumes that the dataset can fit into the computer's memory, which might not be valid for extremely large\ndatasets.
Analyzes default rates and population distribution across credit score bands.
\n\n
Purpose
\n\n
The Score Band Default Rates test evaluates the discriminatory power of credit scores by analyzing\ndefault rates across different score bands. This helps validate score effectiveness, supports\npolicy decisions, and provides insights into portfolio risk distribution.
\n\n
Test Mechanism
\n\n
The test segments the score distribution into bands and calculates key metrics for each band:
\n\n\n
Population count and percentage in each band
\n
Default rate within each band
\n
Cumulative statistics across bands\nThe results show how well the scores separate good and bad accounts.
Assesses patterns and seasonality in a time series dataset by decomposing its features into foundational components.
\n\n
Purpose
\n\n
The Seasonal Decompose test aims to decompose the features of a time series dataset into their fundamental\ncomponents: observed, trend, seasonal, and residuals. By utilizing the Seasonal Decomposition of Time Series by\nLoess (STL) method, the test identifies underlying patterns, predominantly seasonality, in the dataset's features.\nThis aids in developing a more comprehensive understanding of the dataset, which in turn facilitates more effective\nmodel validation.
\n\n
Test Mechanism
\n\n
The testing process leverages the seasonal_decompose function from the statsmodels.tsa.seasonal library to\nevaluate each feature in the dataset. It isolates each feature into four components\u2014observed, trend, seasonal, and\nresiduals\u2014and generates six subplot graphs per feature for visual interpretation. Prior to decomposition, the test\nscrutinizes and removes any non-finite values, ensuring the reliability of the analysis.
\n\n
Signs of High Risk
\n\n
\n
Non-Finiteness: Datasets with a high number of non-finite values may flag as high risk since these values are\nomitted before conducting the seasonal decomposition.
\n
Frequent Warnings: Chronic failure to infer the frequency for a scrutinized feature indicates high risk.
\n
High Seasonality: A significant seasonal component could potentially render forecasts unreliable due to\noverwhelming seasonal variation.
\n
\n\n
Strengths
\n\n
\n
Seasonality Detection: Accurately discerns hidden seasonality patterns in dataset features.
\n
Visualization: Facilitates interpretation and comprehension through graphical representations.
\n
Unrestricted Usage: Not confined to any specific regression model, promoting wide-ranging applicability.
\n
\n\n
Limitations
\n\n
\n
Dependence on Assumptions: Assumes that dataset features are periodically distributed. Features with no\ninferable frequency are excluded from the test.
\n
Handling Non-Finite Values: Disregards non-finite values during analysis, potentially resulting in an\nincomplete understanding of the dataset.
\n
Unreliability with Noisy Datasets: Produces unreliable results when used with datasets that contain heavy\nnoise.
Evaluates feature-wise normality of training data using the Shapiro-Wilk test.
\n\n
Purpose
\n\n
The Shapiro-Wilk test is utilized to investigate whether a particular dataset conforms to the standard normal\ndistribution. This analysis is crucial in machine learning modeling because the normality of the data can\nprofoundly impact the performance of the model. This metric is especially useful in evaluating various features of\nthe dataset in both classification and regression tasks.
\n\n
Test Mechanism
\n\n
The Shapiro-Wilk test is conducted on each feature column of the training dataset to determine if the data\ncontained fall within the normal distribution. The test presents a statistic and a p-value, with the p-value\nserving to validate or repudiate the null hypothesis, which is that the tested data is normally distributed.
\n\n
Signs of High Risk
\n\n
\n
A p-value that falls below 0.05 signifies a high risk as it discards the null hypothesis, indicating that the\ndata does not adhere to the normal distribution.
\n
For machine learning models built on the presumption of data normality, such an outcome could result in subpar\nperformance or incorrect predictions.
\n
\n\n
Strengths
\n\n
\n
The Shapiro-Wilk test is esteemed for its level of accuracy, thereby making it particularly well-suited to\ndatasets of small to moderate sizes.
\n
It proves its versatility through its efficient functioning in both classification and regression tasks.
\n
By separately testing each feature column, the Shapiro-Wilk test can raise an alarm if a specific feature does\nnot comply with the normality.
\n
\n\n
Limitations
\n\n
\n
The Shapiro-Wilk test's sensitivity can be a disadvantage as it often rejects the null hypothesis (i.e., data is\nnormally distributed), even for minor deviations, especially in large datasets. This may lead to unwarranted 'false\nalarms' of high risk by deeming the data as not normally distributed even if it approximates normal distribution.
\n
Exceptional care must be taken in managing missing data or outliers prior to testing as these can greatly skew\nthe results.
\n
Lastly, the Shapiro-Wilk test is not optimally suited for processing data with pronounced skewness or kurtosis.
Evaluates the skewness of numerical data in a dataset to check against a defined threshold, aiming to ensure data\nquality and optimize model performance.
\n\n
Purpose
\n\n
The purpose of the Skewness test is to measure the asymmetry in the distribution of data within a predictive\nmachine learning model. Specifically, it evaluates the divergence of said distribution from a normal distribution.\nUnderstanding the level of skewness helps identify data quality issues, which are crucial for optimizing the\nperformance of traditional machine learning models in both classification and regression settings.
\n\n
Test Mechanism
\n\n
This test calculates the skewness of numerical columns in the dataset, focusing specifically on numerical data\ntypes. The calculated skewness value is then compared against a predetermined maximum threshold, which is set by\ndefault to 1. If the skewness value is less than this maximum threshold, the test passes; otherwise, it fails. The\ntest results, along with the skewness values and column names, are then recorded for further analysis.
\n\n
Signs of High Risk
\n\n
\n
Substantial skewness levels that significantly exceed the maximum threshold.
\n
Persistent skewness in the data, indicating potential issues with the foundational assumptions of the machine\nlearning model.
\n
Subpar model performance, erroneous predictions, or biased inferences due to skewed data distributions.
\n
\n\n
Strengths
\n\n
\n
Fast and efficient identification of unequal data distributions within a machine learning model.
\n
Adjustable maximum threshold parameter, allowing for customization based on user needs.
\n
Provides a clear quantitative measure to mitigate model risks related to data skewness.
\n
\n\n
Limitations
\n\n
\n
Only evaluates numeric columns, potentially missing skewness or bias in non-numeric data.
\n
Assumes that data should follow a normal distribution, which may not always be applicable to real-world data.
\n
Subjective threshold for risk grading, requiring expert input and recurrent iterations for refinement.
Assesses potential correlations between pairs of time series variables through visualization to enhance\nunderstanding of their relationships.
\n\n
Purpose
\n\n
The SpreadPlot test aims to graphically illustrate and analyze the relationships between pairs of time series\nvariables within a given dataset. This facilitated understanding helps in identifying and assessing potential time\nseries correlations, such as cointegration, between the variables.
\n\n
Test Mechanism
\n\n
The SpreadPlot test computes and represents the spread between each pair of time series variables in the dataset.\nSpecifically, the difference between two variables is calculated and presented as a line graph. This process is\niterated for each unique pair of variables in the dataset, allowing for comprehensive visualization of their\nrelationships.
\n\n
Signs of High Risk
\n\n
\n
Large fluctuations in the spread over a given timespan.
\n
Unexpected patterns or trends that may signal potential risks in the underlying correlations between the\nvariables.
\n
Presence of significant missing data or extreme outlier values, which could potentially skew the spread and\nindicate high risk.
\n
\n\n
Strengths
\n\n
\n
Allows for thorough visual examination and interpretation of the correlations between time-series pairs.
\n
Aids in revealing complex relationships like cointegration.
\n
Enhances interpretability by visualizing the relationships, thereby helping in spotting outliers and trends.
\n
Capable of handling numerous variable pairs from the dataset through a versatile and adaptable process.
\n
\n\n
Limitations
\n\n
\n
Primarily serves as a visualization tool and does not offer quantitative measurements or statistics to\nobjectively determine relationships.
\n
Heavily relies on the quality and granularity of the data\u2014missing data or outliers can notably disturb the\ninterpretation of relationships.
\n
Can become inefficient or difficult to interpret with a high number of variables due to the profuse number of\nplots.
\n
Might not completely capture intricate non-linear relationships between the variables.
Generates and visualizes bar plots for each category in categorical features to evaluate the dataset's composition.
\n\n
Purpose
\n\n
The purpose of this metric is to visually analyze categorical data using bar plots. It is intended to evaluate the\ndataset's composition by displaying the counts of each category in each categorical feature.
\n\n
Test Mechanism
\n\n
The provided dataset is first checked to determine if it contains any categorical variables. If no categorical\ncolumns are found, the tool raises a ValueError. For each categorical variable in the dataset, a separate bar plot\nis generated. The number of occurrences for each category is calculated and displayed on the plot. If a dataset\ncontains multiple categorical columns, multiple bar plots are produced.
\n\n
Signs of High Risk
\n\n
\n
High risk could occur if the categorical variables exhibit an extreme imbalance, with categories having very few\ninstances possibly being underrepresented in the model, which could affect the model's performance and its ability\nto generalize.
\n
Another sign of risk is if there are too many categories in a single variable, which could lead to overfitting\nand make the model complex.
\n
\n\n
Strengths
\n\n
\n
Provides a visual and intuitively understandable representation of categorical data.
\n
Aids in the analysis of variable distributions.
\n
Helps in easily identifying imbalances or rare categories that could affect the model's performance.
\n
\n\n
Limitations
\n\n
\n
This method only works with categorical data and won't apply to numerical variables.
\n
It does not provide informative value when there are too many categories, as the bar chart could become cluttered\nand hard to interpret.
\n
Offers no insights into the model's performance or precision, but rather provides a descriptive analysis of the\ninput.
Generates histograms to provide graphical insight into the distribution of time intervals in a model's datetime\ndata.
\n\n
Purpose
\n\n
The TabularDateTimeHistograms metric is designed to provide graphical insight into the distribution of time\nintervals in a machine learning model's datetime data. By plotting histograms of differences between consecutive\ndate entries in all datetime variables, it enables an examination of the underlying pattern of time series data and\nidentification of anomalies.
\n\n
Test Mechanism
\n\n
This test operates by first identifying all datetime columns and extracting them from the dataset. For each\ndatetime column, it next computes the differences (in days) between consecutive dates, excluding zero values, and\nvisualizes these differences in a histogram. The Plotly library's histogram function is used to generate\nhistograms, which are labeled appropriately and provide a graphical representation of the frequency of different\nday intervals in the dataset.
\n\n
Signs of High Risk
\n\n
\n
If no datetime columns are detected in the dataset, this would lead to a ValueError. Hence, the absence of\ndatetime columns signifies a high risk.
\n
A severely skewed or irregular distribution depicted in the histogram may indicate possible complications with\nthe data, such as faulty timestamps or abnormalities.
\n
\n\n
Strengths
\n\n
\n
The metric offers a visual overview of time interval frequencies within the dataset, supporting the recognition\nof inherent patterns.
\n
Histogram plots can aid in the detection of potential outliers and data anomalies, contributing to an assessment\nof data quality.
\n
The metric is versatile, compatible with a range of task types, including classification and regression, and can\nwork with multiple datetime variables if present.
\n
\n\n
Limitations
\n\n
\n
A major weakness of this metric is its dependence on the visual examination of data, as it does not provide a\nmeasurable evaluation of the model.
\n
The metric might overlook complex or multi-dimensional trends in the data.
\n
The test is only applicable to datasets containing datetime columns and will fail if such columns are unavailable.
\n
The interpretation of the histograms relies heavily on the domain expertise and experience of the reviewer.
Summarizes key descriptive statistics for numerical, categorical, and datetime variables in a dataset.
\n\n
Purpose
\n\n
The main purpose of this metric is to gather and present the descriptive statistics of numerical, categorical, and\ndatetime variables present in a dataset. The attributes it measures include the count, mean, minimum and maximum\nvalues, percentage of missing values, data types of fields, and unique values for categorical fields, among others.
\n\n
Test Mechanism
\n\n
The test first segregates the variables in the dataset according to their data types (numerical, categorical, or\ndatetime). Then, it compiles summary statistics for each type of variable. The specifics of these statistics vary\ndepending on the type of variable:
\n\n
\n
For numerical variables, the metric extracts descriptors like count, mean, minimum and maximum values, count of\nmissing values, and data types.
\n
For categorical variables, it counts the number of unique values, displays unique values, counts missing values,\nand identifies data types.
\n
For datetime variables, it counts the number of unique values, identifies the earliest and latest dates, counts\nmissing values, and identifies data types.
\n
\n\n
Signs of High Risk
\n\n
\n
Masses of missing values in the descriptive statistics results could hint at high risk or failure, indicating\npotential data collection, integrity, and quality issues.
\n
Detection of inappropriate distributions for numerical variables, like having negative values for variables that\nare always supposed to be positive.
\n
Identifying inappropriate data types, like a continuous variable being encoded as a categorical type.
\n
\n\n
Strengths
\n\n
\n
Provides a comprehensive overview of the dataset.
\n
Gives a snapshot into the essence of the numerical, categorical, and datetime fields.
\n
Identifies potential data quality issues such as missing values or inconsistencies crucial for building credible\nmachine learning models.
\n
The metadata, including the data type and missing value information, are vital for anyone including data\nscientists dealing with the dataset before the modeling process.
\n
\n\n
Limitations
\n\n
\n
It does not perform any deeper statistical analysis or tests on the data.
\n
It does not handle issues such as outliers, or relationships between variables.
\n
It offers no insights into potential correlations or possible interactions between variables.
\n
It does not investigate the potential impact of missing values on the performance of the machine learning models.
\n
It does not explore potential transformation requirements that may be necessary to enhance the performance of the\nchosen algorithm.
Generates histograms for each numerical feature in a dataset to provide visual insights into data distribution and\ndetect potential issues.
\n\n
Purpose
\n\n
The purpose of this test is to provide visual analysis of numerical data through the generation of histograms for\neach numerical feature in the dataset. Histograms aid in the exploratory analysis of data, offering insight into\nthe distribution of the data, skewness, presence of outliers, and central tendencies. It helps in understanding if\nthe inputs to the model are normally distributed, which is a common assumption in many machine learning algorithms.
\n\n
Test Mechanism
\n\n
This test scans the provided dataset and extracts all the numerical columns. For each numerical column, it\nconstructs a histogram using plotly, with 50 bins. The deployment of histograms offers a robust visual aid,\nensuring unruffled identification and understanding of numerical data distribution patterns.
\n\n
Signs of High Risk
\n\n
\n
A high degree of skewness
\n
Unexpected data distributions
\n
Existence of extreme outliers in the histograms
\n
\n\n
These may indicate issues with the data that the model is receiving. If data for a numerical feature is expected to\nfollow a certain distribution (like a normal distribution) but does not, it could lead to sub-par performance by\nthe model. As such these instances should be treated as high-risk indicators.
\n\n
Strengths
\n\n
\n
Provides a simple, easy-to-interpret visualization of how data for each numerical attribute is distributed.
\n
Helps detect skewed values and outliers that could potentially harm the AI model's performance.
\n
Can be applied to large datasets and multiple numerical variables conveniently.
\n
\n\n
Limitations
\n\n
\n
Only works with numerical data, thus ignoring non-numerical or categorical data.
\n
Does not analyze relationships between different features, only the individual feature distributions.
\n
Is a univariate analysis and may miss patterns or anomalies that only appear when considering multiple variables\ntogether.
\n
Does not provide any insight into how these features affect the output of the model; it is purely an input\nanalysis tool.
Generates bar plots visualizing the default rates of categorical features for a classification machine learning\nmodel.
\n\n
Purpose
\n\n
This test, implemented as a metric, is designed to provide an intuitive, graphical summary of the decision-making\npatterns exhibited by a categorical classification machine learning model. The model's performance is evaluated\nusing bar plots depicting the ratio of target rates\u2014meaning the proportion of positive classes\u2014for different\ncategorical inputs. This allows for an easy, at-a-glance understanding of the model's accuracy.
\n\n
Test Mechanism
\n\n
The test involves creating a pair of bar plots for each categorical feature in the dataset. The first plot depicts\nthe frequency of each category in the dataset, with each category visually distinguished by its unique color. The\nsecond plot shows the mean target rate of each category (sourced from the \"default_column\"). Plotly, a Python\nlibrary, is used to generate these plots, with distinct plots created for each feature. If no specific columns are\nselected, the test will generate plots for each categorical column in the dataset.
\n\n
Signs of High Risk
\n\n
\n
Inconsistent or non-binary values in the \"default_column\" could complicate or render impossible the calculation\nof average target rates.
\n
Particularly low or high target rates for a specific category might suggest that the model is misclassifying\ninstances of that category.
\n
\n\n
Strengths
\n\n
\n
This test offers a visually interpretable breakdown of the model's decisions, providing an easy way to spot\nirregularities, inconsistencies, or patterns.
\n
Its flexibility allows for the inspection of one or multiple columns, as needed.
\n
\n\n
Limitations
\n\n
\n
The readability of the bar plots drops as the number of distinct categories increases in the dataset, which can\nmake them harder to understand and less useful.
Generates a detailed analysis for the provided time series dataset, summarizing key statistics to identify trends,\npatterns, and data quality issues.
\n\n
Purpose
\n\n
The TimeSeriesDescription function aims to analyze an individual time series by providing a summary of key\nstatistics. This helps in understanding trends, patterns, and data quality issues within the time series.
\n\n
Test Mechanism
\n\n
The function extracts the time series data and provides a summary of key statistics. The dataset is expected to\nhave a datetime index. The function checks this and raises an error if the index is not in datetime format. For\neach variable (column) in the dataset, appropriate statistics including start date, end date, frequency, number of\nmissing values, count, min, and max values are calculated.
\n\n
Signs of High Risk
\n\n
\n
If the index of the dataset is not in datetime format, it could lead to errors in time-series analysis.
\n
Inconsistent or missing data within the dataset might affect the analysis of trends and patterns.
\n
\n\n
Strengths
\n\n
\n
Provides a comprehensive summary of key statistics for each variable, helping to identify data quality issues\nsuch as missing values.
\n
Helps in understanding the distribution and range of the data by including min and max values.
\n
\n\n
Limitations
\n\n
\n
Assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access the pandas\nDataFrame.
\n
Only analyzes datasets with a datetime index and will raise an error for other types of indices.
\n
Does not handle large datasets efficiently; performance may degrade with very large datasets.
Evaluates the descriptive statistics of a time series dataset to identify trends, patterns, and data quality issues.
\n\n
Purpose
\n\n
The purpose of the TimeSeriesDescriptiveStatistics function is to analyze an individual time series by providing a\nsummary of key descriptive statistics. This analysis helps in understanding trends, patterns, and data quality\nissues within the time series dataset.
\n\n
Test Mechanism
\n\n
The function extracts the time series data and provides a summary of key descriptive statistics. The dataset is\nexpected to have a datetime index, and the function will check this and raise an error if the index is not in a\ndatetime format. For each variable (column) in the dataset, appropriate statistics, including start date, end date,\nmin, mean, max, skewness, kurtosis, and count, are calculated.
\n\n
Signs of High Risk
\n\n
\n
If the index of the dataset is not in datetime format, it could lead to errors in time-series analysis.
\n
Inconsistent or missing data within the dataset might affect the analysis of trends and patterns.
\n
\n\n
Strengths
\n\n
\n
Provides a comprehensive summary of key descriptive statistics for each variable.
\n
Helps identify data quality issues and understand the distribution of the data.
\n
\n\n
Limitations
\n\n
\n
Assumes the dataset is provided as a DataFrameDataset object with a .df attribute to access the pandas DataFrame.
\n
Only analyzes datasets with a datetime index and will raise an error for other types of indices.
\n
Does not handle large datasets efficiently, and performance may degrade with very large datasets.
Evaluates consistency of time series data frequency and generates a frequency plot.
\n\n
Purpose
\n\n
The purpose of the TimeSeriesFrequency test is to evaluate the consistency in the frequency of data points in a\ntime-series dataset. This test inspects the intervals or duration between each data point to determine if a fixed\npattern (such as daily, weekly, or monthly) exists. The identification of such patterns is crucial to time-series\nanalysis as any irregularities could lead to erroneous results and hinder the model's capacity for identifying\ntrends and patterns.
\n\n
Test Mechanism
\n\n
Initially, the test checks if the dataframe index is in datetime format. Subsequently, it utilizes pandas'\ninfer_freq method to identify the frequency of each data series within the dataframe. The infer_freq method\nattempts to establish the frequency of a time series and returns both the frequency string and a dictionary\nrelating these strings to their respective labels. The test compares the frequencies of all datasets. If they share\na common frequency, the test passes, but it fails if they do not. Additionally, Plotly is used to create a\nfrequency plot, offering a visual depiction of the time differences between consecutive entries in the dataframe\nindex.
\n\n
Signs of High Risk
\n\n
\n
The test fails, indicating multiple unique frequencies within the dataset. This failure could suggest irregular\nintervals between observations, potentially interrupting pattern recognition or trend analysis.
\n
The presence of missing or null frequencies could be an indication of inconsistencies in data or gaps within the\ndata collection process.
\n
\n\n
Strengths
\n\n
\n
This test uses a systematic approach to checking the consistency of data frequency within a time-series dataset.
\n
It increases the model's reliability by asserting the consistency of observations over time, an essential factor\nin time-series analysis.
\n
The test generates a visual plot, providing an intuitive representation of the dataset's frequency distribution,\nwhich caters to visual learners and aids in interpretation and explanation.
\n
\n\n
Limitations
\n\n
\n
This test is only applicable to time-series datasets and hence not suitable for other types of datasets.
\n
The infer_freq method might not always correctly infer frequency when faced with missing or irregular data\npoints.
\n
Depending on context or the model under development, mixed frequencies might sometimes be acceptable, but this\ntest considers them a failing condition.
Visualizes distribution of time-series data using histograms and Kernel Density Estimation (KDE) lines.
\n\n
Purpose
\n\n
The TimeSeriesHistogram test aims to perform a histogram analysis on time-series data to assess the distribution of\nvalues within a dataset over time. This test is useful for regression tasks and can be applied to various types of\ndata, such as internet traffic, stock prices, and weather data, providing insights into the probability\ndistribution, skewness, and kurtosis of the dataset.
\n\n
Test Mechanism
\n\n
This test operates on a specific column within the dataset that must have a datetime type index. For each column in\nthe dataset, a histogram is created using Plotly's histplot function. If the dataset includes more than one\ntime-series, a distinct histogram is plotted for each series. Additionally, a Kernel Density Estimate (KDE) line is\ndrawn for each histogram, visualizing the data's underlying probability distribution. The x and y-axis labels are\nhidden to focus solely on the data distribution.
\n\n
Signs of High Risk
\n\n
\n
The dataset lacks a column with a datetime type index.
\n
The specified columns do not exist within the dataset.
\n
High skewness or kurtosis in the data distribution, indicating potential bias.
\n
Presence of significant outliers in the data distribution.
\n
\n\n
Strengths
\n\n
\n
Serves as a visual diagnostic tool for understanding data behavior and distribution trends.
\n
Effective for analyzing both single and multiple time-series data.
\n
KDE line provides a smooth estimate of the overall trend in data distribution.
\n
\n\n
Limitations
\n\n
\n
Provides a high-level view without specific numeric measures such as skewness or kurtosis.
\n
The histogram loses some detail due to binning of data values.
\n
Cannot handle non-numeric data columns.
\n
Histogram shape may be sensitive to the number of bins used.
Generates and analyses time-series data through line plots revealing trends, patterns, anomalies over time.
\n\n
Purpose
\n\n
The TimeSeriesLinePlot metric is designed to generate and analyze time series data through the creation of line\nplots. This assists in the initial inspection of the data by providing a visual representation of patterns, trends,\nseasonality, irregularity, and anomalies that may be present in the dataset over a period of time.
\n\n
Test Mechanism
\n\n
The mechanism for this Python class involves extracting the column names from the provided dataset and subsequently\ngenerating line plots for each column using the Plotly Python library. For every column in the dataset, a\ntime-series line plot is created where the values are plotted against the dataset's datetime index. It is important\nto note that indexes that are not of datetime type will result in a ValueError.
\n\n
Signs of High Risk
\n\n
\n
Presence of time-series data that does not have datetime indices.
\n
Provided columns do not exist in the provided dataset.
\n
The detection of anomalous patterns or irregularities in the time-series plots, indicating potential high model\ninstability or probable predictive error.
\n
\n\n
Strengths
\n\n
\n
The visual representation of complex time series data, which simplifies understanding and helps in recognizing\ntemporal trends, patterns, and anomalies.
\n
The adaptability of the metric, which allows it to effectively work with multiple time series within the same\ndataset.
\n
Enables the identification of anomalies and irregular patterns through visual inspection, assisting in spotting\npotential data or model performance problems.
\n
\n\n
Limitations
\n\n
\n
The effectiveness of the metric is heavily reliant on the quality and patterns of the provided time series data.
\n
Exclusively a visual tool, it lacks the capability to provide quantitative measurements, making it less effective\nfor comparing and ranking multiple models or when specific numerical diagnostics are needed.
\n
The metric necessitates that the time-specific data has been transformed into a datetime index, with the data\nformatted correctly.
\n
The metric has an inherent limitation in that it cannot extract deeper statistical insights from the time series\ndata, which can limit its efficacy with complex data structures and phenomena.
Validates time-series data quality by confirming the count of missing values is below a certain threshold.
\n\n
Purpose
\n\n
This test is designed to validate the quality of a historical time-series dataset by verifying that the number of\nmissing values is below a specified threshold. As time-series models greatly depend on the continuity and\ntemporality of data points, missing values could compromise the model's performance. Consequently, this test aims\nto ensure data quality and readiness for the machine learning model, safeguarding its predictive capacity.
\n\n
Test Mechanism
\n\n
The test method commences by validating if the dataset has a datetime index; if not, an error is raised. It\nestablishes a lower limit threshold for missing values and performs a missing values check on each column of the\ndataset. An object for the test result is created stating whether the number of missing values is within the\nspecified threshold. Additionally, the test calculates the percentage of missing values alongside the raw count.
\n\n
Signs of High Risk
\n\n
\n
The number of missing values in any column of the dataset surpasses the threshold, marking a failure and a\nhigh-risk scenario. The reasons could range from incomplete data collection, faulty sensors to data preprocessing\nerrors.
\n
\n\n
Strengths
\n\n
\n
Effectively identifies missing values which could adversely affect the model\u2019s performance.
\n
Applicable and customizable through the threshold parameter across different data sets.
\n
Goes beyond raw numbers by calculating the percentage of missing values, offering a more relative understanding\nof data scarcity.
\n
\n\n
Limitations
\n\n
\n
Although it identifies missing values, the test does not provide solutions to handle them.
\n
The test demands that the dataset should have a datetime index, hence limiting its use only to time series\nanalysis.
\n
The test's sensitivity to the 'min_threshold' parameter may raise false alarms if set too strictly or may\noverlook problematic data if set too loosely.
\n
Solely focuses on the 'missingness' of the data and might fall short in addressing other aspects of data quality.
Identifies and visualizes outliers in time-series data using the z-score method.
\n\n
Purpose
\n\n
This test is designed to identify outliers in time-series data using the z-score method. It's vital for ensuring\ndata quality before modeling, as outliers can skew predictive models and significantly impact their overall\nperformance.
\n\n
Test Mechanism
\n\n
The test processes a given dataset which must have datetime indexing, checks if a 'zscore_threshold' parameter has\nbeen supplied, and identifies columns with numeric data types. After finding numeric columns, the implementer then\napplies the z-score method to each numeric column, identifying outliers based on the threshold provided. Each\noutlier is listed together with their variable name, z-score, timestamp, and relative threshold in a dictionary and\nconverted to a DataFrame for convenient output. Additionally, it produces visual plots for each time series\nillustrating outliers in the context of the broader dataset. The 'zscore_threshold' parameter sets the limit beyond\nwhich a data point will be labeled as an outlier. The default threshold is set at 3, indicating that any data point\nthat falls 3 standard deviations away from the mean will be marked as an outlier.
\n\n
Signs of High Risk
\n\n
\n
Many or substantial outliers are present within the dataset, indicating significant anomalies.
\n
Data points with z-scores higher than the set threshold.
\n
Potential impact on the performance of machine learning models if outliers are not properly addressed.
\n
\n\n
Strengths
\n\n
\n
The z-score method is a popular and robust method for identifying outliers in a dataset.
\n
Simplifies time series maintenance by requiring a datetime index.
\n
Identifies outliers for each numeric feature individually.
\n
Provides an elaborate report showing variables, dates, z-scores, and pass/fail tests.
\n
Offers visual inspection for detected outliers through plots.
\n
\n\n
Limitations
\n\n
\n
The test only identifies outliers in numeric columns, not in categorical variables.
\n
The utility and accuracy of z-scores can be limited if the data doesn't follow a normal distribution.
\n
The method relies on a subjective z-score threshold for deciding what constitutes an outlier, which might not\nalways be suitable depending on the dataset and use case.
\n
It does not address possible ways to handle identified outliers in the data.
\n
The requirement for a datetime index could limit its application.
Identifies numerical columns in a dataset that contain an excessive number of zero values, defined by a threshold\npercentage.
\n\n
Purpose
\n\n
The 'TooManyZeroValues' test is utilized to identify numerical columns in the dataset that may present a quantity\nof zero values considered excessive. The aim is to detect situations where these may implicate data sparsity or a\nlack of variation, limiting their effectiveness within a machine learning model. The definition of 'too many' is\nquantified as a percentage of total values, with a default set to 3%.
\n\n
Test Mechanism
\n\n
This test is conducted by looping through each column in the dataset and categorizing those that pertain to\nnumerical data. On identifying a numerical column, the function computes the total quantity of zero values and\ntheir ratio to the total row count. Should the proportion exceed a pre-set threshold parameter, set by default at\n0.03 or 3%, the column is considered to have failed the test. The results for each column are summarized and\nreported, indicating the count and percentage of zero values for each numerical column, alongside a status\nindicating whether the column has passed or failed the test.
\n\n
Signs of High Risk
\n\n
\n
Numerical columns showing a high ratio of zero values when compared to the total count of rows (exceeding the\npredetermined threshold).
\n
Columns characterized by zero values across the board suggest a complete lack of data variation, signifying high\nrisk.
\n
\n\n
Strengths
\n\n
\n
Assists in highlighting columns featuring an excess of zero values that could otherwise go unnoticed within a\nlarge dataset.
\n
Provides the flexibility to alter the threshold that determines when the quantity of zero values becomes 'too\nmany', thus catering to specific needs of a particular analysis or model.
\n
Offers feedback in the form of both counts and percentages of zero values, which allows a closer inspection of\nthe distribution and proportion of zeros within a column.
\n
Targets specifically numerical data, thereby avoiding inappropriate application to non-numerical columns and\nmitigating the risk of false test failures.
\n
\n\n
Limitations
\n\n
\n
Is exclusively designed to check for zero values and doesn\u2019t assess the potential impact of other values that\ncould affect the dataset, such as extremely high or low figures, missing values, or outliers.
\n
Lacks the ability to detect a repetitive pattern of zeros, which could be significant in time-series or\nlongitudinal data.
\n
Zero values can actually be meaningful in some contexts; therefore, tagging them as 'too many' could potentially\nmisinterpret the data to some extent.
\n
This test does not take into consideration the context of the dataset, and fails to recognize that within certain\ncolumns, a high number of zero values could be quite normal and not necessarily an indicator of poor data quality.
\n
Cannot evaluate non-numerical or categorical columns, which might bring with them different types of concerns or\nissues.
Verifies the diversity of the dataset by ensuring that the count of unique rows exceeds a prescribed threshold.
\n\n
Purpose
\n\n
The UniqueRows test is designed to gauge the quality of the data supplied to the machine learning model by\nverifying that the count of distinct rows in the dataset exceeds a specific threshold, thereby ensuring a varied\ncollection of data. Diversity in data is essential for training an unbiased and robust model that excels when faced\nwith novel data.
\n\n
Test Mechanism
\n\n
The testing process starts with calculating the total number of rows in the dataset. Subsequently, the count of\nunique rows is determined for each column in the dataset. If the percentage of unique rows (calculated as the ratio\nof unique rows to the overall row count) is less than the prescribed minimum percentage threshold given as a\nfunction parameter, the test passes. The results are cached and a final pass or fail verdict is given based on\nwhether all columns have successfully passed the test.
\n\n
Signs of High Risk
\n\n
\n
A lack of diversity in data columns, demonstrated by a count of unique rows that falls short of the preset\nminimum percentage threshold, is indicative of high risk.
\n
This lack of variety in the data signals potential issues with data quality, possibly leading to overfitting in\nthe model and issues with generalization, thus posing a significant risk.
\n
\n\n
Strengths
\n\n
\n
The UniqueRows test is efficient in evaluating the data's diversity across each information column in the dataset.
\n
This test provides a quick, systematic method to assess data quality based on uniqueness, which can be pivotal in\ndeveloping effective and unbiased machine learning models.
\n
\n\n
Limitations
\n\n
\n
A limitation of the UniqueRows test is its assumption that the data's quality is directly proportionate to its\nuniqueness, which may not always hold true. There might be contexts where certain non-unique rows are essential and\nshould not be overlooked.
\n
The test does not consider the relative 'importance' of each column in predicting the output, treating all\ncolumns equally.
\n
This test may not be suitable or useful for categorical variables, where the count of unique categories is\ninherently limited.
Generates visualizations of Weight of Evidence (WoE) and Information Value (IV) for understanding predictive power\nof categorical variables in a data set.
\n\n
Purpose
\n\n
This test is designed to visualize the Weight of Evidence (WoE) and Information Value (IV) for categorical\nvariables in a provided dataset. By showcasing the data distribution across different categories of each feature,\nit aids in understanding each variable's predictive power in the context of a classification-based machine learning\nmodel. Commonly used in credit scoring models, WoE and IV are robust statistical methods for evaluating a\nvariable's predictive power.
\n\n
Test Mechanism
\n\n
The test implementation follows defined steps. Initially, it selects non-numeric columns from the dataset and\nchanges them to string type, paving the way for accurate binning. It then performs an automated WoE binning\noperation on these selected features, effectively categorizing the potential values of a variable into distinct\nbins. After the binning process, the function generates two separate visualizations (a scatter chart for WoE values\nand a bar chart for IV) for each variable. These visual presentations are formed according to the spread of each\nmetric across various categories of each feature.
\n\n
Signs of High Risk
\n\n
\n
Errors occurring during the binning process.
\n
Challenges in converting non-numeric columns into string data type.
\n
Misbalance in the distribution of WoE and IV, with certain bins overtaking others conspicuously. This could\ndenote that the model is disproportionately dependent on certain variables or categories for predictions, an\nindication of potential risks to its robustness and generalizability.
\n
\n\n
Strengths
\n\n
\n
Provides a detailed visual representation of the relationship between feature categories and the target variable.\nThis grants an intuitive understanding of each feature's contribution to the model.
\n
Allows for easy identification of features with high impact, facilitating feature selection and enhancing\ncomprehension of the model's decision logic.
\n
WoE conversions are monotonic, upholding the rank ordering of the original data points, which simplifies analysis.
\n
\n\n
Limitations
\n\n
\n
The method is largely reliant on the binning process, and an inappropriate binning threshold or bin number choice\nmight result in a misrepresentation of the variable's distribution.
\n
While excellent for categorical data, the encoding of continuous variables into categorical can sometimes lead to\ninformation loss.
\n
Extreme or outlier values can dramatically affect the computation of WoE and IV, skewing results.
\n
The method requires a sufficient number of events per bin to generate a reliable information value and weight of\nevidence.
Assesses the Weight of Evidence (WoE) and Information Value (IV) of each feature to evaluate its predictive power\nin a binary classification model.
\n\n
Purpose
\n\n
The Weight of Evidence (WoE) and Information Value (IV) test is designed to evaluate the predictive power of each\nfeature in a machine learning model. This test generates binned groups of values from each feature, computes the\nWoE and IV for each bin, and provides insights into the relationship between each feature and the target variable,\nillustrating their contribution to the model's predictive capabilities.
\n\n
Test Mechanism
\n\n
The test uses the scorecardpy.woebin method to perform automatic binning of the dataset based on WoE. The method\naccepts a list of break points for binning numeric variables through the parameter breaks_adj. If no breaks are\nprovided, it uses default binning. The bins are then used to calculate the WoE and IV values, effectively creating\na dataframe that includes the bin boundaries, WoE, and IV values for each feature. A target variable is required\nin the dataset to perform this analysis.
\n\n
Signs of High Risk
\n\n
\n
High IV values, indicating variables with excessive predictive power which might lead to overfitting.
\n
Errors during the binning process, potentially due to inappropriate data types or poorly defined bins.
\n
\n\n
Strengths
\n\n
\n
Highly effective for feature selection in binary classification problems, as it quantifies the predictive\ninformation within each feature concerning the binary outcome.
\n
The WoE transformation creates a monotonic relationship between the target and independent variables.
\n
\n\n
Limitations
\n\n
\n
Primarily designed for binary classification tasks, making it less applicable or reliable for multi-class\nclassification or regression tasks.
\n
Potential difficulties if the dataset has many features, non-binnable features, or non-numeric features.
\n
The metric does not help in distinguishing whether the observed predictive factor is due to data randomness or a\ntrue phenomenon.
Evaluates the order of integration and stationarity of time series data using the Zivot-Andrews unit root test.
\n\n
Purpose
\n\n
The Zivot-Andrews Arch metric is used to evaluate the order of integration for time series data in a machine\nlearning model. It's designed to test for stationarity, a crucial aspect of time series analysis, where data points\nare independent of time. Stationarity means that the statistical properties such as mean, variance, and\nautocorrelation are constant over time.
\n\n
Test Mechanism
\n\n
The Zivot-Andrews unit root test is performed on each feature in the dataset using the ZivotAndrews function from\nthe arch.unitroot module. This function returns several metrics for each feature, including the statistical\nvalue, p-value (probability value), the number of lags used, and the number of observations. The p-value is used to\ndecide on the null hypothesis (the time series has a unit root and is non-stationary) based on a chosen level of\nsignificance.
\n\n
Signs of High Risk
\n\n
\n
A high p-value suggests high risk, indicating insufficient evidence to reject the null hypothesis, implying that\nthe time series has a unit root and is non-stationary.
\n
Non-stationary time series data can lead to misleading statistics and unreliable machine learning models.
\n
\n\n
Strengths
\n\n
\n
Dynamically tests for stationarity against structural breaks in time series data, offering robust evaluation of\nstationarity in features.
\n
Especially beneficial with financial, economic, or other time-series data where data observations lack a\nconsistent pattern and structural breaks may occur.
\n
\n\n
Limitations
\n\n
\n
Assumes data is derived from a single-equation, autoregressive model, making it less appropriate for multivariate\ntime series data or data not aligning with this model.
\n
May not account for unexpected shocks or changes in the series trend, both of which can significantly impact data\nstationarity.
Assesses the most frequent non-stopwords in a text column for identifying prevalent language patterns.
\n\n
Purpose
\n\n
The CommonWords metric is used to identify and visualize the most prevalent words within a specified text column of\na dataset. This provides insights into the prevalent language patterns and vocabulary, especially useful in Natural\nLanguage Processing (NLP) tasks such as text classification and text summarization.
\n\n
Test Mechanism
\n\n
The test methodology involves splitting the specified text column's entries into words, collating them into a\ncorpus, and then counting the frequency of each word using the Counter. The forty most frequently occurring\nnon-stopwords are then visualized in an interactive bar chart using Plotly, where the x-axis represents the words,\nand the y-axis indicates their frequency of occurrence.
\n\n
Signs of High Risk
\n\n
\n
A lack of distinct words within the list, or the most common words being stopwords.
\n
Frequent occurrence of irrelevant or inappropriate words could point out a poorly curated or noisy dataset.
\n
An error returned due to the absence of a valid Dataset object, indicating high risk as the metric cannot be\neffectively implemented without it.
\n
\n\n
Strengths
\n\n
\n
The metric provides clear insights into the language features \u2013 specifically word frequency \u2013 of unstructured\ntext data.
\n
It can reveal prominent vocabulary and language patterns, which prove vital for feature extraction in NLP tasks.
\n
The interactive visualization helps in quickly capturing the patterns and understanding the data intuitively.
\n
\n\n
Limitations
\n\n
\n
The test disregards semantic or context-related information as it solely focuses on word frequency.
\n
It intentionally ignores stopwords, which might carry necessary significance in certain scenarios.
\n
The applicability is limited to English-language text data as English stopwords are used for filtering, hence\ncannot account for data in other languages.
\n
The metric requires a valid Dataset object, indicating a dependency condition that limits its broader\napplicability.
Assesses hashtag frequency in a text column, highlighting usage trends and potential dataset bias or spam.
\n\n
Purpose
\n\n
The Hashtags test is designed to measure the frequency of hashtags used within a given text column in a dataset. It\nis particularly useful for natural language processing tasks such as text classification and text summarization.\nThe goal is to identify common trends and patterns in the use of hashtags, which can serve as critical indicators\nor features within a machine learning model.
\n\n
Test Mechanism
\n\n
The test implements a regular expression (regex) to extract all hashtags from the specified text column. For each\nhashtag found, it makes a tally of its occurrences. It then outputs a list of the top N hashtags (default is 25,\nbut customizable), sorted by their counts in descending order. The results are also visualized in a bar plot, with\nfrequency counts on the y-axis and the corresponding hashtags on the x-axis.
\n\n
Signs of High Risk
\n\n
\n
A low diversity in the usage of hashtags, as indicated by a few hashtags being used disproportionately more than\nothers.
\n
Repeated usage of one or few hashtags can be indicative of spam or a biased dataset.
\n
If there are no or extremely few hashtags found in the dataset, it perhaps signifies that the text data does not\ncontain structured social media data.
\n
\n\n
Strengths
\n\n
\n
Provides a concise visual representation of the frequency of hashtags, which can be critical for understanding\ntrends about a particular topic in text data.
\n
Instrumental in tasks specifically related to social media text analytics, such as opinion analysis and trend\ndiscovery.
\n
Adaptable, allowing the flexibility to determine the number of top hashtags to be analyzed.
\n
\n\n
Limitations
\n\n
\n
Assumes the presence of hashtags and therefore may not be applicable for text datasets that do not contain\nhashtags (e.g., formal documents, scientific literature).
\n
Language-specific limitations of hashtag formulations are not taken into account.
\n
Does not account for typographical errors, variations, or synonyms in hashtags.
\n
Does not provide context or sentiment associated with the hashtags, so the information provided may have limited\nutility on its own.
Assesses the diversity of languages in a textual dataset by detecting and visualizing the distribution of languages.
\n\n
Purpose
\n\n
The Language Detection test aims to identify and visualize the distribution of languages present within a textual\ndataset. This test helps in understanding the diversity of languages in the data, which is crucial for developing\nand validating multilingual models.
\n\n
Test Mechanism
\n\n
This test operates by:
\n\n
\n
Checking if the dataset has a specified text column.
\n
Using a language detection library to determine the language of each text entry in the dataset.
\n
Generating a histogram plot of the language distribution, with language codes on the x-axis and their frequencies\non the y-axis.
\n
\n\n
If the text column is not specified, a ValueError is raised to ensure proper dataset configuration.
\n\n
Signs of High Risk
\n\n
\n
A high proportion of entries returning \"Unknown\" language codes.
\n
Detection of unexpectedly diverse or incorrect language codes, indicating potential data quality issues.
\n
Significant imbalance in language distribution, which might indicate potential biases in the dataset.
\n
\n\n
Strengths
\n\n
\n
Provides a visual representation of language diversity within the dataset.
\n
Helps identify data quality issues related to incorrect or unknown language detection.
\n
Useful for ensuring that multilingual models have adequate and appropriate representation from various languages.
\n
\n\n
Limitations
\n\n
\n
Dependency on the accuracy of the language detection library, which may not be perfect.
\n
Languages with similar structures or limited text length may be incorrectly classified.
\n
The test returns \"Unknown\" for entries where language detection fails, which might mask underlying issues with\ncertain languages or text formats.
Calculates and visualizes frequencies of '@' prefixed mentions in a text-based dataset for NLP model analysis.
\n\n
Purpose
\n\n
The \"Mentions\" test is designed to gauge the quality of data in a Natural Language Processing (NLP) or text-focused\nMachine Learning model. The primary objective is to identify and calculate the frequency of 'mentions' within a\nchosen text column of a dataset. A 'mention' in this context refers to individual text elements that are prefixed\nby '@'. The output of this test reveals the most frequently mentioned entities or usernames, which can be integral\nfor applications such as social media analyses or customer sentiment analyses.
\n\n
Test Mechanism
\n\n
The test first verifies the existence of a text column in the provided dataset. It then employs a regular\nexpression pattern to extract mentions from the text. Subsequently, the frequency of each unique mention is\ncalculated. The test selects the most frequent mentions based on default or user-defined parameters, the default\nbeing the top 25, for representation. This process of thresholding forms the core of the test. A treemap plot\nvisualizes the test results, where the size of each rectangle corresponds to the frequency of a particular mention.
\n\n
Signs of High Risk
\n\n
\n
The lack of a valid text column in the dataset, which would result in the failure of the test execution.
\n
The absence of any mentions within the text data, indicating that there might not be any text associated with\n'@'. This situation could point toward sparse or poor-quality data, thereby hampering the model's generalization or\nlearning capabilities.
\n
\n\n
Strengths
\n\n
\n
The test is specifically optimized for text-based datasets which gives it distinct power in the context of NLP.
\n
It enables quick identification and visually appealing representation of the predominant elements or mentions.
\n
It can provide crucial insights about the most frequently mentioned entities or usernames.
\n
\n\n
Limitations
\n\n
\n
The test only recognizes mentions that are prefixed by '@', hence useful textual aspects not preceded by '@'\nmight be ignored.
\n
This test isn't suited for datasets devoid of textual data.
\n
It does not provide insights on less frequently occurring data or outliers, which means potentially significant\npatterns could be overlooked.
Analyzes the polarity and subjectivity of text data within a given dataset to visualize the sentiment distribution.
\n\n
Purpose
\n\n
The Polarity and Subjectivity test is designed to evaluate the sentiment expressed in textual data. By analyzing\nthese aspects, it helps to identify the emotional tone and subjectivity of the dataset, which could be crucial in\nunderstanding customer feedback, social media sentiments, or other text-related data.
\n\n
Test Mechanism
\n\n
This test uses TextBlob to compute the polarity and subjectivity scores of textual data in a given dataset. The\nmechanism includes:
\n\n
\n
Iterating through each text entry in the specified column of the dataset.
\n
Applying the TextBlob library to compute the polarity (ranging from -1 for negative sentiment to +1 for positive\nsentiment) and subjectivity (ranging from 0 for objective to 1 for subjective) for each entry.
\n
Creating a scatter plot using Plotly to visualize the relationship between polarity and subjectivity.
\n
\n\n
Signs of High Risk
\n\n
\n
High concentration of negative polarity values indicating prevalent negative sentiments.
\n
High subjectivity scores suggesting the text data is largely opinion-based rather than factual.
\n
Disproportionate clusters of extreme scores (e.g., many points near -1 or +1 polarity).
\n
\n\n
Strengths
\n\n
\n
Quantifies sentiment and subjectivity which can provide actionable insights.
\n
Visualizes sentiment distribution, aiding in easy interpretation.
\n
Utilizes well-established TextBlob library for sentiment analysis.
\n
\n\n
Limitations
\n\n
\n
Polarity and subjectivity calculations may oversimplify nuanced text sentiments.
\n
Reliance on TextBlob which may not be accurate for all domains or contexts.
\n
Visualization could become cluttered with very large datasets, making interpretation difficult.
Analyzes and visualizes the frequency distribution of punctuation usage in a given text dataset.
\n\n
Purpose
\n\n
The Punctuations Metric's primary purpose is to analyze the frequency of punctuation usage within a given text\ndataset. This is often used in Natural Language Processing tasks, such as text classification and text\nsummarization.
\n\n
Test Mechanism
\n\n
The test begins by verifying that the input \"dataset\" is of the type VMDataset. The count_mode parameter must be\neither \"token\" (counts punctuation marks as individual tokens) or \"word\" (counts punctuation marks within words).\nFollowing that, a corpus is created from the dataset by splitting its text on spaces. Each unique punctuation\ncharacter in the text corpus is then tallied. The frequency distribution of each punctuation symbol is visualized\nas a bar graph, with these results being stored as Figures and associated with the main Punctuations object.
\n\n
Signs of High Risk
\n\n
\n
Excessive or unusual frequency of specific punctuation marks, potentially denoting dubious quality, data\ncorruption, or skewed data.
\n
\n\n
Strengths
\n\n
\n
Provides valuable insights into the distribution of punctuation usage in a text dataset.
\n
Important in validating the quality, consistency, and nature of the data.
\n
Can provide hints about the style or tonality of the text corpus, such as informal and emotional context\nindicated by frequent exclamation marks.
\n
\n\n
Limitations
\n\n
\n
Focuses solely on punctuation usage, potentially missing other important textual characteristics.
\n
General cultural or tonality assumptions based on punctuation distribution can be misguiding, as these vary\nacross different languages and contexts.
\n
Less effective with languages that use non-standard or different punctuation.
\n
Visualization may lack interpretability when there are many unique punctuation marks in the dataset.
Analyzes the sentiment of text data within a dataset using the VADER sentiment analysis tool.
\n\n
Purpose
\n\n
The Sentiment test evaluates the overall sentiment of text data within a dataset. By analyzing sentiment scores, it\naims to ensure that the model is interpreting text data accurately and is not biased towards a particular sentiment.
\n\n
Test Mechanism
\n\n
This test uses the VADER (Valence Aware Dictionary and sEntiment Reasoner) SentimentIntensityAnalyzer. It processes\neach text entry in a specified column of the dataset to calculate the compound sentiment score, which represents\nthe overall sentiment polarity. The distribution of these sentiment scores is then visualized using a KDE (Kernel\nDensity Estimation) plot, highlighting any skewness or concentration in sentiment.
\n\n
Signs of High Risk
\n\n
\n
Extreme polarity in sentiment scores, indicating potential bias.
\n
Unusual concentration of sentiment scores in a specific range.
\n
Significant deviation from expected sentiment distribution for the given text data.
\n
\n\n
Strengths
\n\n
\n
Provides a clear visual representation of sentiment distribution.
\n
Uses a well-established sentiment analysis tool (VADER).
\n
Can handle a wide range of text data, making it flexible for various applications.
\n
\n\n
Limitations
\n\n
\n
May not capture nuanced or context-specific sentiments.
\n
Relies heavily on the accuracy of the VADER sentiment analysis tool.
\n
Visualization alone may not provide comprehensive insights into underlying causes of sentiment distribution.
Evaluates and visualizes the frequency of English stop words in a text dataset against a defined threshold.
\n\n
Purpose
\n\n
The StopWords threshold test is a tool designed for assessing the quality of text data in an ML model. It focuses\non the identification and analysis of \"stop words\" in a given dataset. Stop words are frequent, common, yet\nsemantically insignificant words (for example: \"the\", \"and\", \"is\") in a language. This test evaluates the\nproportion of stop words to the total word count in the dataset, in essence, scrutinizing the frequency of stop\nword usage. The core objective is to highlight the prevalent stop words based on their usage frequency, which can\nbe instrumental in cleaning the data from noise and improving ML model performance.
\n\n
Test Mechanism
\n\n
The StopWords test initiates on receiving an input of a 'VMDataset' object. Absence of such an object will trigger\nan error. The methodology involves inspection of the text column of the VMDataset to create a 'corpus' (a\ncollection of written texts). Leveraging the Natural Language Toolkit's (NLTK) stop word repository, the test\nscreens the corpus for any stop words and documents their frequency. It further calculates the percentage usage of\neach stop word compared to the total word count in the corpus. This percentage is evaluated against a predefined\n'min_percent_threshold'. If this threshold is breached, the test returns a failed output. Top prevailing stop words\nalong with their usage percentages are returned, facilitated by a bar chart visualization of these stop words and\ntheir frequency.
\n\n
Signs of High Risk
\n\n
\n
A percentage of any stop words exceeding the predefined 'min_percent_threshold'.
\n
High frequency of stop words in the dataset which may adversely affect the application's analytical performance\ndue to noise creation.
\n
\n\n
Strengths
\n\n
\n
The ability to scrutinize and quantify the usage of stop words.
\n
Provides insights into potential noise in the text data due to stop words.
\n
Directly aids in enhancing model training efficiency.
\n
Includes a bar chart visualization feature to easily interpret and action upon the stop words frequency\ninformation.
\n
\n\n
Limitations
\n\n
\n
The test only supports English stop words, making it less effective with datasets of other languages.
\n
The 'min_percent_threshold' parameter may require fine-tuning for different datasets, impacting the overall\neffectiveness of the test.
\n
Contextual use of the stop words within the dataset is not considered, potentially overlooking their significance\nin certain contexts.
\n
The test focuses specifically on the frequency of stop words, not providing direct measures of model performance\nor predictive accuracy.
Conducts comprehensive textual analysis on a dataset using NLTK to evaluate various parameters and generate\nvisualizations.
\n\n
Purpose
\n\n
The TextDescription test aims to conduct a thorough textual analysis of a dataset using the NLTK (Natural Language\nToolkit) library. It evaluates various metrics such as total words, total sentences, average sentence length, total\nparagraphs, total unique words, most common words, total punctuations, and lexical diversity. The goal is to\nunderstand the nature of the text and anticipate challenges machine learning models might face in text processing,\nlanguage understanding, or summarization tasks.
\n\n
Test Mechanism
\n\n
The test works by:
\n\n
\n
Parsing the dataset and tokenizing the text into words, sentences, and paragraphs using NLTK.
\n
Removing stopwords and unwanted tokens.
\n
Calculating parameters like total words, total sentences, average sentence length, total paragraphs, total unique\nwords, total punctuations, and lexical diversity.
\n
Generating scatter plots to visualize correlations between various metrics (e.g., Total Words vs Total Sentences).
\n
\n\n
Signs of High Risk
\n\n
\n
Anomalies or increased complexity in lexical diversity.
\n
Longer sentences and paragraphs.
\n
High uniqueness of words.
\n
Large number of unwanted tokens.
\n
Missing or erroneous visualizations.
\n
\n\n
Strengths
\n\n
\n
Essential for pre-processing text data in machine learning models.
\n
Provides a comprehensive breakdown of text data, aiding in understanding its complexity.
\n
Generates visualizations to help comprehend text structure and complexity.
\n
\n\n
Limitations
\n\n
\n
Highly dependent on the NLTK library, limiting the test to supported languages.
\n
Limited customization for removing undesirable tokens and stop words.
\n
Does not consider semantic or grammatical complexities.
\n
Assumes well-structured documents, which may result in inaccuracies with poorly formatted text.
Assesses the toxicity of text data within a dataset to visualize the distribution of toxicity scores.
\n\n
Purpose
\n\n
The Toxicity test aims to evaluate the level of toxic content present in a text dataset by leveraging a pre-trained\ntoxicity model. It helps in identifying potentially harmful or offensive language that may negatively impact users\nor stakeholders.
\n\n
Test Mechanism
\n\n
This test uses a pre-trained toxicity evaluation model and applies it to each text entry in the specified column of\na dataset\u2019s dataframe. The procedure involves:
\n\n
\n
Loading a pre-trained toxicity model.
\n
Extracting the text from the specified column in the dataset.
\n
Computing toxicity scores for each text entry.
\n
Generating a KDE (Kernel Density Estimate) plot to visualize the distribution of these toxicity scores.
\n
\n\n
Signs of High Risk
\n\n
\n
High concentration of high toxicity scores in the KDE plot.
\n
A significant proportion of text entries with toxicity scores above a predefined threshold.
\n
Wide distribution of toxicity scores, indicating inconsistency in content quality.
\n
\n\n
Strengths
\n\n
\n
Provides a visual representation of toxicity distribution, making it easier to identify outliers.
\n
Uses a robust pre-trained model for toxicity evaluation.
\n
Can process large text datasets efficiently.
\n
\n\n
Limitations
\n\n
\n
Depends on the accuracy and bias of the pre-trained toxicity model.
\n
Does not provide context-specific insights, which may be necessary for nuanced understanding.
\n
May not capture all forms of subtle or indirect toxic language.
Assesses the quality of machine-generated text using BERTScore metrics and visualizes results through histograms\nand bar charts, alongside compiling a comprehensive table of descriptive statistics.
\n\n
Purpose
\n\n
This function is designed to assess the quality of text generated by machine learning models using BERTScore\nmetrics. BERTScore evaluates text generation models' performance by calculating precision, recall, and F1 score\nbased on BERT contextual embeddings.
\n\n
Test Mechanism
\n\n
The function starts by extracting the true and predicted values from the provided dataset and model. It then\ninitializes the BERTScore evaluator. For each pair of true and predicted texts, the function calculates the\nBERTScore metrics and compiles them into a dataframe. Histograms and bar charts are generated for each BERTScore\nmetric (Precision, Recall, and F1 Score) to visualize their distribution. Additionally, a table of descriptive\nstatistics (mean, median, standard deviation, minimum, and maximum) is compiled for each metric, providing a\ncomprehensive summary of the model's performance. The test uses the evaluation_model param to specify the\nhuggingface model to use for evaluation. microsoft/deberta-xlarge-mnli is the best-performing model but is\nvery large and may be slow without a GPU. microsoft/deberta-large-mnli is a smaller model that is faster to\nrun and distilbert-base-uncased is much lighter and can run on a CPU but is less accurate.
\n\n
Signs of High Risk
\n\n
\n
Consistently low scores across BERTScore metrics could indicate poor quality in the generated text, suggesting\nthat the model fails to capture the essential content of the reference texts.
\n
Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
\n
Low recall scores may indicate that important information from the reference text is being omitted.
\n
An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the\nmodel's ability to balance informativeness and conciseness.
\n
\n\n
Strengths
\n\n
\n
Provides a multifaceted evaluation of text quality through different BERTScore metrics, offering a detailed view\nof model performance.
\n
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the\nscores.
\n
Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
\n
\n\n
Limitations
\n\n
\n
BERTScore relies on the contextual embeddings from BERT models, which may not fully capture all nuances of text\nsimilarity.
\n
The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
\n
While useful for comparison, BERTScore metrics alone do not provide a complete assessment of a model's\nperformance and should be supplemented with other metrics and qualitative analysis.
Evaluates the quality of machine-generated text using BLEU metrics and visualizes the results through histograms\nand bar charts, alongside compiling a comprehensive table of descriptive statistics for BLEU scores.
\n\n
Purpose
\n\n
This function is designed to assess the quality of text generated by machine learning models using the BLEU metric.\nBLEU, which stands for Bilingual Evaluation Understudy, is a metric used to evaluate the overlap of n-grams between\nthe machine-generated text and reference texts. This evaluation is crucial for tasks such as text summarization,\nmachine translation, and text generation, where the goal is to produce text that accurately reflects the content\nand meaning of human-crafted references.
\n\n
Test Mechanism
\n\n
The function starts by extracting the true and predicted values from the provided dataset and model. It then\ninitializes the BLEU evaluator. For each pair of true and predicted texts, the function calculates the BLEU scores\nand compiles them into a dataframe. Histograms and bar charts are generated for the BLEU scores to visualize their\ndistribution. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and\nmaximum) is compiled for the BLEU scores, providing a comprehensive summary of the model's performance.
\n\n
Signs of High Risk
\n\n
\n
Consistently low BLEU scores could indicate poor quality in the generated text, suggesting that the model fails\nto capture the essential content of the reference texts.
\n
Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
\n
Low recall scores may indicate that important information from the reference text is being omitted.
\n
An imbalanced performance between precision and recall, reflected by a low BLEU score, could signal issues in the\nmodel's ability to balance informativeness and conciseness.
\n
\n\n
Strengths
\n\n
\n
Provides a straightforward and widely-used evaluation of text quality through BLEU scores.
\n
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the\nscores.
\n
Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
\n
\n\n
Limitations
\n\n
\n
BLEU metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or\ngrammatical quality of the text.
\n
The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
\n
While useful for comparison, BLEU scores alone do not provide a complete assessment of a model's performance and\nshould be supplemented with other metrics and qualitative analysis.
Assesses the performance of clustering models by comparing the distribution of cluster sizes in model predictions\nwith the actual data.
\n\n
Purpose
\n\n
The Cluster Size Distribution test aims to assess the performance of clustering models by comparing the\ndistribution of cluster sizes in the model's predictions with the actual data. This comparison helps determine if\nthe clustering model's output aligns well with the true cluster distribution, providing insights into the model's\naccuracy and performance.
\n\n
Test Mechanism
\n\n
The test mechanism involves the following steps:
\n\n
\n
Run the clustering model on the provided dataset to obtain predictions.
\n
Convert both the actual and predicted outputs into pandas dataframes.
\n
Use pandas built-in functions to derive the cluster size distributions from these dataframes.
\n
Construct two histograms: one for the actual cluster size distribution and one for the predicted distribution.
\n
Plot the histograms side-by-side for visual comparison.
\n
\n\n
Signs of High Risk
\n\n
\n
Discrepancies between the actual cluster size distribution and the predicted cluster size distribution.
\n
Irregular distribution of data across clusters in the predicted outcomes.
\n
High number of outlier clusters suggesting the model struggles to correctly group data.
\n
\n\n
Strengths
\n\n
\n
Provides a visual and intuitive way to compare the clustering model's performance against actual data.
\n
Effectively reveals where the model may be over- or underestimating cluster sizes.
\n
Versatile as it works well with any clustering model.
\n
\n\n
Limitations
\n\n
\n
Assumes that the actual cluster distribution is optimal, which may not always be the case.
\n
Relies heavily on visual comparison, which could be subjective and may not offer a precise numerical measure of\nperformance.
\n
May not fully capture other important aspects of clustering, such as cluster density, distances between clusters,\nand the shape of clusters.
Evaluates a Natural Language Generation model's ability to generate contextually relevant and factually correct\ntext, visualizing the results through histograms and bar charts, alongside compiling a comprehensive table of\ndescriptive statistics for contextual recall scores.
\n\n
Purpose
\n\n
The Contextual Recall metric is used to evaluate the ability of a natural language generation (NLG) model to\ngenerate text that appropriately reflects the given context or prompt. It measures the model's capability to\nremember and reproduce the main context in its resulting output. This metric is critical in natural language\nprocessing tasks, as the coherency and contextuality of the generated text are essential.
\n\n
Test Mechanism
\n\n
The function starts by extracting the true and predicted values from the provided dataset and model. It then\ntokenizes the reference and candidate texts into discernible words or tokens using NLTK. The token overlap between\nthe reference and candidate texts is identified, and the Contextual Recall score is computed by dividing the number\nof overlapping tokens by the total number of tokens in the reference text. Scores are calculated for each test\ndataset instance, resulting in an array of scores. These scores are visualized using a histogram and a bar chart to\nshow score variations across different rows. Additionally, a table of descriptive statistics (mean, median,\nstandard deviation, minimum, and maximum) is compiled for the contextual recall scores, providing a comprehensive\nsummary of the model's performance.
\n\n
Signs of High Risk
\n\n
\n
Low contextual recall scores could indicate that the model is not effectively reflecting the original context in\nits output, leading to incoherent or contextually misaligned text.
\n
A consistent trend of low recall scores could suggest underperformance of the model.
\n
\n\n
Strengths
\n\n
\n
Provides a quantifiable measure of a model's adherence to the context and factual elements of the generated\nnarrative.
\n
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of\ncontextual recall scores.
\n
Descriptive statistics offer a concise summary of the model's performance in generating contextually relevant\ntexts.
\n
\n\n
Limitations
\n\n
\n
The focus on word overlap could result in high scores for texts that use many common words, even when these texts\nlack coherence or meaningful context.
\n
This metric does not consider the order of words, which could lead to overestimated scores for scrambled outputs.
\n
Models that effectively use infrequent words might be undervalued, as these words might not overlap as often.
Evaluates the discriminatory power of each individual feature within a binary classification model by calculating\nthe Area Under the Curve (AUC) for each feature separately.
\n\n
Purpose
\n\n
The central objective of this metric is to quantify how well each feature on its own can differentiate between the\ntwo classes in a binary classification problem. It serves as a univariate analysis tool that can help in\npre-modeling feature selection or post-modeling interpretation.
\n\n
Test Mechanism
\n\n
For each feature, the metric treats the feature values as raw scores to compute the AUC against the actual binary\noutcomes. It provides an AUC value for each feature, offering a simple yet powerful indication of each feature's\nunivariate classification strength.
\n\n
Signs of High Risk
\n\n
\n
A feature with a low AUC score may not be contributing significantly to the differentiation between the two\nclasses, which could be a concern if it is expected to be predictive.
\n
Conversely, a surprisingly high AUC for a feature not believed to be informative may suggest data leakage or\nother issues with the data.
\n
\n\n
Strengths
\n\n
\n
By isolating each feature, it highlights the individual contribution of features to the classification task\nwithout the influence of other variables.
\n
Useful for both initial feature evaluation and for providing insights into the model's reliance on individual\nfeatures after model training.
\n
\n\n
Limitations
\n\n
\n
Does not reflect the combined effects of features or any interaction between them, which can be critical in\ncertain models.
\n
The AUC values are calculated without considering the model's use of the features, which could lead to different\ninterpretations of feature importance when considering the model holistically.
\n
This metric is applicable only to binary classification tasks and cannot be directly extended to multiclass\nclassification or regression without modifications.
Assesses the quality of machine-generated translations by comparing them to human-produced references using the\nMETEOR score, which evaluates precision, recall, and word order.
\n\n
Purpose
\n\n
The METEOR (Metric for Evaluation of Translation with Explicit ORdering) score is designed to evaluate the quality\nof machine translations by comparing them against reference translations. It emphasizes both the accuracy and\nfluency of translations, incorporating precision, recall, and word order into its assessment.
\n\n
Test Mechanism
\n\n
The function starts by extracting the true and predicted values from the provided dataset and model. The METEOR\nscore is computed for each pair of machine-generated translation (prediction) and its corresponding human-produced\nreference. This is done by considering unigram matches between the translations, including matches based on surface\nforms, stemmed forms, and synonyms. The score is a combination of unigram precision and recall, adjusted for word\norder through a fragmentation penalty. Scores are compiled into a dataframe, and histograms and bar charts are\ngenerated to visualize the distribution of METEOR scores. Additionally, a table of descriptive statistics (mean,\nmedian, standard deviation, minimum, and maximum) is compiled for the METEOR scores, providing a comprehensive\nsummary of the model's performance.
\n\n
Signs of High Risk
\n\n
\n
Lower METEOR scores can indicate a lack of alignment between the machine-generated translations and their\nhuman-produced references, highlighting potential deficiencies in both the accuracy and fluency of translations.
\n
Significant discrepancies in word order or an excessive fragmentation penalty could signal issues with how the\ntranslation model processes and reconstructs sentence structures, potentially compromising the natural flow of\ntranslated text.
\n
Persistent underperformance across a variety of text types or linguistic contexts might suggest a broader\ninability of the model to adapt to the nuances of different languages or dialects, pointing towards gaps in its\ntraining or inherent limitations.
\n
\n\n
Strengths
\n\n
\n
Incorporates a balanced consideration of precision and recall, weighted towards recall to reflect the importance\nof content coverage in translations.
\n
Directly accounts for word order, offering a nuanced evaluation of translation fluency beyond simple lexical\nmatching.
\n
Adapts to various forms of lexical similarity, including synonyms and stemmed forms, allowing for flexible\nmatching.
\n
\n\n
Limitations
\n\n
\n
While comprehensive, the complexity of METEOR's calculation can make it computationally intensive, especially for\nlarge datasets.
\n
The use of external resources for synonym and stemming matching may introduce variability based on the resources'\nquality and relevance to the specific translation task.
Compare metadata of different models and generate a summary table with the results.
\n\n
Purpose: The purpose of this function is to compare the metadata of different models, including information about their architecture, framework, framework version, and programming language.
\n\n
Test Mechanism: The function retrieves the metadata for each model using get_model_info, renames columns according to a predefined set of labels, and compiles this information into a summary table.
\n\n
Signs of High Risk:
\n\n
\n
Inconsistent or missing metadata across models can indicate potential issues in model documentation or management.
\n
Significant differences in framework versions or programming languages might pose challenges in model integration and deployment.
\n
\n\n
Strengths:
\n\n
\n
Provides a clear comparison of essential model metadata.
\n
Standardizes metadata labels for easier interpretation and comparison.
\n
Helps identify potential compatibility or consistency issues across models.
\n
\n\n
Limitations:
\n\n
\n
Assumes that the get_model_info function returns all necessary metadata fields.
\n
Relies on the correctness and completeness of the metadata provided by each model.
\n
Does not include detailed parameter information, focusing instead on high-level metadata.
Assesses normality and behavior of residuals in regression models through visualization and statistical tests.
\n\n
Purpose
\n\n
The Model Prediction Residuals test aims to visualize the residuals of model predictions and assess their normality\nusing the Kolmogorov-Smirnov (KS) test. It helps to identify potential issues related to model assumptions and\neffectiveness.
\n\n
Test Mechanism
\n\n
The function calculates residuals and generates\ntwo figures: one for the time series of residuals and one for the histogram of residuals.\nIt also calculates the KS test for normality and summarizes the results in a table.
\n\n
Signs of High Risk
\n\n
\n
Residuals are not normally distributed, indicating potential issues with model assumptions.
\n
High skewness or kurtosis in the residuals, which may suggest model misspecification.
\n
\n\n
Strengths
\n\n
\n
Provides clear visualizations of residuals over time and their distribution.
\n
Includes statistical tests to assess the normality of residuals.
\n
Helps in identifying potential model misspecifications and assumption violations.
\n
\n\n
Limitations
\n\n
\n
Assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access the pandas\nDataFrame.
\n
Only generates plots for datasets with a datetime index, resulting in errors for other types of indices.
Assesses the sentiment and potential biases in text generated by NLP models by computing and visualizing regard\nscores.
\n\n
Purpose
\n\n
The RegardScore test aims to evaluate the levels of regard (positive, negative, neutral, or other) in texts\ngenerated by NLP models. It helps in understanding the sentiment and bias present in the generated content.
\n\n
Test Mechanism
\n\n
This test extracts the true and predicted values from the provided dataset and model. It then computes the regard\nscores for each text instance using a preloaded regard evaluation tool. The scores are compiled into dataframes,\nand visualizations such as histograms and bar charts are generated to display the distribution of regard scores.\nAdditionally, descriptive statistics (mean, median, standard deviation, minimum, and maximum) are calculated for\nthe regard scores, providing a comprehensive overview of the model's performance.
\n\n
Signs of High Risk
\n\n
\n
Noticeable skewness in the histogram, especially when comparing the predicted regard scores with the target\nregard scores, can indicate biases or inconsistencies in the model.
\n
Lack of neutral scores in the model's predictions, despite a balanced distribution in the target data, might\nsignal an issue.
\n
\n\n
Strengths
\n\n
\n
Provides a clear evaluation of regard levels in generated texts, aiding in ensuring content appropriateness.
\n
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of\nregard scores.
\n
Descriptive statistics offer a concise summary of the model's performance in generating texts with balanced\nsentiments.
\n
\n\n
Limitations
\n\n
\n
The accuracy of the regard scores is contingent upon the underlying regard tool.
\n
The scores provide a broad overview but do not specify which portions or tokens of the text are responsible for\nhigh regard.
\n
Supplementary, in-depth analysis might be needed for granular insights.
Evaluates regression model performance using residual distribution and actual vs. predicted plots.
\n\n
Purpose
\n\n
The RegressionResidualsPlot metric aims to evaluate the performance of regression models. By generating and\nanalyzing two plots \u2013 a distribution of residuals and a scatter plot of actual versus predicted values \u2013 this tool\nhelps to visually appraise how well the model predicts and the nature of errors it makes.
\n\n
Test Mechanism
\n\n
The process begins by extracting the true output values (y_true) and the model's predicted values (y_pred).\nResiduals are computed by subtracting predicted from true values. These residuals are then visualized using a\nhistogram to display their distribution. Additionally, a scatter plot is derived to compare true values against\npredicted values, together with a \"Perfect Fit\" line, which represents an ideal match (predicted values equal\nactual values), facilitating the assessment of the model's predictive accuracy.
\n\n
Signs of High Risk
\n\n
\n
Residuals showing a non-normal distribution, especially those with frequent extreme values.
\n
Significant deviations of predicted values from actual values in the scatter plot.
\n
Sparse density of data points near the \"Perfect Fit\" line in the scatter plot, indicating poor prediction\naccuracy.
\n
Visible patterns or trends in the residuals plot, suggesting the model's failure to capture the underlying data\nstructure adequately.
\n
\n\n
Strengths
\n\n
\n
Provides a direct, visually intuitive assessment of a regression model\u2019s accuracy and handling of data.
\n
Visual plots can highlight issues of underfitting or overfitting.
\n
Can reveal systematic deviations or trends that purely numerical metrics might miss.
\n
Applicable across various regression model types.
\n
\n\n
Limitations
\n\n
\n
Relies on visual interpretation, which can be subjective and less precise than numerical evaluations.
\n
May be difficult to interpret in cases with multi-dimensional outputs due to the plots\u2019 two-dimensional nature.
\n
Overlapping data points in the residuals plot can complicate interpretation efforts.
\n
Does not summarize model performance into a single quantifiable metric, which might be needed for comparative or\nsummary analyses.
Assesses the quality of machine-generated text using ROUGE metrics and visualizes the results to provide\ncomprehensive performance insights.
\n\n
Purpose
\n\n
The ROUGE Score test is designed to evaluate the quality of text generated by machine learning models using various\nROUGE metrics. ROUGE, which stands for Recall-Oriented Understudy for Gisting Evaluation, measures the overlap of\nn-grams, word sequences, and word pairs between machine-generated text and reference texts. This evaluation is\ncrucial for tasks like text summarization, machine translation, and text generation, where the goal is to produce\ntext that accurately reflects the content and meaning of human-crafted references.
\n\n
Test Mechanism
\n\n
The test extracts the true and predicted values from the provided dataset and model. It initializes the ROUGE\nevaluator with the specified metric (e.g., ROUGE-1). For each pair of true and predicted texts, it calculates the\nROUGE scores and compiles them into a dataframe. Histograms and bar charts are generated for each ROUGE metric\n(Precision, Recall, and F1 Score) to visualize their distribution. Additionally, a table of descriptive statistics\n(mean, median, standard deviation, minimum, and maximum) is compiled for each metric, providing a comprehensive\nsummary of the model's performance.
\n\n
Signs of High Risk
\n\n
\n
Consistently low scores across ROUGE metrics could indicate poor quality in the generated text, suggesting that\nthe model fails to capture the essential content of the reference texts.
\n
Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
\n
Low recall scores may indicate that important information from the reference text is being omitted.
\n
An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the\nmodel's ability to balance informativeness and conciseness.
\n
\n\n
Strengths
\n\n
\n
Provides a multifaceted evaluation of text quality through different ROUGE metrics, offering a detailed view of\nmodel performance.
\n
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the\nscores.
\n
Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
\n
\n\n
Limitations
\n\n
\n
ROUGE metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or\ngrammatical quality of the text.
\n
The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
\n
While useful for comparison, ROUGE scores alone do not provide a complete assessment of a model's performance and\nshould be supplemented with other metrics and qualitative analysis.
Assesses predictive accuracy and uncertainty in time series models, highlighting breaches beyond confidence\nintervals.
\n\n
Purpose
\n\n
The purpose of the Time Series Prediction with Confidence Intervals (CI) test is to visualize the actual versus\npredicted values for time series data, including confidence intervals, and to compute and report the number of\nbreaches beyond these intervals. This helps in evaluating the reliability and accuracy of the model's predictions.
\n\n
Test Mechanism
\n\n
The function performs the following steps:
\n\n
\n
Calculates the standard deviation of prediction errors.
\n
Determines the confidence intervals using a specified confidence level, typically 95%.
\n
Counts the number of actual values that fall outside the confidence intervals, referred to as breaches.
\n
Generates a plot visualizing the actual values, predicted values, and confidence intervals.
\n
Returns a DataFrame summarizing the breach information, including the total breaches, upper breaches, and lower\nbreaches.
\n
\n\n
Signs of High Risk
\n\n
\n
A high number of breaches indicates that the model's predictions are not reliable within the specified confidence\nlevel.
\n
Significant deviations between actual and predicted values may highlight model inadequacies or issues with data\nquality.
\n
\n\n
Strengths
\n\n
\n
Provides a visual representation of prediction accuracy and the uncertainty around predictions.
\n
Includes a statistical measure of prediction reliability through confidence intervals.
\n
Computes and reports breaches, offering a quantitative assessment of prediction performance.
\n
\n\n
Limitations
\n\n
\n
Assumes that the dataset is provided as a DataFrameDataset object with a datetime index.
\n
Requires that dataset.y_pred(model) returns the predicted values for the model.
\n
The calculation of confidence intervals assumes normally distributed errors, which may not hold for all datasets.
Evaluates the R-Squared values of regression models over specified time segments in time series data to assess\nsegment-wise model performance.
\n\n
Purpose
\n\n
The TimeSeriesR2SquareBySegments test aims to evaluate the R-Squared values for several regression models across\ndifferent segments of time series data. This helps in determining how well the models explain the variability in\nthe data within each specific time segment.
\n\n
Test Mechanism
\n\n
\n
Provides a visual representation of model performance across different time segments.
\n
Allows for identification of segments where the model performs poorly.
\n
Calculating the R-Squared values for each segment.
\n
Generating a bar chart to visually represent the R-Squared values across different models and segments.
\n
\n\n
Signs of High Risk
\n\n
\n
Significantly low R-Squared values for certain time segments, indicating poor model performance in those periods.
\n
Large variability in R-Squared values across different segments for the same model, suggesting inconsistent\nperformance.
\n
\n\n
Strengths
\n\n
\n
Provides a visual representation of how well models perform over different time periods.
\n
Helps identify time segments where models may need improvement or retraining.
\n
Facilitates comparison between multiple models in a straightforward manner.
\n
\n\n
Limitations
\n\n
\n
Assumes datasets are provided as DataFrameDataset objects with the attributes y, y_pred, and\nfeature_columns.
\n
Requires that dataset.y_pred(model) returns predicted values for the model.
\n
Assumes that both y_true and y_pred are pandas Series with datetime indices, which may not always be the case.
\n
May not account for more nuanced temporal dependencies within the segments.
Evaluates the token disparity between reference and generated texts, visualizing the results through histograms and\nbar charts, alongside compiling a comprehensive table of descriptive statistics for token counts.
\n\n
Purpose
\n\n
The Token Disparity test aims to assess the difference in the number of tokens between reference texts and texts\ngenerated by the model. Understanding token disparity is essential for evaluating how well the generated content\nmatches the expected length and richness of the reference texts.
\n\n
Test Mechanism
\n\n
The test extracts true and predicted values from the dataset and model. It computes the number of tokens in each\nreference and generated text. The results are visualized using histograms and bar charts to display the\ndistribution of token counts. Additionally, a table of descriptive statistics, including the mean, median, standard\ndeviation, minimum, and maximum token counts, is compiled to provide a detailed summary of token usage.
\n\n
Signs of High Risk
\n\n
\n
Significant disparity in token counts between reference and generated texts could indicate issues with text\ngeneration quality, such as verbosity or lack of detail.
\n
Consistently low token counts in generated texts compared to references might suggest that the model is producing\nincomplete or overly concise outputs.
\n
\n\n
Strengths
\n\n
\n
Provides a simple yet effective evaluation of text length and token usage.
\n
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of\ntoken counts.
\n
Descriptive statistics offer a concise summary of the model's performance in generating texts of appropriate\nlength.
\n
\n\n
Limitations
\n\n
\n
Token counts alone do not provide a complete assessment of text quality and should be supplemented with other\nmetrics and qualitative analysis.
Assesses the toxicity levels of texts generated by NLP models to identify and mitigate harmful or offensive content.
\n\n
Purpose
\n\n
The ToxicityScore metric is designed to evaluate the toxicity levels of texts generated by models. This is crucial\nfor identifying and mitigating harmful or offensive content in machine-generated texts.
\n\n
Test Mechanism
\n\n
The function starts by extracting the input, true, and predicted values from the provided dataset and model. The\ntoxicity score is computed for each text using a preloaded toxicity evaluation tool. The scores are compiled into\ndataframes, and histograms and bar charts are generated to visualize the distribution of toxicity scores.\nAdditionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is\ncompiled for the toxicity scores, providing a comprehensive summary of the model's performance.
\n\n
Signs of High Risk
\n\n
\n
Drastic spikes in toxicity scores indicate potentially toxic content within the associated text segment.
\n
Persistent high toxicity scores across multiple texts may suggest systemic issues in the model's text generation\nprocess.
\n
\n\n
Strengths
\n\n
\n
Provides a clear evaluation of toxicity levels in generated texts, helping to ensure content safety and\nappropriateness.
\n
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of\ntoxicity scores.
\n
Descriptive statistics offer a concise summary of the model's performance in generating non-toxic texts.
\n
\n\n
Limitations
\n\n
\n
The accuracy of the toxicity scores is contingent upon the underlying toxicity tool.
\n
The scores provide a broad overview but do not specify which portions or tokens of the text are responsible for\nhigh toxicity.
\n
Supplementary, in-depth analysis might be needed for granular insights.
Evaluates clustering model performance by measuring mutual information between true and predicted labels, adjusting\nfor chance.
\n\n
Purpose
\n\n
The purpose of this metric (Adjusted Mutual Information) is to evaluate the performance of a machine learning\nmodel, more specifically, a clustering model. It measures the mutual information between the true labels and the\nones predicted by the model, adjusting for chance.
\n\n
Test Mechanism
\n\n
The Adjusted Mutual Information (AMI) uses sklearn's adjusted_mutual_info_score function. This function\ncalculates the mutual information between the true labels and the ones predicted while correcting for the chance\ncorrelation expected due to random label assignments. This test requires the model, the training dataset, and the\ntest dataset as inputs.
\n\n
Signs of High Risk
\n\n
\n
Low Adjusted Mutual Information Score: This score ranges between 0 and 1. A low score (closer to 0) can indicate\npoor model performance as the predicted labels do not align well with the true labels.
\n
In case of high-dimensional data, if the algorithm shows high scores, this could also be a potential risk as AMI\nmay not perform reliably.
\n
\n\n
Strengths
\n\n
\n
The AMI metric takes into account the randomness of the predicted labels, which makes it more robust than the\nsimple Mutual Information.
\n
The scale of AMI is not dependent on the sizes of the clustering, allowing for comparability between different\ndatasets or models.
\n
Good for comparing the output of clustering algorithms where the number of clusters is not known a priori.
\n
\n\n
Limitations
\n\n
\n
Adjusted Mutual Information does not take into account the continuous nature of some data. As a result, it may\nnot be the best choice for regression or other continuous types of tasks.
\n
AMI has the drawback of being biased towards clusterings with a higher number of clusters.
\n
In comparison to other metrics, AMI can be slower to compute.
\n
The interpretability of the score can be complex as it depends on the understanding of information theory\nconcepts.
Measures the similarity between two data clusters using the Adjusted Rand Index (ARI) metric in clustering machine\nlearning models.
\n\n
Purpose
\n\n
The Adjusted Rand Index (ARI) metric is intended to measure the similarity between two data clusters. This metric\nis specifically used for clustering machine learning models to quantify how well the model is clustering and\nproducing data groups. It involves comparing the model's produced clusters against the actual (true) clusters found\nin the dataset.
\n\n
Test Mechanism
\n\n
The Adjusted Rand Index (ARI) is calculated using the adjusted_rand_score method from the sklearn.metrics\nmodule in Python. The test requires inputs including the model itself and the model's training and test datasets.\nThe model's computed clusters and the true clusters are compared, and the similarities are measured to compute the\nARI.
\n\n
Signs of High Risk
\n\n
\n
If the ARI is close to zero, it signifies that the model's cluster assignments are random and do not match the\nactual dataset clusters, indicating a high risk.
\n
An ARI of less than zero indicates that the model's clustering performance is worse than random.
\n
\n\n
Strengths
\n\n
\n
ARI is normalized and provides a consistent metric between -1 and +1, irrespective of raw cluster sizes or\ndataset size variations.
\n
It does not require a ground truth for computation, making it ideal for unsupervised learning model evaluations.
\n
It penalizes for false positives and false negatives, providing a robust measure of clustering quality.
\n
\n\n
Limitations
\n\n
\n
In real-world situations, true clustering is often unknown, which can hinder the practical application of the ARI.
\n
The ARI requires all individual data instances to be independent, which may not always hold true.
\n
It may be difficult to interpret the implications of an ARI score without context or a benchmark, as it is\nheavily dependent on the characteristics of the dataset used.
Evaluates the calibration of probability estimates by comparing predicted probabilities against observed\nfrequencies.
\n\n
Purpose
\n\n
The Calibration Curve test assesses how well a model's predicted probabilities align with actual\nobserved frequencies. This is crucial for applications requiring accurate probability estimates,\nsuch as risk assessment, decision-making systems, and cost-sensitive applications where probability\ncalibration directly impacts business decisions.
\n\n
Test Mechanism
\n\n
The test uses sklearn's calibration_curve function to:
\n\n\n
Sort predictions into bins based on predicted probabilities
\n
Calculate the mean predicted probability in each bin
\n
Compare against the observed frequency of positive cases
\n
Plot the results against the perfect calibration line (y=x)\nThe resulting curve shows how well the predicted probabilities match empirical probabilities.
\n\n\n
Signs of High Risk
\n\n
\n
Significant deviation from the perfect calibration line
\n
Systematic overconfidence (predictions too close to 0 or 1)
\n
Systematic underconfidence (predictions clustered around 0.5)
\n
Empty or sparse bins indicating poor probability coverage
\n
Sharp discontinuities in the calibration curve
\n
Different calibration patterns across different probability ranges
\n
Consistent over/under estimation in critical probability regions
\n
Large confidence intervals in certain probability ranges
\n
\n\n
Strengths
\n\n
\n
Visual and intuitive interpretation of probability quality
\n
Identifies systematic biases in probability estimates
\n
Supports probability threshold selection
\n
Helps understand model confidence patterns
\n
Applicable across different classification models
\n
Enables comparison between different models
\n
Guides potential need for recalibration
\n
Critical for risk-sensitive applications
\n
\n\n
Limitations
\n\n
\n
Sensitive to the number of bins chosen
\n
Requires sufficient samples in each bin for reliable estimates
\n
May mask local calibration issues within bins
\n
Does not account for feature-dependent calibration issues
\n
Limited to binary classification problems
\n
Cannot detect all forms of miscalibration
\n
Assumes bin boundaries are appropriate for the problem
Evaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy,\nand ROC AUC scores.
\n\n
Purpose
\n\n
The Classifier Performance test is designed to evaluate the performance of Machine Learning classification models.\nIt accomplishes this by computing precision, recall, F1-Score, and accuracy, as well as the ROC AUC (Receiver\noperating characteristic - Area under the curve) scores, thereby providing a comprehensive analytic view of the\nmodels' performance. The test is adaptable, handling binary and multiclass models equally effectively.
\n\n
Test Mechanism
\n\n
The test produces a report that includes precision, recall, F1-Score, and accuracy, by leveraging the\nclassification_report from scikit-learn's metrics module. For multiclass models, macro and weighted averages for\nthese scores are also calculated. Additionally, the ROC AUC scores are calculated and included in the report using\nthe multiclass_roc_auc_score function. The outcome of the test (report format) differs based on whether the model\nis binary or multiclass.
\n\n
Signs of High Risk
\n\n
\n
Low values for precision, recall, F1-Score, accuracy, and ROC AUC, indicating poor performance.
\n
Imbalance in precision and recall scores.
\n
A low ROC AUC score, especially scores close to 0.5 or lower, suggesting a failing model.
\n
\n\n
Strengths
\n\n
\n
Versatile, capable of assessing both binary and multiclass models.
\n
Utilizes a variety of commonly employed performance metrics, offering a comprehensive view of model performance.
\n
The use of ROC-AUC as a metric is beneficial for evaluating unbalanced datasets.
\n
\n\n
Limitations
\n\n
\n
Assumes correctly identified labels for binary classification models.
\n
Specifically designed for classification models and not suitable for regression models.
\n
May provide limited insights if the test dataset does not represent real-world scenarios adequately.
Analyzes and visualizes different threshold optimization methods for binary classification models.
\n\n
Purpose
\n\n
The Classifier Threshold Optimization test identifies optimal decision thresholds using various\nmethods to balance different performance metrics. This helps adapt the model's decision boundary\nto specific business requirements, such as minimizing false positives in fraud detection or\nachieving target recall in medical diagnosis.
\n\n
Test Mechanism
\n\n
The test implements multiple threshold optimization methods:
F1-score optimization (balancing precision and recall)
\n
Precision-Recall equality point
\n
Target recall achievement
\n
Naive (0.5) threshold\nFor each method, it computes ROC and PR curves, identifies optimal points, and provides\ncomprehensive performance metrics at each threshold.
\n\n\n
Signs of High Risk
\n\n
\n
Large discrepancies between different optimization methods
\n
Optimal thresholds far from the default 0.5
\n
Poor performance metrics across all thresholds
\n
Significant gap between achieved and target recall
\n
Unstable thresholds across different methods
\n
Extreme trade-offs between precision and recall
\n
Threshold optimization showing minimal impact
\n
Business metrics not improving with optimization
\n
\n\n
Strengths
\n\n
\n
Multiple optimization strategies for different needs
\n
Visual and numerical results for comparison
\n
Support for business-driven optimization (target recall)
\n
Comprehensive performance metrics at each threshold
\n
Integration with ROC and PR curves
\n
Handles class imbalance through various metrics
\n
Enables informed threshold selection
\n
Supports cost-sensitive decision making
\n
\n\n
Limitations
\n\n
\n
Assumes cost of false positives/negatives are known
\n
May need adjustment for highly imbalanced datasets
\n
Threshold might not be stable across different samples
\n
Cannot handle multi-class problems directly
\n
Optimization methods may conflict with business needs
\n
Requires sufficient validation data
\n
May not capture temporal changes in optimal threshold
\n
Single threshold may not be optimal for all subgroups
\n
\n\n
Arguments:
\n\n
\n
dataset: VMDataset containing features and target
\n
model: VMModel containing predictions
\n
methods: List of methods to compare (default: ['youden', 'f1', 'precision_recall'])
\n
target_recall: Target recall value if using 'target_recall' method
\n
\n\n
Returns:
\n\n
\n
Dictionary containing:\n - table: DataFrame comparing different threshold optimization methods\n (using weighted averages for precision, recall, and f1)\n - figure: Plotly figure showing ROC and PR curves with optimal thresholds
Measures the intra-cluster similarity of a clustering model using cosine similarity.
\n\n
Purpose
\n\n
The purpose of this metric is to measure how similar the data points within each cluster of a clustering model are.\nThis is done using cosine similarity, which compares the multi-dimensional direction (but not magnitude) of data\nvectors. From a Model Risk Management perspective, this metric is used to quantitatively validate that clusters\nformed by a model have high intra-cluster similarity.
\n\n
Test Mechanism
\n\n
This test works by first extracting the true and predicted clusters of the model's training data. Then, it computes\nthe centroid (average data point) of each cluster. Next, it calculates the cosine similarity between each data\npoint within a cluster and its respective centroid. Finally, it outputs the mean cosine similarity of each cluster,\nhighlighting how similar, on average, data points in a cluster are to the cluster's centroid.
\n\n
Signs of High Risk
\n\n
\n
Low mean cosine similarity for one or more clusters: If the mean cosine similarity is low, the data points within\nthe respective cluster have high variance in their directions. This can be indicative of poor clustering,\nsuggesting that the model might not be suitably separating the data into distinct patterns.
\n
High disparity between mean cosine similarity values across clusters: If there's a significant difference in mean\ncosine similarity across different clusters, this could indicate imbalance in how the model forms clusters.
\n
\n\n
Strengths
\n\n
\n
Cosine similarity operates in a multi-dimensional space, making it effective for measuring similarity in high\ndimensional datasets, typical for many machine learning problems.
\n
It provides an agnostic view of the cluster performance by only considering the direction (and not the magnitude)\nof each vector.
\n
This metric is not dependent on the scale of the variables, making it equally effective on different scales.
\n
\n\n
Limitations
\n\n
\n
Cosine similarity does not consider magnitudes (i.e. lengths) of vectors, only their direction. This means it may\noverlook instances where clusters have been adequately separated in terms of magnitude.
\n
This method summarily assumes that centroids represent the average behavior of data points in each cluster. This\nmight not always be true, especially in clusters with high amounts of variance or non-spherical shapes.
\n
It primarily works with continuous variables and is not suitable for binary or categorical variables.
\n
Lastly, although rare, perfect perpendicular vectors (cosine similarity = 0) could be within the same cluster,\nwhich may give an inaccurate representation of a 'bad' cluster due to low cosine similarity score.
Evaluates the performance of clustering machine learning models using multiple established metrics.
\n\n
Purpose
\n\n
The ClusterPerformanceMetrics test is used to assess the performance and validity of clustering machine learning\nmodels. It evaluates homogeneity, completeness, V measure score, the Adjusted Rand Index, the Adjusted Mutual\nInformation, and the Fowlkes-Mallows score of the model. These metrics provide a holistic understanding of the\nmodel's ability to accurately form clusters of the given dataset.
\n\n
Test Mechanism
\n\n
The ClusterPerformanceMetrics test runs a clustering ML model over a given dataset and then calculates six\nmetrics using the Scikit-learn metrics computation functions: Homogeneity Score, Completeness Score, V Measure,\nAdjusted Rand Index (ARI), Adjusted Mutual Information (AMI), and Fowlkes-Mallows Score. It then returns the result\nas a summary, presenting the metric values for both training and testing datasets.
\n\n
Signs of High Risk
\n\n
\n
Low Homogeneity Score: Indicates that the clusters formed contain a variety of classes, resulting in less pure\nclusters.
\n
Low Completeness Score: Suggests that class instances are scattered across multiple clusters rather than being\ngathered in a single cluster.
\n
Low V Measure: Reports a low overall clustering performance.
\n
ARI close to 0 or Negative: Implies that clustering results are random or disagree with the true labels.
\n
AMI close to 0: Means that clustering labels are random compared with the true labels.
\n
Low Fowlkes-Mallows score: Signifies less precise and poor clustering performance in terms of precision and\nrecall.
\n
\n\n
Strengths
\n\n
\n
Provides a comprehensive view of clustering model performance by examining multiple clustering metrics.
\n
Uses established and widely accepted metrics from scikit-learn, providing reliability in the results.
\n
Able to provide performance metrics for both training and testing datasets.
\n
Clearly defined and human-readable descriptions of each score make it easy to understand what each score\nrepresents.
\n
\n\n
Limitations
\n\n
\n
Only applies to clustering models; not suitable for other types of machine learning models.
\n
Does not test for overfitting or underfitting in the clustering model.
\n
All the scores rely on ground truth labels, the absence or inaccuracy of which can lead to misleading results.
\n
Does not consider aspects like computational efficiency of the model or its capability to handle high dimensional\ndata.
Evaluates a clustering model's capacity to categorize instances from a single class into the same cluster.
\n\n
Purpose
\n\n
The Completeness Score metric is used to assess the performance of clustering models. It measures the extent to\nwhich all the data points that are members of a given class are elements of the same cluster. The aim is to\ndetermine the capability of the model to categorize all instances from a single class into the same cluster.
\n\n
Test Mechanism
\n\n
This test takes three inputs, a model and its associated training and testing datasets. It invokes the\ncompleteness_score function from the sklearn library on the labels predicted by the model. High scores indicate\nthat data points from the same class generally appear in the same cluster, while low scores suggest the opposite.
\n\n
Signs of High Risk
\n\n
\n
Low completeness score: This suggests that the model struggles to group instances from the same class into one\ncluster, indicating poor clustering performance.
\n
\n\n
Strengths
\n\n
\n
The Completeness Score provides an effective method for assessing the performance of a clustering model,\nspecifically its ability to group class instances together.
\n
This test metric conveniently relies on the capabilities provided by the sklearn library, ensuring consistent and\nreliable test results.
\n
\n\n
Limitations
\n\n
\n
This metric only evaluates a specific aspect of clustering, meaning it may not provide a holistic or complete\nview of the model's performance.
\n
It cannot assess the effectiveness of the model in differentiating between separate classes, as it is solely\nfocused on how well data points from the same class are grouped.
\n
The Completeness Score only applies to clustering models; it cannot be used for other types of machine learning\nmodels.
Evaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix\nheatmap.
\n\n
Purpose
\n\n
The Confusion Matrix tester is designed to assess the performance of a classification Machine Learning model. This\nperformance is evaluated based on how well the model is able to correctly classify True Positives, True Negatives,\nFalse Positives, and False Negatives - fundamental aspects of model accuracy.
\n\n
Test Mechanism
\n\n
The mechanism used involves taking the predicted results (y_test_predict) from the classification model and\ncomparing them against the actual values (y_test_true). A confusion matrix is built using the unique labels\nextracted from y_test_true, employing scikit-learn's metrics. The matrix is then visually rendered with the help\nof Plotly's create_annotated_heatmap function. A heatmap is created which provides a two-dimensional graphical\nrepresentation of the model's performance, showcasing distributions of True Positives (TP), True Negatives (TN),\nFalse Positives (FP), and False Negatives (FN).
\n\n
Signs of High Risk
\n\n
\n
High numbers of False Positives (FP) and False Negatives (FN), depicting that the model is not effectively\nclassifying the values.
\n
Low numbers of True Positives (TP) and True Negatives (TN), implying that the model is struggling with correctly\nidentifying class labels.
\n
\n\n
Strengths
\n\n
\n
It provides a simplified yet comprehensive visual snapshot of the classification model's predictive performance.
\n
It distinctly brings out True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives\n(FN), thus making it easier to focus on potential areas of improvement.
\n
The matrix is beneficial in dealing with multi-class classification problems as it can provide a simple view of\ncomplex model performances.
\n
It aids in understanding the different types of errors that the model could potentially make, as it provides\nin-depth insights into Type-I and Type-II errors.
\n
\n\n
Limitations
\n\n
\n
In cases of unbalanced classes, the effectiveness of the confusion matrix might be lessened. It may wrongly\ninterpret the accuracy of a model that is essentially just predicting the majority class.
\n
It does not provide a single unified statistic that could evaluate the overall performance of the model.\nDifferent aspects of the model's performance are evaluated separately instead.
\n
It mainly serves as a descriptive tool and does not offer the capability for statistical hypothesis testing.
\n
Risks of misinterpretation exist because the matrix doesn't directly provide precision, recall, or F1-score data.\nThese metrics have to be computed separately.
Compute feature importance scores for a given model and generate a summary table\nwith the top important features.
\n\n
Purpose
\n\n
The Feature Importance Comparison test is designed to compare the feature importance scores for different models\nwhen applied to various datasets. By doing so, it aims to identify the most impactful features and assess the\nconsistency of feature importance across models.
\n\n
Test Mechanism
\n\n
This test works by iterating through each dataset-model pair and calculating permutation feature importance (PFI)\nscores. It then generates a summary table containing the top num_features important features for each model. The\nprocess involves:
\n\n
\n
Extracting features and target data from each dataset.
\n
Computing PFI scores using sklearn.inspection.permutation_importance.
\n
Sorting and selecting the top features based on their importance scores.
\n
Compiling these features into a summary table for comparison.
\n
\n\n
Signs of High Risk
\n\n
\n
Key features expected to be important are ranked low, indicating potential issues with model training or data\nquality.
\n
High variance in feature importance scores across different models, suggesting instability in feature selection.
\n
\n\n
Strengths
\n\n
\n
Provides a clear comparison of the most important features for each model.
\n
Uses permutation importance, which is a model-agnostic method and can be applied to any estimator.
\n
\n\n
Limitations
\n\n
\n
Assumes that the dataset is provided as a DataFrameDataset object with x_df and y_df methods to access\nfeature and target data.
\n
Requires that model.model is compatible with sklearn.inspection.permutation_importance.
\n
The function's output is dependent on the number of features specified by num_features, which defaults to 3 but\ncan be adjusted.
Evaluates the similarity between predicted and actual cluster assignments in a model using the Fowlkes-Mallows\nscore.
\n\n
Purpose
\n\n
The FowlkesMallowsScore is a performance metric used to validate clustering algorithms within machine learning\nmodels. The score intends to evaluate the matching grade between two clusters. It measures the similarity between\nthe predicted and actual cluster assignments, thus gauging the accuracy of the model's clustering capability.
\n\n
Test Mechanism
\n\n
The FowlkesMallowsScore method applies the fowlkes_mallows_score function from the sklearn library to evaluate\nthe model's accuracy in clustering different types of data. The test fetches the datasets from the model's training\nand testing datasets as inputs then compares the resulting clusters against the previously known clusters to obtain\na score. A high score indicates a better clustering performance by the model.
\n\n
Signs of High Risk
\n\n
\n
A low Fowlkes-Mallows score (near zero): This indicates that the model's clustering capability is poor and the\nalgorithm isn't properly grouping data.
\n
Inconsistently low scores across different datasets: This may indicate that the model's clustering performance is\nnot robust and the model may fail when applied to unseen data.
\n
\n\n
Strengths
\n\n
\n
The Fowlkes-Mallows score is a simple and effective method for evaluating the performance of clustering\nalgorithms.
\n
This metric takes into account both precision and recall in its calculation, therefore providing a balanced and\ncomprehensive measure of model performance.
\n
The Fowlkes-Mallows score is non-biased meaning it treats False Positives and False Negatives equally.
\n
\n\n
Limitations
\n\n
\n
As a pairwise-based method, this score can be computationally intensive for large datasets and can become\nunfeasible as the size of the dataset increases.
\n
The Fowlkes-Mallows score works best with balanced distribution of samples across clusters. If this condition is\nnot met, the score can be skewed.
\n
It does not handle mismatching numbers of clusters between the true and predicted labels. As such, it may return\nmisleading results if the predicted labels suggest a different number of clusters than what is in the true labels.
Assesses clustering homogeneity by comparing true and predicted labels, scoring from 0 (heterogeneous) to 1\n(homogeneous).
\n\n
Purpose
\n\n
The Homogeneity Score encapsulated in this performance test is used to measure the homogeneity of the clusters\nformed by a machine learning model. In simple terms, a clustering result satisfies homogeneity if all of its\nclusters contain only points which are members of a single class.
\n\n
Test Mechanism
\n\n
This test uses the homogeneity_score function from the sklearn.metrics library to compare the ground truth\nclass labels of the training and testing sets with the labels predicted by the given model. The returned score is a\nmetric of the clustering accuracy, and ranges from 0.0 to 1.0, with 1.0 denoting the highest possible degree of\nhomogeneity.
\n\n
Signs of High Risk
\n\n
\n
A score close to 0: This denotes that clusters are highly heterogenous and points within the same cluster might\nnot belong to the same class.
\n
A significantly lower score for testing data compared to the score for training data: This can indicate\noverfitting, where the model has learned to perfectly match the training data but fails to perform well on unseen\ndata.
\n
\n\n
Strengths
\n\n
\n
It provides a simple quantitative measure of the degree to which clusters contain points from only one class.
\n
Useful for validating clustering solutions where the ground truth \u2014 class membership of points \u2014 is known.
\n
It's agnostic to the absolute labels, and cares only that the points within the same cluster have the same class\nlabel.
\n
\n\n
Limitations
\n\n
\n
The Homogeneity Score is not useful for clustering solutions where the ground truth labels are not known.
\n
It doesn\u2019t work well with differently sized clusters since it gives predominance to larger clusters.
\n
The score does not address the actual number of clusters formed, or the evenness of cluster sizes. It only checks\nthe homogeneity within the given clusters created by the model.
Performs exhaustive grid search over specified parameter ranges to find optimal model configurations\nacross different metrics and decision thresholds.
\n\n
Purpose
\n\n
The Hyperparameter Tuning test systematically explores the model's parameter space to identify optimal\nconfigurations. It supports multiple optimization metrics and decision thresholds, providing a comprehensive\nview of how different parameter combinations affect various aspects of model performance.
\n\n
Test Mechanism
\n\n
The test uses scikit-learn's GridSearchCV to perform cross-validation for each parameter combination.\nFor each specified threshold and optimization metric, it creates a scoring dictionary with\nthreshold-adjusted metrics, performs grid search with cross-validation, records best parameters and\ncorresponding scores, and combines results into a comparative table. This process is repeated for each\noptimization metric to provide a comprehensive view of model performance under different configurations.
\n\n
Signs of High Risk
\n\n
\n
Large performance variations across different parameter combinations
\n
Significant discrepancies between different optimization metrics
\n
Best parameters at the edges of the parameter grid
\n
Unstable performance across different thresholds
\n
Overly complex model configurations (risk of overfitting)
\n
Very different optimal parameters for different metrics
\n
Cross-validation scores showing high variance
\n
Extreme parameter values in best configurations
\n
\n\n
Strengths
\n\n
\n
Comprehensive exploration of parameter space
\n
Supports multiple optimization metrics
\n
Allows threshold optimization
\n
Provides comparative view across different configurations
\n
Uses cross-validation for robust evaluation
\n
Helps understand trade-offs between different metrics
\n
Enables systematic parameter selection
\n
Supports both classification and clustering tasks
\n
\n\n
Limitations
\n\n
\n
Computationally expensive for large parameter grids
\n
May not find global optimum (limited to grid points)
\n
Cannot handle dependencies between parameters
\n
Memory intensive for large datasets
\n
Limited to scikit-learn compatible models
\n
Cross-validation splits may not preserve time series structure
\n
Grid search may miss optimal values between grid points
\n
Resource intensive for high-dimensional parameter spaces
Optimizes the number of clusters in K-means models using Elbow and Silhouette methods.
\n\n
Purpose
\n\n
This metric is used to optimize the number of clusters used in K-means clustering models. It intends to measure and\nevaluate the optimal number of clusters by leveraging two methodologies, namely the Elbow method and the Silhouette\nmethod. This is crucial as an inappropriate number of clusters can either overly simplify or overcomplicate the\nstructure of the data, thereby undermining the effectiveness of the model.
\n\n
Test Mechanism
\n\n
The test mechanism involves iterating over a predefined range of cluster numbers and applying both the Elbow method\nand the Silhouette method. The Elbow method computes the sum of the minimum euclidean distances between data points\nand their respective cluster centers (distortion). This value decreases as the number of clusters increases; the\noptimal number is typically at the 'elbow' point where the decrease in distortion becomes less pronounced.\nMeanwhile, the Silhouette method calculates the average silhouette score for each data point in the dataset,\nproviding a measure of how similar each item is to its own cluster compared to other clusters. The optimal number\nof clusters under this method is the one that maximizes the average silhouette score. The results of both methods\nare plotted for visual inspection.
\n\n
Signs of High Risk
\n\n
\n
A high distortion value or a low silhouette average score for the optimal number of clusters.
\n
No clear 'elbow' point or plateau observed in the distortion plot, or a uniformly low silhouette average score\nacross different numbers of clusters, suggesting the data is not amenable to clustering.
\n
An optimal cluster number that is unreasonably high or low, suggestive of overfitting or underfitting,\nrespectively.
\n
\n\n
Strengths
\n\n
\n
Provides both a visual and quantitative method to determine the optimal number of clusters.
\n
Leverages two different methods (Elbow and Silhouette), thereby affording robustness and versatility in assessing\nthe data's clusterability.
\n
Facilitates improved model performance by allowing for an informed selection of the number of clusters.
\n
\n\n
Limitations
\n\n
\n
Assumes that a suitable number of clusters exists in the data, which may not always be true, especially for\ncomplex or noisy data.
\n
Both methods may fail to provide definitive answers when the data lacks clear cluster structures.
\n
Might not be straightforward to determine the 'elbow' point or maximize the silhouette average score, especially\nin larger and complicated datasets.
\n
Assumes spherical clusters (due to using the Euclidean distance in the Elbow method), which might not align with\nthe actual structure of the data.
Checks if the model's prediction accuracy meets or surpasses a specified threshold.
\n\n
Purpose
\n\n
The Minimum Accuracy test\u2019s objective is to verify whether the model's prediction accuracy on a specific dataset\nmeets or surpasses a predetermined minimum threshold. Accuracy, which is simply the ratio of correct predictions to\ntotal predictions, is a key metric for evaluating the model's performance. Considering binary as well as multiclass\nclassifications, accurate labeling becomes indispensable.
\n\n
Test Mechanism
\n\n
The test mechanism involves contrasting the model's accuracy score with a preset minimum threshold value, with the\ndefault being 0.7. The accuracy score is computed utilizing sklearn\u2019s accuracy_score method, where the true\nlabels y_true and predicted labels class_pred are compared. If the accuracy score is above the threshold, the\ntest receives a passing mark. The test returns the result along with the accuracy score and threshold used for the\ntest.
\n\n
Signs of High Risk
\n\n
\n
Model fails to achieve or surpass the predefined score threshold.
\n
Persistent scores below the threshold, indicating a high risk of inaccurate predictions.
\n
\n\n
Strengths
\n\n
\n
Simplicity, presenting a straightforward measure of holistic model performance across all classes.
\n
Particularly advantageous when classes are balanced.
\n
Versatile, as it can be implemented on both binary and multiclass classification tasks.
\n
\n\n
Limitations
\n\n
\n
Misleading accuracy scores when classes in the dataset are highly imbalanced.
\n
Favoritism towards the majority class, giving an inaccurate perception of model performance.
\n
Inability to measure the model's precision, recall, or capacity to manage false positives or false negatives.
\n
Focused on overall correctness and may not be sufficient for all types of model analytics.
Assesses if the model's F1 score on the validation set meets a predefined minimum threshold, ensuring balanced\nperformance between precision and recall.
\n\n
Purpose
\n\n
The main objective of this test is to ensure that the F1 score, a balanced measure of precision and recall, of the\nmodel meets or surpasses a predefined threshold on the validation dataset. The F1 score is highly useful for\ngauging model performance in classification tasks, especially in cases where the distribution of positive and\nnegative classes is skewed.
\n\n
Test Mechanism
\n\n
The F1 score for the validation dataset is computed through scikit-learn's metrics in Python. The scoring mechanism\ndiffers based on the classification problem: for multi-class problems, macro averaging is used, and for binary\nclassification, the built-in f1_score calculation is used. The obtained F1 score is then assessed against the\npredefined minimum F1 score that is expected from the model.
\n\n
Signs of High Risk
\n\n
\n
If a model returns an F1 score that is less than the established threshold, it is regarded as high risk.
\n
A low F1 score might suggest that the model is not finding an optimal balance between precision and recall,\nfailing to effectively identify positive classes while minimizing false positives.
\n
\n\n
Strengths
\n\n
\n
Provides a balanced measure of a model's performance by accounting for both false positives and false negatives.
\n
Particularly advantageous in scenarios with imbalanced class distribution, where accuracy can be misleading.
\n
Flexibility in setting the threshold value allows tailored minimum acceptable performance standards.
\n
\n\n
Limitations
\n\n
\n
May not be suitable for all types of models and machine learning tasks.
\n
The F1 score assumes an equal cost for false positives and false negatives, which may not be true in some\nreal-world scenarios.
\n
Practitioners might need to rely on other metrics such as precision, recall, or the ROC-AUC score that align more\nclosely with specific requirements.
Validates model by checking if the ROC AUC score meets or surpasses a specified threshold.
\n\n
Purpose
\n\n
The Minimum ROC AUC Score test is used to determine the model's performance by ensuring that the Receiver Operating\nCharacteristic Area Under the Curve (ROC AUC) score on the validation dataset meets or exceeds a predefined\nthreshold. The ROC AUC score indicates how well the model can distinguish between different classes, making it a\ncrucial measure in binary and multiclass classification tasks.
\n\n
Test Mechanism
\n\n
This test implementation calculates the multiclass ROC AUC score on the true target values and the model's\npredictions. The test converts the multi-class target variables into binary format using LabelBinarizer before\ncomputing the score. If this ROC AUC score is higher than the predefined threshold (defaulted to 0.5), the test\npasses; otherwise, it fails. The results, including the ROC AUC score, the threshold, and whether the test passed\nor failed, are then stored in a ThresholdTestResult object.
\n\n
Signs of High Risk
\n\n
\n
A high risk or failure in the model's performance as related to this metric would be represented by a low ROC AUC\nscore, specifically any score lower than the predefined minimum threshold. This suggests that the model is\nstruggling to distinguish between different classes effectively.
\n
\n\n
Strengths
\n\n
\n
The test considers both the true positive rate and false positive rate, providing a comprehensive performance\nmeasure.
\n
ROC AUC score is threshold-independent meaning it measures the model's quality across various classification\nthresholds.
\n
Works robustly with binary as well as multi-class classification problems.
\n
\n\n
Limitations
\n\n
\n
ROC AUC may not be useful if the class distribution is highly imbalanced; it could perform well in terms of AUC\nbut still fail to predict the minority class.
\n
The test does not provide insight into what specific aspects of the model are causing poor performance if the ROC\nAUC score is unsatisfactory.
\n
The use of macro average for multiclass ROC AUC score implies equal weightage to each class, which might not be\nappropriate if the classes are imbalanced.
Extracts and displays model parameters in a structured format for transparency and reproducibility.
\n\n
Purpose
\n\n
The Model Parameters test is designed to provide transparency into model configuration and ensure\nreproducibility of machine learning models. It accomplishes this by extracting and presenting all\nrelevant parameters that define the model's behavior, making it easier to audit, validate, and\nreproduce model training.
\n\n
Test Mechanism
\n\n
The test leverages scikit-learn's API convention of get_params() to extract model parameters. It\nproduces a structured DataFrame containing parameter names and their corresponding values. For models\nthat follow scikit-learn's API (including XGBoost, RandomForest, and other estimators), all\nparameters are automatically extracted and displayed.
\n\n
Signs of High Risk
\n\n
\n
Missing crucial parameters that should be explicitly set
\n
Extreme parameter values that could indicate overfitting (e.g., unlimited tree depth)
\n
Inconsistent parameters across different versions of the same model type
\n
Parameter combinations known to cause instability or poor performance
\n
Default values used for critical parameters that should be tuned
\n
\n\n
Strengths
\n\n
\n
Universal compatibility with scikit-learn API-compliant models
\n
Ensures transparency in model configuration
\n
Facilitates model reproducibility and version control
\n
Enables systematic parameter auditing
\n
Supports both classification and regression models
\n
Helps identify potential configuration issues
\n
\n\n
Limitations
\n\n
\n
Only works with models implementing scikit-learn's get_params() method
\n
Cannot capture dynamic parameters set during model training
\n
Does not validate parameter values for model-specific appropriateness
\n
Parameter meanings and impacts may vary across different model types
\n
Cannot detect indirect parameter interactions or their effects on model performance
Evaluates and compares the performance of multiple Machine Learning models using various metrics like accuracy,\nprecision, recall, and F1 score.
\n\n
Purpose
\n\n
The Models Performance Comparison test aims to evaluate and compare the performance of various Machine Learning\nmodels using test data. It employs multiple metrics such as accuracy, precision, recall, and the F1 score, among\nothers, to assess model performance and assist in selecting the most effective model for the designated task.
\n\n
Test Mechanism
\n\n
The test employs Scikit-learn\u2019s performance metrics to evaluate each model's performance for both binary and\nmulticlass classification tasks. To compare performances, the test runs each model against the test dataset, then\nproduces a comprehensive classification report. This report includes metrics such as accuracy, precision, recall,\nand the F1 score. Based on whether the task at hand is binary or multiclass classification, it calculates metrics\nfor all the classes and their weighted averages, macro averages, and per-class metrics. The test will be skipped if\nno models are supplied.
\n\n
Signs of High Risk
\n\n
\n
Low scores in accuracy, precision, recall, and F1 metrics indicate a potentially high risk.
\n
A low area under the Receiver Operating Characteristic (ROC) curve (roc_auc score) is another possible indicator\nof high risk.
\n
If the metrics scores are significantly lower than alternative models, this might suggest a high risk of failure.
\n
\n\n
Strengths
\n\n
\n
Provides a simple way to compare the performance of multiple models, accommodating both binary and multiclass\nclassification tasks.
\n
Offers a holistic view of model performance through a comprehensive report of key performance metrics.
\n
The inclusion of the ROC AUC score is advantageous, as this robust performance metric can effectively handle\nclass imbalance issues.
\n
\n\n
Limitations
\n\n
\n
May not be suitable for more complex performance evaluations that consider factors such as prediction speed,\ncomputational cost, or business-specific constraints.
\n
The test's reliability depends on the provided test dataset; hence, the selected models' performance could vary\nwith unseen data or changes in the data distribution.
\n
The ROC AUC score might not be as meaningful or easily interpretable for multilabel/multiclass tasks.
Assesses potential overfitting in a model's predictions, identifying regions where performance between training and\ntesting sets deviates significantly.
\n\n
Purpose
\n\n
The Overfit Diagnosis test aims to identify areas in a model's predictions where there is a significant difference\nin performance between the training and testing sets. This test helps to pinpoint specific regions or feature\nsegments where the model may be overfitting.
\n\n
Test Mechanism
\n\n
This test compares the model's performance on training versus test data, grouped by feature columns. It calculates\nthe difference between the training and test performance for each group and identifies regions where this\ndifference exceeds a specified threshold:
\n\n
\n
The test works for both classification and regression models.
\n
It defaults to using the AUC metric for classification models and the MSE metric for regression models.
\n
The threshold for identifying overfitting regions is set to 0.04 by default.
\n
The test calculates the performance metrics for each feature segment and plots regions where the performance gap\nexceeds the threshold.
\n
\n\n
Signs of High Risk
\n\n
\n
Significant gaps between training and test performance metrics for specific feature segments.
\n
Multiple regions with performance gaps exceeding the defined threshold.
\n
Higher than expected differences in predicted versus actual values in the test set compared to the training set.
\n
\n\n
Strengths
\n\n
\n
Identifies specific areas where overfitting occurs.
Assesses the significance of each feature in a model by evaluating the impact on model performance when feature\nvalues are randomly rearranged.
\n\n
Purpose
\n\n
The Permutation Feature Importance (PFI) metric aims to assess the importance of each feature used by the Machine\nLearning model. The significance is measured by evaluating the decrease in the model's performance when the\nfeature's values are randomly arranged.
\n\n
Test Mechanism
\n\n
PFI is calculated via the permutation_importance method from the sklearn.inspection module. This method\nshuffles the columns of the feature dataset and measures the impact on the model's performance. A significant\ndecrease in performance after permutating a feature's values deems the feature as important. On the other hand, if\nperformance remains the same, the feature is likely not important. The output of the PFI metric is a figure\nillustrating the importance of each feature.
\n\n
Signs of High Risk
\n\n
\n
The model heavily relies on a feature with highly variable or easily permutable values, indicating instability.
\n
A feature deemed unimportant by the model but expected to have a significant effect on the outcome based on\ndomain knowledge is not influencing the model's predictions.
\n
\n\n
Strengths
\n\n
\n
Provides insights into the importance of different features and may reveal underlying data structure.
\n
Can indicate overfitting if a particular feature or set of features overly impacts the model's predictions.
\n
Model-agnostic and can be used with any classifier that provides a measure of prediction accuracy before and\nafter feature permutation.
\n
\n\n
Limitations
\n\n
\n
Does not imply causality; it only presents the amount of information that a feature provides for the prediction\ntask.
\n
Does not account for interactions between features. If features are correlated, the permutation importance may\nallocate importance to one and not the other.
\n
Cannot interact with certain libraries like statsmodels, pytorch, catboost, etc., thus limiting its applicability.
Assesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across\ndifferent datasets.
\n\n
Purpose
\n\n
The Population Stability Index (PSI) serves as a quantitative assessment for evaluating the stability of a machine\nlearning model's output distributions when comparing two different datasets. Typically, these would be a\ndevelopment and a validation dataset or two datasets collected at different periods. The PSI provides a measurable\nindication of any significant shift in the model's performance over time or noticeable changes in the\ncharacteristics of the population the model is making predictions for.
\n\n
Test Mechanism
\n\n
The implementation of the PSI in this script involves calculating the PSI for each feature between the training and\ntest datasets. Data from both datasets is sorted and placed into either a predetermined number of bins or\nquantiles. The boundaries for these bins are initially determined based on the distribution of the training data.\nThe contents of each bin are calculated and their respective proportions determined. Subsequently, the PSI is\nderived for each bin through a logarithmic transformation of the ratio of the proportions of data for each feature\nin the training and test datasets. The PSI, along with the proportions of data in each bin for both datasets, are\ndisplayed in a summary table, a grouped bar chart, and a scatter plot.
\n\n
Signs of High Risk
\n\n
\n
A high PSI value is a clear indicator of high risk. Such a value suggests a significant shift in the model\npredictions or severe changes in the characteristics of the underlying population.
\n
This ultimately suggests that the model may not be performing as well as expected and that it may be less\nreliable for making future predictions.
\n
\n\n
Strengths
\n\n
\n
The PSI provides a quantitative measure of the stability of a model over time or across different samples, making\nit an invaluable tool for evaluating changes in a model's performance.
\n
It allows for direct comparisons across different features based on the PSI value.
\n
The calculation and interpretation of the PSI are straightforward, facilitating its use in model risk management.
\n
The use of visual aids such as tables and charts further simplifies the comprehension and interpretation of the\nPSI.
\n
\n\n
Limitations
\n\n
\n
The PSI test does not account for the interdependence between features: features that are dependent on one\nanother may show similar shifts in their distributions, which in turn may result in similar PSI values.
\n
The PSI test does not inherently provide insights into why there are differences in distributions or why the PSI\nvalues may have changed.
\n
The test may not handle features with significant outliers adequately.
\n
Additionally, the PSI test is performed on model predictions, not on the underlying data distributions which can\nlead to misinterpretations. Any changes in PSI could be due to shifts in the model (model drift), changes in the\nrelationships between features and the target variable (concept drift), or both. However, distinguishing between\nthese causes is non-trivial.
Evaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve.
\n\n
Purpose
\n\n
The Precision Recall Curve metric is intended to evaluate the trade-off between precision and recall in\nclassification models, particularly binary classification models. It assesses the model's capacity to produce\naccurate results (high precision), as well as its ability to capture a majority of all positive instances (high\nrecall).
\n\n
Test Mechanism
\n\n
The test extracts ground truth labels and prediction probabilities from the model's test dataset. It applies the\nprecision_recall_curve method from the sklearn metrics module to these extracted labels and predictions, which\ncomputes a precision-recall pair for each possible threshold. This calculation results in an array of precision and\nrecall scores that can be plotted against each other to form the Precision-Recall Curve. This curve is then\nvisually represented by using Plotly's scatter plot.
\n\n
Signs of High Risk
\n\n
\n
A lower area under the Precision-Recall Curve signifies high risk.
\n
This corresponds to a model yielding a high amount of false positives (low precision) and/or false negatives (low\nrecall).
\n
If the curve is closer to the bottom left of the plot, rather than being closer to the top right corner, it can\nbe a sign of high risk.
\n
\n\n
Strengths
\n\n
\n
This metric aptly represents the balance between precision (minimizing false positives) and recall (minimizing\nfalse negatives), which is especially critical in scenarios where both values are significant.
\n
Through the graphic representation, it enables an intuitive understanding of the model's performance across\ndifferent threshold levels.
\n
\n\n
Limitations
\n\n
\n
This metric is only applicable to binary classification models - it raises errors for multiclass classification\nmodels or Foundation models.
\n
It may not fully represent the overall accuracy of the model if the cost of false positives and false negatives\nare extremely different, or if the dataset is heavily imbalanced.
Evaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic\n(ROC) curve and calculating the Area Under Curve (AUC) score.
\n\n
Purpose
\n\n
The Receiver Operating Characteristic (ROC) curve is designed to evaluate the performance of binary classification\nmodels. This curve illustrates the balance between the True Positive Rate (TPR) and False Positive Rate (FPR)\nacross various threshold levels. In combination with the Area Under the Curve (AUC), the ROC curve aims to measure\nthe model's discrimination ability between the two defined classes in a binary classification problem (e.g.,\ndefault vs non-default). Ideally, a higher AUC score signifies superior model performance in accurately\ndistinguishing between the positive and negative classes.
\n\n
Test Mechanism
\n\n
First, this script selects the target model and datasets that require binary classification. It then calculates the\npredicted probabilities for the test set, and uses this data, along with the true outcomes, to generate and plot\nthe ROC curve. Additionally, it includes a line signifying randomness (AUC of 0.5). The AUC score for the model's\nROC curve is also computed, presenting a numerical estimation of the model's performance. If any Infinite values\nare detected in the ROC threshold, these are effectively eliminated. The resulting ROC curve, AUC score, and\nthresholds are consequently saved for future reference.
\n\n
Signs of High Risk
\n\n
\n
A high risk is potentially linked to the model's performance if the AUC score drops below or nears 0.5.
\n
Another warning sign would be the ROC curve lying closer to the line of randomness, indicating no discriminative\nability.
\n
For the model to be deemed competent at its classification tasks, it is crucial that the AUC score is\nsignificantly above 0.5.
\n
\n\n
Strengths
\n\n
\n
The ROC Curve offers an inclusive visual depiction of a model's discriminative power throughout all conceivable\nclassification thresholds, unlike other metrics that solely disclose model performance at one fixed threshold.
\n
Despite the proportions of the dataset, the AUC Score, which represents the entire ROC curve as a single data\npoint, continues to be consistent, proving to be the ideal choice for such situations.
\n
\n\n
Limitations
\n\n
\n
The primary limitation is that this test is exclusively structured for binary classification tasks, thus limiting\nits application towards other model types.
\n
Furthermore, its performance might be subpar with models that output probabilities highly skewed towards 0 or 1.
\n
At the extreme, the ROC curve could reflect high performance even when the majority of classifications are\nincorrect, provided that the model's ranking format is retained. This phenomenon is commonly termed the \"Class\nImbalance Problem\".
Assesses the performance and error distribution of a regression model using various error metrics.
\n\n
Purpose
\n\n
The purpose of the Regression Errors test is to measure the performance of a regression model by calculating\nseveral error metrics. This evaluation helps determine the model's accuracy and potential issues like overfitting\nor bias by analyzing differences in error metrics between the training and testing datasets.
\n\n
Test Mechanism
\n\n
The test computes the following error metrics:
\n\n
\n
Mean Absolute Error (MAE): Average of the absolute differences between true values and predicted values.
\n
Mean Squared Error (MSE): Average of the squared differences between true values and predicted values.
\n
Root Mean Squared Error (RMSE): Square root of the mean squared error.
\n
Mean Absolute Percentage Error (MAPE): Average of the absolute differences between true values and predicted\nvalues, divided by the true values, and expressed as a percentage.
\n
Mean Bias Deviation (MBD): Average bias between true values and predicted values.
\n
\n\n
These metrics are calculated separately for the training and testing datasets and compared to identify\ndiscrepancies.
\n\n
Signs of High Risk
\n\n
\n
High values for MAE, MSE, RMSE, or MAPE indicating poor model performance.
\n
Large differences in error metrics between the training and testing datasets, suggesting overfitting.
\n
Significant deviation of MBD from zero, indicating systematic bias in model predictions.
\n
\n\n
Strengths
\n\n
\n
Provides a comprehensive overview of model performance through multiple error metrics.
\n
Individual metrics offer specific insights, e.g., MAE for interpretability, MSE for emphasizing larger errors.
\n
RMSE is useful for being in the same unit as the target variable.
\n
MAPE allows the error to be expressed as a percentage.
\n
MBD detects systematic bias in model predictions.
\n
\n\n
Limitations
\n\n
\n
MAE and MSE are sensitive to outliers.
\n
RMSE heavily penalizes larger errors, which might not always be desirable.
\n
MAPE can be misleading when actual values are near zero.
\n
MBD may not be suitable if bias varies with the magnitude of actual values.
\n
These metrics may not capture all nuances of model performance and should be interpreted with domain-specific\ncontext.
Assesses multiple regression error metrics to compare model performance across different datasets, emphasizing\nsystematic overestimation or underestimation and large percentage errors.
\n\n
Purpose
\n\n
The purpose of this test is to compare regression errors for different models applied to various datasets. It aims\nto examine model performance using multiple error metrics, thereby identifying areas where models may be\nunderperforming or exhibiting bias.
\n\n
Test Mechanism
\n\n
The function iterates through each dataset-model pair and calculates various error metrics, including Mean Absolute\nError (MAE), Mean Squared Error (MSE), Mean Absolute Percentage Error (MAPE), and Mean Bias Deviation (MBD). The\nresults are summarized in a table, which provides a comprehensive view of each model's performance on the datasets.
\n\n
Signs of High Risk
\n\n
\n
High Mean Absolute Error (MAE) or Mean Squared Error (MSE), indicating poor model performance.
\n
High Mean Absolute Percentage Error (MAPE), suggesting large percentage errors, especially problematic if the\ntrue values are small.
\n
Mean Bias Deviation (MBD) significantly different from zero, indicating systematic overestimation or\nunderestimation by the model.
\n
\n\n
Strengths
\n\n
\n
Provides multiple error metrics to assess model performance from different perspectives.
\n
Includes a check to avoid division by zero when calculating MAPE.
\n
\n\n
Limitations
\n\n
\n
Assumes that the dataset is provided as a DataFrameDataset object with y, y_pred, and feature_columns\nattributes.
\n
Relies on the logger from validmind.logging to warn about zero values in y_true, which should be correctly\nimplemented and imported.
\n
Requires that dataset.y_pred(model) returns the predicted values for the model.
Evaluates the performance of a regression model using five different metrics: MAE, MSE, RMSE, MAPE, and MBD.
\n\n
Purpose
\n\n
The Regression Models Performance Comparison metric is used to measure the performance of regression models. It\ncalculates multiple evaluation metrics, including Mean Absolute Error (MAE), Mean Squared Error (MSE),\nRoot Mean Squared Error (RMSE), Mean Absolute Percentage Error (MAPE), and Mean Bias Deviation (MBD), thereby\nenabling a comprehensive view of model performance.
\n\n
Test Mechanism
\n\n
The test uses the sklearn library to calculate the MAE, MSE, RMSE, MAPE, and MBD. These calculations encapsulate both\nthe direction and the magnitude of error in predictions, thereby providing a multi-faceted view of model accuracy.
\n\n
Signs of High Risk
\n\n
\n
High values of MAE, MSE, RMSE, and MAPE, which indicate a high error rate and imply a larger departure of the\nmodel's predictions from the true values.
\n
A large value of MBD, which shows a consistent bias in the model\u2019s predictions.
\n
\n\n
Strengths
\n\n
\n
The metric evaluates models on five different metrics offering a comprehensive analysis of model performance.
\n
It is designed to handle regression tasks and can be seamlessly integrated with libraries like sklearn.
\n
\n\n
Limitations
\n\n
\n
The metric only evaluates regression models and does not evaluate classification models.
\n
The test assumes that the models have been trained and tested appropriately prior to evaluation. It does not\nhandle pre-processing, feature selection, or other stages in the model lifecycle.
Assesses the overall goodness-of-fit of a regression model by evaluating R-squared (R2) and Adjusted R-squared (Adj\nR2) scores to determine the model's explanatory power over the dependent variable.
\n\n
Purpose
\n\n
The purpose of the RegressionR2Square Metric test is to measure the overall goodness-of-fit of a regression model.\nSpecifically, this Python-based test evaluates the R-squared (R2) and Adjusted R-squared (Adj R2) scores, which are\nstatistical measures used to assess the strength of the relationship between the model's predictors and the\nresponse variable.
\n\n
Test Mechanism
\n\n
The test deploys the r2_score method from the Scikit-learn metrics module to measure the R2 score on both\ntraining and test sets. This score reflects the proportion of the variance in the dependent variable that is\npredictable from the independent variables. The test also calculates the Adjusted R2 score, which accounts for the\nnumber of predictors in the model to penalize model complexity and reduce overfitting. The Adjusted R2 score will\nbe smaller if unnecessary predictors are included in the model.
\n\n
Signs of High Risk
\n\n
\n
Low R2 or Adjusted R2 scores, suggesting that the model does not explain much variation in the dependent variable.
\n
Significant discrepancy between R2 scores on the training set and test set, indicating overfitting and poor\ngeneralization to unseen data.
\n
\n\n
Strengths
\n\n
\n
Widely-used measure in regression analysis, providing a sound general indication of model performance.
\n
Easy to interpret and understand, as it represents the proportion of the dependent variable's variance explained\nby the independent variables.
\n
Adjusted R2 score helps control overfitting by penalizing unnecessary predictors.
\n
\n\n
Limitations
\n\n
\n
Sensitive to the inclusion of unnecessary predictors even though Adjusted R2 penalizes complexity.
\n
Less reliable in cases of non-linear relationships or when the underlying assumptions of linear regression are\nviolated.
\n
Does not provide insight on whether the correct regression model was used or if key assumptions have been met.
Compares R-Squared and Adjusted R-Squared values for different regression models across multiple datasets to assess\nmodel performance and relevance of features.
\n\n
Purpose
\n\n
The Regression R2 Square Comparison test aims to compare the R-Squared and Adjusted R-Squared values for different\nregression models across various datasets. It helps in assessing how well each model explains the variability in\nthe dataset, and whether the models include irrelevant features.
\n\n
Test Mechanism
\n\n
This test operates by:
\n\n
\n
Iterating through each dataset-model pair.
\n
Calculating the R-Squared values to measure how much of the variability in the dataset is explained by the model.
\n
Calculating the Adjusted R-Squared values, which adjust the R-Squared based on the number of predictors in the\nmodel, making it more reliable when comparing models with different numbers of features.
\n
Generating a summary table containing these values for each combination of dataset and model.
\n
\n\n
Signs of High Risk
\n\n
\n
If the R-Squared values are significantly low, it indicates the model isn't explaining much of the variability in\nthe dataset.
\n
A significant difference between R-Squared and Adjusted R-Squared values might indicate that the model includes\nirrelevant features.
\n
\n\n
Strengths
\n\n
\n
Provides a quantitative measure of model performance in terms of variance explained.
\n
Adjusted R-Squared accounts for the number of predictors, making it a more reliable measure when comparing models\nwith different numbers of features.
\n
Useful for time-series forecasting and regression tasks.
\n
\n\n
Limitations
\n\n
\n
Assumes the dataset is provided as a DataFrameDataset object with y, y_pred, and feature_columns attributes.
\n
Relies on adj_r2_score from the statsmodels.statsutils module, which needs to be correctly implemented and\nimported.
\n
Requires that dataset.y_pred(model) returns the predicted values for the model.
Assesses the robustness of a machine learning model by evaluating performance decay under noisy conditions.
\n\n
Purpose
\n\n
The Robustness Diagnosis test aims to evaluate the resilience of a machine learning model when subjected to\nperturbations or noise in its input data. This is essential for understanding the model's ability to handle\nreal-world scenarios where data may be imperfect or corrupted.
\n\n
Test Mechanism
\n\n
This test introduces Gaussian noise to the numeric input features of the datasets at varying scales of standard\ndeviation. The performance of the model is then measured using a specified metric. The process includes:
\n\n
\n
Adding Gaussian noise to numerical input features based on scaling factors.
\n
Evaluating the model's performance on the perturbed data using metrics like AUC for classification tasks and MSE\nfor regression tasks.
\n
Aggregating and plotting the results to visualize performance decay relative to perturbation size.
\n
\n\n
Signs of High Risk
\n\n
\n
A significant drop in performance metrics with minimal noise.
\n
Performance decay values exceeding the specified threshold.
\n
Consistent failure to meet performance standards across multiple perturbation scales.
\n
\n\n
Strengths
\n\n
\n
Provides insights into the model's robustness against noisy or corrupted data.
\n
Utilizes a variety of performance metrics suitable for both classification and regression tasks.
\n
Visualization helps in understanding the extent of performance degradation.
\n
\n\n
Limitations
\n\n
\n
Gaussian noise might not adequately represent all types of real-world data perturbations.
\n
Performance thresholds are somewhat arbitrary and might need tuning.
\n
The test may not account for more complex or unstructured noise patterns that could affect model robustness.
Selects SHAP values for binary or multiclass classification.
\n\n
For regression models, returns the SHAP values directly as there are no classes.
\n\n
Arguments:
\n\n
\n
shap_values: The SHAP values returned by the SHAP explainer. For multiclass\nclassification, this will be a list where each element corresponds to a class.\nFor regression, this will be a single array of SHAP values.
\n
class_of_interest: The class index for which to retrieve SHAP values. If None\n(default), the function will assume binary classification and use class 1\nby default.
\n
\n\n
Returns:
\n\n
\n
The SHAP values for the specified class (classification) or for the regression\n output.
\n
\n\n
Raises:
\n\n
\n
ValueError: If class_of_interest is specified and is out of bounds for the\nnumber of classes.
Evaluates and visualizes global feature importance using SHAP values for model explanation and risk identification.
\n\n
Purpose
\n\n
The SHAP (SHapley Additive exPlanations) Global Importance metric aims to elucidate model outcomes by attributing\nthem to the contributing features. It assigns a quantifiable global importance to each feature via their respective\nabsolute Shapley values, thereby making it suitable for tasks like classification (both binary and multiclass).\nThis metric forms an essential part of model risk management.
\n\n
Test Mechanism
\n\n
The exam begins with the selection of a suitable explainer which aligns with the model's type. For tree-based\nmodels like XGBClassifier, RandomForestClassifier, CatBoostClassifier, TreeExplainer is used whereas for linear\nmodels like LogisticRegression, XGBRegressor, LinearRegression, it is the LinearExplainer. Once the explainer\ncalculates the Shapley values, these values are visualized using two specific graphical representations:
\n\n\n
Mean Importance Plot: This graph portrays the significance of individual features based on their absolute\nShapley values. It calculates the average of these absolute Shapley values across all instances to highlight the\nglobal importance of features.
\n
Summary Plot: This visual tool combines the feature importance with their effects. Every dot on this chart\nrepresents a Shapley value for a certain feature in a specific case. The vertical axis is denoted by the feature\nwhereas the horizontal one corresponds to the Shapley value. A color gradient indicates the value of the feature,\ngradually changing from low to high. Features are systematically organized in accordance with their importance.
\n\n\n
Signs of High Risk
\n\n
\n
Overemphasis on certain features in SHAP importance plots, thus hinting at the possibility of model overfitting
\n
Anomalies such as unexpected or illogical features showing high importance, which might suggest that the model's\ndecisions are rooted in incorrect or undesirable reasoning
\n
A SHAP summary plot filled with high variability or scattered data points, indicating a cause for concern
\n
\n\n
Strengths
\n\n
\n
SHAP does more than just illustrating global feature significance, it offers a detailed perspective on how\ndifferent features shape the model's decision-making logic for each instance.
\n
It provides clear insights into model behavior.
\n
\n\n
Limitations
\n\n
\n
High-dimensional data can convolute interpretations.
\n
Associating importance with tangible real-world impact still involves a certain degree of subjectivity.
Analyzes the alignment between credit scores and predicted probabilities.
\n\n
Purpose
\n\n
The Score-Probability Alignment test evaluates how well credit scores align with\npredicted default probabilities. This helps validate score scaling, identify potential\ncalibration issues, and ensure scores reflect risk appropriately.
\n\n
Test Mechanism
\n\n
The test:
\n\n\n
Groups scores into bins
\n
Calculates average predicted probability per bin
\n
Tests monotonicity of relationship
\n
Analyzes probability distribution within score bands
\n\n\n
Signs of High Risk
\n\n
\n
Non-monotonic relationship between scores and probabilities
\n
Large probability variations within score bands
\n
Unexpected probability jumps between adjacent bands
\n
Poor alignment with expected odds-to-score relationship
\n
Inconsistent probability patterns across score ranges
\n
Clustering of probabilities at extreme values
\n
Score bands with similar probability profiles
\n
Unstable probability estimates in key decision bands
\n
\n\n
Strengths
\n\n
\n
Direct validation of score-to-probability relationship
Calculates and visualizes Silhouette Score, assessing the degree of data point suitability to its cluster in ML\nmodels.
\n\n
Purpose
\n\n
This test calculates the Silhouette Score, which is a model performance metric used in clustering applications.\nPrimarily, the Silhouette Score evaluates how similar a data point is to its own cluster compared to other\nclusters. The metric ranges between -1 and 1, where a high value indicates that the object is well matched to its\nown cluster and poorly matched to neighboring clusters. Thus, the goal is to achieve a high Silhouette Score,\nimplying well-separated clusters.
\n\n
Test Mechanism
\n\n
The test first extracts the true and predicted labels from the model's training data. The test runs the Silhouette\nScore function, which takes as input the training dataset features and the predicted labels, subsequently\ncalculating the average score. This average Silhouette Score is printed for reference. The script then calculates\nthe silhouette coefficients for each data point, helping to form the Silhouette Plot. Each cluster is represented\nin this plot, with color distinguishing between different clusters. A red dashed line indicates the average\nSilhouette Score. The Silhouette Scores are also collected into a structured table, facilitating model performance\nanalysis and comparison.
\n\n
Signs of High Risk
\n\n
\n
A low Silhouette Score, potentially indicating that the clusters are not well separated and that data points may\nnot be fitting well to their respective clusters.
\n
A Silhouette Plot displaying overlapping clusters or the absence of clear distinctions between clusters visually\nalso suggests poor clustering performance.
\n
\n\n
Strengths
\n\n
\n
The Silhouette Score provides a clear and quantitative measure of how well data points have been grouped into\nclusters, offering insights into model performance.
\n
The Silhouette Plot provides an intuitive, graphical representation of the clustering mechanism, aiding visual\nassessments of model performance.
\n
It does not require ground truth labels, so it's useful when true cluster assignments are not known.
\n
\n\n
Limitations
\n\n
\n
The Silhouette Score may be susceptible to the influence of outliers, which could impact its accuracy and\nreliability.
\n
It assumes the clusters are convex and isotropic, which might not be the case with complex datasets.
\n
Due to the average nature of the Silhouette Score, the metric does not account for individual data point\nassignment nuances, so potentially relevant details may be omitted.
\n
Computationally expensive for large datasets, as it requires pairwise distance computations.
Tests if model performance degradation between training and test datasets exceeds a predefined threshold.
\n\n
Purpose
\n\n
The TrainingTestDegradation class serves as a test to verify that the degradation in performance between the\ntraining and test datasets does not exceed a predefined threshold. This test measures the model's ability to\ngeneralize from its training data to unseen test data, assessing key classification metrics such as accuracy,\nprecision, recall, and f1 score to verify the model's robustness and reliability.
\n\n
Test Mechanism
\n\n
The code applies several predefined metrics, including accuracy, precision, recall, and f1 scores, to the model's\npredictions for both the training and test datasets. It calculates the degradation as the difference between the\ntraining score and test score divided by the training score. The test is considered successful if the degradation\nfor each metric is less than the preset maximum threshold of 10%. The results are summarized in a table showing\neach metric's train score, test score, degradation percentage, and pass/fail status.
\n\n
Signs of High Risk
\n\n
\n
A degradation percentage that exceeds the maximum allowed threshold of 10% for any of the evaluated metrics.
\n
A high difference or gap between the metric scores on the training and the test datasets.
\n
The 'Pass/Fail' column displaying 'Fail' for any of the evaluated metrics.
\n
\n\n
Strengths
\n\n
\n
Provides a quantitative measure of the model's ability to generalize to unseen data, which is key for predicting\nits practical real-world performance.
\n
By evaluating multiple metrics, it takes into account different facets of model performance and enables a more\nholistic evaluation.
\n
The use of a variable predefined threshold allows the flexibility to adjust the acceptability criteria for\ndifferent scenarios.
\n
\n\n
Limitations
\n\n
\n
The test compares raw performance on training and test data but does not factor in the nature of the data. Areas\nwith less representation in the training set might still perform poorly on unseen data.
\n
It requires good coverage and balance in the test and training datasets to produce reliable results, which may\nnot always be available.
\n
The test is currently only designed for classification tasks.
Evaluates homogeneity and completeness of a clustering model using the V Measure Score.
\n\n
Purpose
\n\n
The purpose of this metric, V Measure Score (V Score), is to evaluate the performance of a clustering model. It\nmeasures the homogeneity and completeness of a set of cluster labels, where homogeneity refers to each cluster\ncontaining only members of a single class and completeness meaning all members of a given class are assigned to the\nsame cluster.
\n\n
Test Mechanism
\n\n
ClusterVMeasure is a class that inherits from another class, ClusterPerformance. It uses the v_measure_score\nfunction from the sklearn module's metrics package. The required inputs to perform this metric are the model, train\ndataset, and test dataset. The test is appropriate for models tasked with clustering.
\n\n
Signs of High Risk
\n\n
\n
Low V Measure Score: A low V Measure Score indicates that the clustering model has poor homogeneity or\ncompleteness, or both. This might signal that the model is failing to correctly cluster the data.
\n
\n\n
Strengths
\n\n
\n
The V Measure Score is a harmonic mean between homogeneity and completeness. This ensures that both attributes\nare taken into account when evaluating the model, providing an overall measure of its cluster validity.
\n
The metric does not require knowledge of the ground truth classes when measuring homogeneity and completeness,\nmaking it applicable in instances where such information is unavailable.
\n
\n\n
Limitations
\n\n
\n
The V Measure Score can be influenced by the number of clusters, which means that it might not always reflect the\nquality of the clustering. Partitioning the data into many small clusters could lead to high homogeneity but low\ncompleteness, leading to a low V Measure Score even if the clustering might be useful.
\n
It assumes equal importance of homogeneity and completeness. In some applications, one may be more important than\nthe other. The V Measure Score does not provide flexibility in assigning different weights to homogeneity and\ncompleteness.
Identifies and visualizes weak spots in a machine learning model's performance across various sections of the\nfeature space.
\n\n
Purpose
\n\n
The weak spots test is applied to evaluate the performance of a machine learning model within specific regions of\nits feature space. This test slices the feature space into various sections, evaluating the model's outputs within\neach section against specific performance metrics (e.g., accuracy, precision, recall, and F1 scores). The ultimate\naim is to identify areas where the model's performance falls below the set thresholds, thereby exposing its\npossible weaknesses and limitations.
\n\n
Test Mechanism
\n\n
The test mechanism adopts an approach of dividing the feature space of the training dataset into numerous bins. The\nmodel's performance metrics (accuracy, precision, recall, F1 scores) are then computed for each bin on both the\ntraining and test datasets. A \"weak spot\" is identified if any of the performance metrics fall below a\npredetermined threshold for a particular bin on the test dataset. The test results are visually plotted as bar\ncharts for each performance metric, indicating the bins which fail to meet the established threshold.
\n\n
Signs of High Risk
\n\n
\n
Any performance metric of the model dropping below the set thresholds.
\n
Significant disparity in performance between the training and test datasets within a bin could be an indication\nof overfitting.
\n
Regions or slices with consistently low performance metrics. Such instances could mean that the model struggles\nto handle specific types of input data adequately, resulting in potentially inaccurate predictions.
\n
\n\n
Strengths
\n\n
\n
The test helps pinpoint precise regions of the feature space where the model's performance is below par, allowing\nfor more targeted improvements to the model.
\n
The graphical presentation of the performance metrics offers an intuitive way to understand the model's\nperformance across different feature areas.
\n
The test exhibits flexibility, letting users set different thresholds for various performance metrics according\nto the specific requirements of the application.
\n
\n\n
Limitations
\n\n
\n
The binning system utilized for the feature space in the test could over-simplify the model's behavior within\neach bin. The granularity of this slicing depends on the chosen 'bins' parameter and can sometimes be arbitrary.
\n
The effectiveness of this test largely hinges on the selection of thresholds for the performance metrics, which\nmay not hold universally applicable and could be subjected to the specifications of a particular model and\napplication.
\n
The test is unable to handle datasets with a text column, limiting its application to numerical or categorical\ndata types only.
\n
Despite its usefulness in highlighting problematic regions, the test does not offer direct suggestions for model\nimprovement.
Evaluates ARIMA models for time-series forecasting, ranking them using Bayesian and Akaike Information Criteria.
\n\n
Purpose
\n\n
The AutoARIMA validation test is designed to evaluate and rank AutoRegressive Integrated Moving Average (ARIMA)\nmodels. These models are primarily used for forecasting time-series data. The validation test automatically fits\nmultiple ARIMA models, with varying parameters, to every variable within the given dataset. The models are then\nranked based on their Bayesian Information Criterion (BIC) and Akaike Information Criterion (AIC) values, which\nprovide a basis for the efficient model selection process.
\n\n
Test Mechanism
\n\n
This metric proceeds by generating an array of feasible combinations of ARIMA model parameters which are within a\nprescribed limit. These limits include max_p, max_d, max_q; they represent the autoregressive, differencing,\nand moving average components respectively. Upon applying these sets of parameters, the validation test fits each\nARIMA model to the time-series data provided. For each model, it subsequently proceeds to calculate and record both\nthe BIC and AIC values, which serve as performance indicators for the model fit. Prior to this parameter fitting\nprocess, the Augmented Dickey-Fuller test for data stationarity is conducted on the data series. If a series is\nfound to be non-stationary, a warning message is sent out, given that ARIMA models necessitate input series to be\nstationary.
\n\n
Signs of High Risk
\n\n
\n
If the p-value of the Augmented Dickey-Fuller test for a variable exceeds 0.05, a warning is logged. This warning\nindicates that the series might not be stationary, leading to potentially inaccurate results.
\n
Consistent failure in fitting ARIMA models (as made evident through logged errors) might disclose issues with\neither the data or model stability.
\n
\n\n
Strengths
\n\n
\n
The AutoARIMA validation test simplifies the often complex task of selecting the most suitable ARIMA model based\non BIC and AIC criteria.
\n
The mechanism incorporates a check for non-stationarity within the data, which is a critical prerequisite for\nARIMA models.
\n
The exhaustive search through all possible combinations of model parameters enhances the likelihood of\nidentifying the best-fit model.
\n
\n\n
Limitations
\n\n
\n
This validation test can be computationally costly as it involves creating and fitting multiple ARIMA models for\nevery variable.
\n
Although the test checks for non-stationarity and logs warnings where present, it does not apply any\ntransformations to the data to establish stationarity.
\n
The selection of models leans solely on BIC and AIC criteria, which may not yield the best predictive model in\nall scenarios.
\n
The test is only applicable to regression tasks involving time-series data, and may not work effectively for\nother types of machine learning tasks.
Visualizes cumulative probabilities of positive and negative classes for both training and testing in classification models.
\n\n
Purpose
\n\n
This metric is utilized to evaluate the distribution of predicted probabilities for positive and negative classes\nin a classification model. It provides a visual assessment of the model's behavior by plotting the cumulative\nprobabilities for positive and negative classes across both the training and test datasets.
\n\n
Test Mechanism
\n\n
The classification model is evaluated by first computing the predicted probabilities for each instance in both\nthe training and test datasets, which are then added as a new column in these sets. The cumulative probabilities\nfor positive and negative classes are subsequently calculated and sorted in ascending order. Cumulative\ndistributions of these probabilities are created for both positive and negative classes across both training and\ntest datasets. These cumulative probabilities are represented visually in a plot, containing two subplots - one for\nthe training data and the other for the test data, with lines representing cumulative distributions of positive and\nnegative classes.
\n\n
Signs of High Risk
\n\n
\n
Imbalanced distribution of probabilities for either positive or negative classes.
\n
Notable discrepancies or significant differences between the cumulative probability distributions for the\ntraining data versus the test data.
\n
Marked discrepancies or large differences between the cumulative probability distributions for positive and\nnegative classes.
\n
\n\n
Strengths
\n\n
\n
Provides a visual illustration of data, which enhances the ease of understanding and interpreting the model's\nbehavior.
\n
Allows for the comparison of model's behavior across training and testing datasets, providing insights about how\nwell the model is generalized.
\n
Differentiates between positive and negative classes and their respective distribution patterns, aiding in\nproblem diagnosis.
\n
\n\n
Limitations
\n\n
\n
Exclusive to classification tasks and specifically to classification models.
\n
Graphical results necessitate human interpretation and may not be directly applicable for automated risk\ndetection.
\n
The method does not give a solitary quantifiable measure of model risk, instead, it offers a visual\nrepresentation and broad distributional information.
\n
If the training and test datasets are not representative of the overall data distribution, the metric could\nprovide misleading results.
Assesses autocorrelation in time series data features using the Durbin-Watson statistic.
\n\n
Purpose
\n\n
The Durbin-Watson Test metric detects autocorrelation in time series data (where a set of data values influences\ntheir predecessors). Autocorrelation is a crucial factor for regression tasks as these often assume the\nindependence of residuals. A model with significant autocorrelation may give unreliable predictions.
\n\n
Test Mechanism
\n\n
Utilizing the durbin_watson function in the statsmodels Python library, the Durbin-Watson (DW) Test metric\ngenerates a statistical value for each feature of the training dataset. The function is looped over all columns of\nthe dataset, calculating and caching the DW value for each column for further analysis. A DW metric value nearing 2\nindicates no autocorrelation. Conversely, values approaching 0 suggest positive autocorrelation, and those leaning\ntowards 4 imply negative autocorrelation.
\n\n
Signs of High Risk
\n\n
\n
If a feature's DW value significantly deviates from 2, it could signal a high risk due to potential\nautocorrelation issues in the dataset.
\n
A value closer to 0 could imply positive autocorrelation, while a value nearer to 4 could point to negative\nautocorrelation, both leading to potentially unreliable prediction models.
\n
\n\n
Strengths
\n\n
\n
The metric specializes in identifying autocorrelation in prediction model residuals.
\n
Autocorrelation detection assists in diagnosing violation of various modeling technique assumptions, particularly\nin regression analysis and time-series data modeling.
\n
\n\n
Limitations
\n\n
\n
The Durbin-Watson Test mainly detects linear autocorrelation and could overlook other types of relationships.
\n
The metric is highly sensitive to data points order. Shuffling the order could lead to notably different results.
\n
The test only checks for first-order autocorrelation (between a variable and its immediate predecessor) and fails\nto detect higher-order autocorrelation.
Evaluates classification model performance using AUC, GINI, and KS metrics for training and test datasets.
\n\n
Purpose
\n\n
The 'GINITable' metric is designed to evaluate the performance of a classification model by emphasizing its\ndiscriminatory power. Specifically, it calculates and presents three important metrics - the Area under the ROC\nCurve (AUC), the GINI coefficient, and the Kolmogorov-Smirnov (KS) statistic - for both training and test datasets.
\n\n
Test Mechanism
\n\n
Using a dictionary for storing performance metrics for both the training and test datasets, the 'GINITable' metric\ncalculates each of these metrics sequentially. The Area under the ROC Curve (AUC) is calculated via the\nroc_auc_score function from the Scikit-Learn library. The GINI coefficient, a measure of statistical dispersion,\nis then computed by doubling the AUC and subtracting 1. Finally, the Kolmogorov-Smirnov (KS) statistic is\ncalculated via the roc_curve function from Scikit-Learn, with the False Positive Rate (FPR) subtracted from the\nTrue Positive Rate (TPR) and the maximum value taken from the resulting data. These metrics are then stored in a\npandas DataFrame for convenient visualization.
\n\n
Signs of High Risk
\n\n
\n
Low values for performance metrics may suggest a reduction in model performance, particularly a low AUC which\nindicates poor classification performance, or a low GINI coefficient, which could suggest a decreased ability to\ndiscriminate different classes.
\n
A high KS value may be an indicator of potential overfitting, as this generally signifies a substantial\ndivergence between positive and negative distributions.
\n
Significant discrepancies between the performance on the training dataset and the test dataset may present\nanother signal of high risk.
\n
\n\n
Strengths
\n\n
\n
Offers three key performance metrics (AUC, GINI, and KS) in one test, providing a more comprehensive evaluation\nof the model.
\n
Provides a direct comparison between the model's performance on training and testing datasets, which aids in\nidentifying potential underfitting or overfitting.
\n
The applied metrics are class-distribution invariant, thereby remaining effective for evaluating model\nperformance even when dealing with imbalanced datasets.
\n
Presents the metrics in a user-friendly table format for easy comprehension and analysis.
\n
\n\n
Limitations
\n\n
\n
The GINI coefficient and KS statistic are both dependent on the AUC value. Therefore, any errors in the\ncalculation of the latter will adversely impact the former metrics too.
\n
Mainly suited for binary classification models and may require modifications for effective application in\nmulti-class scenarios.
\n
The metrics used are threshold-dependent and may exhibit high variability based on the chosen cut-off points.
\n
The test does not incorporate a method to efficiently handle missing or inefficiently processed data, which could\nlead to inaccuracies in the metrics if the data is not appropriately preprocessed.
Assesses whether each feature in the dataset aligns with a normal distribution using the Kolmogorov-Smirnov test.
\n\n
Purpose
\n\n
The Kolmogorov-Smirnov (KS) test evaluates the distribution of features in a dataset to determine their alignment\nwith a normal distribution. This is important because many statistical methods and machine learning models assume\nnormality in the data distribution.
\n\n
Test Mechanism
\n\n
This test calculates the KS statistic and corresponding p-value for each feature in the dataset. It does so by\ncomparing the cumulative distribution function of the feature with an ideal normal distribution. The KS statistic\nand p-value for each feature are then stored in a dictionary. The p-value threshold to reject the normal\ndistribution hypothesis is not preset, providing flexibility for different applications.
\n\n
Signs of High Risk
\n\n
\n
Elevated KS statistic for a feature combined with a low p-value, indicating a significant divergence from a\nnormal distribution.
\n
Features with notable deviations that could create problems if the model assumes normality in data distribution.
\n
\n\n
Strengths
\n\n
\n
The KS test is sensitive to differences in the location and shape of empirical cumulative distribution functions.
\n
It is non-parametric and adaptable to various datasets, as it does not assume any specific data distribution.
\n
Provides detailed insights into the distribution of individual features.
\n
\n\n
Limitations
\n\n
\n
The test's sensitivity to disparities in the tails of data distribution might cause false alarms about\nnon-normality.
\n
Less effective for multivariate distributions, as it is designed for univariate distributions.
\n
Does not identify specific types of non-normality, such as skewness or kurtosis, which could impact model fitting.
Assesses the normality of feature distributions in an ML model's training dataset using the Lilliefors test.
\n\n
Purpose
\n\n
The purpose of this metric is to utilize the Lilliefors test, named in honor of the Swedish statistician Hubert\nLilliefors, in order to assess whether the features of the machine learning model's training dataset conform to a\nnormal distribution. This is done because the assumption of normal distribution plays a vital role in numerous\nstatistical procedures as well as numerous machine learning models. Should the features fail to follow a normal\ndistribution, some model types may not operate at optimal efficiency. This can potentially lead to inaccurate\npredictions.
\n\n
Test Mechanism
\n\n
The application of this test happens across all feature columns within the training dataset. For each feature, the\nLilliefors test returns a test statistic and p-value. The test statistic quantifies how far the feature's\ndistribution is from an ideal normal distribution, whereas the p-value aids in determining the statistical\nrelevance of this deviation. The final results are stored within a dictionary, the keys of which correspond to the\nname of the feature column, and the values being another dictionary which houses the test statistic and p-value.
\n\n
Signs of High Risk
\n\n
\n
If the p-value corresponding to a specific feature sinks below a pre-established significance level, generally\nset at 0.05, then it can be deduced that the distribution of that feature significantly deviates from a normal\ndistribution. This can present a high risk for models that assume normality, as these models may perform\ninaccurately or inefficiently in the presence of such a feature.
\n
\n\n
Strengths
\n\n
\n
One advantage of the Lilliefors test is its utility irrespective of whether the mean and variance of the normal\ndistribution are known in advance. This makes it a more robust option in real-world situations where these values\nmight not be known.
\n
The test has the ability to screen every feature column, offering a holistic view of the dataset.
\n
\n\n
Limitations
\n\n
\n
Despite the practical applications of the Lilliefors test in validating normality, it does come with some\nlimitations.
\n
It is only capable of testing unidimensional data, thus rendering it ineffective for datasets with interactions\nbetween features or multi-dimensional phenomena.
\n
The test might not be as sensitive as some other tests (like the Anderson-Darling test) in detecting deviations\nfrom a normal distribution.
\n
Like any other statistical test, Lilliefors test may also produce false positives or negatives. Hence, banking\nsolely on this test, without considering other characteristics of the data, may give rise to risks.
Assesses the predictive probability distribution for binary classification to evaluate model performance and\npotential overfitting or bias.
\n\n
Purpose
\n\n
The Prediction Probabilities Histogram test is designed to generate histograms displaying the Probability of\nDefault (PD) predictions for both positive and negative classes in training and testing datasets. This helps in\nevaluating the performance of a classification model.
\n\n
Test Mechanism
\n\n
The metric follows these steps to execute the test:
\n\n
\n
Extracts the target column from both the train and test datasets.
\n
Uses the model's predict function to calculate probabilities.
\n
Adds these probabilities as a new column to the training and testing dataframes.
\n
Generates histograms for each class (0 or 1) within the training and testing datasets.
\n
Sets different opacities for the histograms to enhance visualization.
\n
Overlays the four histograms (two for training and two for testing) on two different subplot frames.
\n
Returns a plotly graph object displaying the visualization.
\n
\n\n
Signs of High Risk
\n\n
\n
Significant discrepancies between the histograms of training and testing data.
\n
Large disparities between the histograms for the positive and negative classes.
\n
Potential overfitting or bias indicated by significant issues.
\n
Unevenly distributed probabilities suggesting inaccurate model predictions.
\n
\n\n
Strengths
\n\n
\n
Offers a visual representation of the PD predictions made by the model, aiding in understanding its behavior.
\n
Assesses both the training and testing datasets, adding depth to model validation.
\n
Highlights disparities between classes, providing insights into class imbalance or data skewness.
\n
Effectively visualizes risk spread, which is particularly beneficial for credit risk prediction.
\n
\n\n
Limitations
\n\n
\n
Specifically tailored for binary classification scenarios and not suited for multi-class classification tasks.
\n
Provides a robust visual representation but lacks a quantifiable measure to assess model performance.
Assesses the significance and uncertainty of predictor variables in a regression model through visualization of\ncoefficients and their 95% confidence intervals.
\n\n
Purpose
\n\n
The RegressionCoeffs metric visualizes the estimated regression coefficients alongside their 95% confidence intervals,\nproviding insights into the impact and significance of predictor variables on the response variable. This visualization\nhelps to understand the variability and uncertainty in the model's estimates, aiding in the evaluation of the\nsignificance of each predictor.
\n\n
Test Mechanism
\n\n
The function operates by extracting the estimated coefficients and their standard errors from the regression model.\nUsing these, it calculates the confidence intervals at a 95% confidence level, which indicates the range within which\nthe true coefficient value is expected to fall 95% of the time. The confidence intervals are computed using the\nZ-value associated with the 95% confidence level. The coefficients and their confidence intervals are then visualized\nin a bar plot. The x-axis represents the predictor variables, the y-axis represents the estimated coefficients, and\nthe error bars depict the confidence intervals.
\n\n
Signs of High Risk
\n\n
\n
The confidence interval for a coefficient contains the zero value, suggesting that the predictor may not significantly\ncontribute to the model.
\n
Multiple coefficients with confidence intervals that include zero, potentially indicating issues with model reliability.
\n
Very wide confidence intervals, which may suggest high uncertainty in the coefficient estimates and potential model\ninstability.
\n
\n\n
Strengths
\n\n
\n
Provides a clear visualization that allows for easy interpretation of the significance and impact of predictor\nvariables.
\n
Includes confidence intervals, which provide additional information about the uncertainty surrounding each coefficient\nestimate.
\n
\n\n
Limitations
\n\n
\n
The method assumes normality of residuals and independence of observations, assumptions that may not always hold true\nin practice.
\n
It does not address issues related to multi-collinearity among predictor variables, which can affect the interpretation\nof coefficients.
\n
This metric is limited to regression tasks using tabular data and is not applicable to other types of machine learning\ntasks or data structures.
Assesses and visualizes the statistical significance of features in a regression model.
\n\n
Purpose
\n\n
The Regression Feature Significance metric assesses the significance of each feature in a given set of regression\nmodel. It creates a visualization displaying p-values for every feature of the model, assisting model developers\nin understanding which features are most influential in their model.
\n\n
Test Mechanism
\n\n
The test mechanism involves extracting the model's coefficients and p-values for each feature, and then plotting these\nvalues. The x-axis on the plot contains the p-values while the y-axis denotes the coefficients of each feature. A\nvertical red line is drawn at the threshold for p-value significance, which is 0.05 by default. Any features with\np-values to the left of this line are considered statistically significant at the chosen level.
\n\n
Signs of High Risk
\n\n
\n
Any feature with a high p-value (greater than the threshold) is considered a potential high risk, as it suggests\nthe feature is not statistically significant and may not be reliably contributing to the model's predictions.
\n
A high number of such features may indicate problems with the model validation, variable selection, and overall\nreliability of the model predictions.
\n
\n\n
Strengths
\n\n
\n
Helps identify the features that significantly contribute to a model's prediction, providing insights into the\nfeature importance.
\n
Provides tangible, easy-to-understand visualizations to interpret the feature significance.
\n
\n\n
Limitations
\n\n
\n
This metric assumes model features are independent, which may not always be the case. Multicollinearity (high\ncorrelation amongst predictors) can cause high variance and unreliable statistical tests of significance.
\n
The p-value strategy for feature selection doesn't take into account the magnitude of the effect, focusing solely\non whether the feature is likely non-zero.
\n
This test is specific to regression models and wouldn't be suitable for other types of ML models.
\n
P-value thresholds are somewhat arbitrary and do not always indicate practical significance, only statistical\nsignificance.
Generates plots to visually compare the forecasted outcomes of a regression model against actual observed values over\na specified date range.
\n\n
Purpose
\n\n
This metric is useful for time-series models or any model where the outcome changes over time, allowing direct\ncomparison of predicted vs actual values. It can help identify overfitting or underfitting situations as well as\ngeneral model performance.
\n\n
Test Mechanism
\n\n
This test generates a plot with the x-axis representing the date ranging from the specified \"start_date\" to the\n\"end_date\", while the y-axis shows the value of the outcome variable. Two lines are plotted: one representing the\nforecasted values and the other representing the observed values. The \"start_date\" and \"end_date\" can be parameters\nof this test; if these parameters are not provided, they are set to the minimum and maximum date available in the\ndataset.
\n\n
Signs of High Risk
\n\n
\n
High risk or failure signs could be deduced visually from the plots if the forecasted line significantly deviates\nfrom the observed line, indicating the model's predicted values are not matching actual outcomes.
\n
A model that struggles to handle the edge conditions like maximum and minimum data points could also be\nconsidered a sign of risk.
\n
\n\n
Strengths
\n\n
\n
Visualization: The plot provides an intuitive and clear illustration of how well the forecast matches the actual\nvalues, making it straightforward even for non-technical stakeholders to interpret.
\n
Flexibility: It allows comparison for multiple models and for specified time periods.
\n
Model Evaluation: It can be useful in identifying overfitting or underfitting situations, as these will manifest\nas discrepancies between the forecasted and observed values.
\n
\n\n
Limitations
\n\n
\n
Interpretation Bias: Interpretation of the plot is subjective and can lead to different conclusions by different\nevaluators.
\n
Lack of Precision: Visual representation might not provide precise values of the deviation.
\n
Inapplicability: Limited to cases where the order of data points (time-series) matters, it might not be of much\nuse in problems that are not related to time series prediction.
Assesses the alignment between forecasted and observed values in regression models through visual plots
\n\n
Purpose
\n\n
This test aims to visually assess the performance of a regression model by comparing its forecasted values against\nthe actual observed values for both the raw and transformed (integrated) data. This helps determine the accuracy\nof the model and can help identify overfitting or underfitting. The integration is applied to highlight the trend\nrather than the absolute level.
\n\n
Test Mechanism
\n\n
This test generates two plots:
\n\n
\n
Raw data vs forecast
\n
Transformed data vs forecast
\n
\n\n
The transformed data is created by performing a cumulative sum on the raw data.
\n\n
Signs of High Risk
\n\n
\n
Significant deviation between forecasted and observed values.
\n
Patterns suggesting overfitting or underfitting.
\n
Large discrepancies in the plotted forecasts, indicating potential issues with model generalizability and\nprecision.
\n
\n\n
Strengths
\n\n
\n
Provides an intuitive, visual way to assess multiple regression models, aiding in easier interpretation and\nevaluation of forecast accuracy.
\n
\n\n
Limitations
\n\n
\n
Relies heavily on visual interpretation, which may vary between individuals.
\n
Does not provide a numerical metric to quantify forecast accuracy, relying solely on visual assessment.
Assesses the sensitivity of a regression model to changes in independent variables by applying shocks and\nvisualizing the impact.
\n\n
Purpose
\n\n
The Regression Sensitivity Plot test is designed to perform sensitivity analysis on regression models. This test\naims to measure the impact of slight changes (shocks) applied to individual variables on the system's outcome while\nkeeping all other variables constant. By doing so, it analyzes the effects of each independent variable on the\ndependent variable within the regression model, helping identify significant risk factors that could substantially\ninfluence the model's output.
\n\n
Test Mechanism
\n\n
This test operates by initially applying shocks of varying magnitudes, defined by specific parameters, to each of\nthe model's features, one at a time. With all other variables held constant, a new prediction is made for each\ndataset subjected to shocks. Any changes in the model's predictions are directly attributed to the shocks applied.\nIf the transformation parameter is set to \"integrate,\" initial predictions and target values undergo transformation\nvia an integration function before being plotted. Finally, a plot demonstrating observed values against predicted\nvalues for each model is generated, showcasing a distinct line graph illustrating predictions for each shock.
\n\n
Signs of High Risk
\n\n
\n
Drastic alterations in model predictions due to minor shocks to an individual variable, indicating high\nsensitivity and potential over-dependence on that variable.
\n
Unusually high or unpredictable shifts in response to shocks, suggesting potential model instability or\noverfitting.
\n
\n\n
Strengths
\n\n
\n
Helps identify variables that strongly influence model outcomes, aiding in understanding feature importance.
\n
Generates visual plots, making results easily interpretable even to non-technical stakeholders.
\n
Useful in identifying overfitting and detecting unstable models that react excessively to minor variable changes.
\n
\n\n
Limitations
\n\n
\n
Operates on the assumption that all other variables remain unchanged during the application of a shock, which may\nnot reflect real-world interdependencies.
\n
Best compatible with linear models and may not effectively evaluate the sensitivity of non-linear models.
\n
Provides a visual representation without a numerical risk measure, potentially introducing subjectivity in\ninterpretation.
Evaluates regression model performance using metrics including R-Squared, Adjusted R-Squared, MSE, and RMSE.
\n\n
Purpose
\n\n
The Regression Model Summary test evaluates the performance of regression models by measuring their predictive\nability regarding dependent variables given changes in the independent variables. It uses conventional regression\nmetrics such as R-Squared, Adjusted R-Squared, Mean Squared Error (MSE), and Root Mean Squared Error (RMSE) to\nassess the model's accuracy and fit.
\n\n
Test Mechanism
\n\n
This test uses the sklearn library to calculate the R-Squared, Adjusted R-Squared, MSE, and RMSE. It outputs a\ntable with the results of these metrics along with the feature columns used by the model.
\n\n
Signs of High Risk
\n\n
\n
Low R-Squared and Adjusted R-Squared values.
\n
High MSE and RMSE values.
\n
\n\n
Strengths
\n\n
\n
Offers an extensive evaluation of regression models by combining four key measures of model accuracy and fit.
\n
Provides a comprehensive view of the model's performance.
\n
Both the R-Squared and Adjusted R-Squared measures are readily interpretable.
\n
\n\n
Limitations
\n\n
\n
RMSE and MSE might be sensitive to outliers.
\n
A high R-Squared or Adjusted R-Squared may not necessarily indicate a good model, especially in cases of\noverfitting.
Assesses the significance of each feature in a model by evaluating the impact on model performance when feature\nvalues are randomly rearranged.
\n\n
Purpose
\n\n
The primary purpose of this metric is to determine which features significantly impact the performance of a\nregression model developed using statsmodels. The metric measures how much the prediction accuracy deteriorates\nwhen each feature's values are permuted.
\n\n
Test Mechanism
\n\n
This metric shuffles the values of each feature one at a time in the dataset, computes the model's performance\nafter each permutation, and compares it to the baseline performance. A significant decrease in performance\nindicates the importance of the feature.
\n\n
Signs of High Risk
\n\n
\n
Significant reliance on a feature that, when permuted, leads to a substantial decrease in performance, suggesting\noverfitting or high model dependency on that feature.
\n
Features identified as unimportant despite known impacts from domain knowledge, suggesting potential issues in\nmodel training or data preprocessing.
\n
\n\n
Strengths
\n\n
\n
Directly assesses the impact of each feature on model performance, providing clear insights into model\ndependencies.
\n
Model-agnostic within the scope of statsmodels, applicable to any regression model that outputs predictions.
\n
\n\n
Limitations
\n\n
\n
The metric is specific to statsmodels and cannot be used with other types of models without adaptation.
\n
It does not capture interactions between features, which can lead to underestimating the importance of correlated\nfeatures.
\n
Assumes independence of features when calculating importance, which might not always hold true.
The Scorecard Histogram test evaluates the distribution of credit scores between default and non-default instances,\nproviding critical insights into the performance and generalizability of credit-risk models.
\n\n
Purpose
\n\n
The Scorecard Histogram test metric provides a visual interpretation of the credit scores generated by a machine\nlearning model for credit-risk classification tasks. It aims to compare the alignment of the model's scoring\ndecisions with the actual outcomes of credit loan applications. It helps in identifying potential discrepancies\nbetween the model's predictions and real-world risk levels.
\n\n
Test Mechanism
\n\n
This metric uses logistic regression to generate a histogram of credit scores for both default (negative class) and\nnon-default (positive class) instances. Using both training and test datasets, the metric calculates the credit\nscore of each instance with a scorecard method, considering the impact of different features on the likelihood of\ndefault. It includes the default point to odds (PDO) scaling factor and predefined target score and odds settings.\nHistograms for training and test sets are computed and plotted separately to offer insights into the model's\ngeneralizability to unseen data.
\n\n
Signs of High Risk
\n\n
\n
Discrepancies between the distributions of training and testing data, indicating a model's poor generalization\nability
\n
Skewed distributions favoring specific scores or classes, representing potential bias
\n
\n\n
Strengths
\n\n
\n
Provides a visual interpretation of the model's credit scoring system, enhancing comprehension of model behavior
\n
Enables a direct comparison between actual and predicted scores for both training and testing data
\n
Its intuitive visualization helps understand the model's ability to differentiate between positive and negative\nclasses
\n
Can unveil patterns or anomalies not easily discerned through numerical metrics alone
\n
\n\n
Limitations
\n\n
\n
Despite its value for visual interpretation, it doesn't quantify the performance of the model and therefore may\nlack precision for thorough model evaluation
\n
The quality of input data can strongly influence the metric, as bias or noise in the data will affect both the\nscore calculation and resultant histogram
\n
Its specificity to credit scoring models limits its applicability across a wider variety of machine learning\ntasks and models
\n
The metric's effectiveness is somewhat tied to the subjective interpretation of the analyst, relying on their\njudgment of the characteristics and implications of the plot.
Assesses potential bias in a Large Language Model by analyzing the distribution and order of exemplars in the\nprompt.
\n\n
Purpose
\n\n
The Bias Evaluation test calculates if and how the order and distribution of exemplars (examples) in a few-shot\nlearning prompt affect the output of a Large Language Model (LLM). The results of this evaluation can be used to\nfine-tune the model's performance and manage any unintended biases in its results.
\n\n
Test Mechanism
\n\n
This test uses two checks:
\n\n\n
Distribution of Exemplars: The number of positive vs. negative examples in a prompt is varied. The test then\nexamines the LLM's classification of a neutral or ambiguous statement under these circumstances.
\n
Order of Exemplars: The sequence in which positive and negative examples are presented to the model is\nmodified. Their resultant effect on the LLM's response is studied.
\n\n\n
For each test case, the LLM grades the input prompt on a scale of 1 to 10. It evaluates whether the examples in the\nprompt could produce biased responses. The test only passes if the score meets or exceeds a predetermined minimum\nthreshold. This threshold is set at 7 by default but can be modified as per the requirements via the test\nparameters.
\n\n
Signs of High Risk
\n\n
\n
A skewed result favoring either positive or negative responses may suggest potential bias in the model. This skew\ncould be caused by an unbalanced distribution of positive and negative exemplars.
\n
If the score given by the model is less than the set minimum threshold, it might indicate a risk of high bias and\nhence poor performance.
\n
\n\n
Strengths
\n\n
\n
This test provides a quantitative measure of potential bias, offering clear guidelines for developers about\nwhether their Large Language Model (LLM) contains significant bias.
\n
It is useful in evaluating the impartiality of the model based on the distribution and sequence of examples.
\n
The flexibility to adjust the minimum required threshold allows tailoring this test to stricter or more lenient\nbias standards.
\n
\n\n
Limitations
\n\n
\n
The test may not pick up on more subtle forms of bias or biases that are not directly related to the distribution\nor order of exemplars.
\n
The test's effectiveness will decrease if the quality or balance of positive and negative exemplars is not\nrepresentative of the problem space the model is intended to solve.
\n
The use of a grading mechanism to gauge bias may not be entirely accurate in every case, particularly when the\ndifference between threshold and score is narrow.
Evaluates and scores the clarity of prompts in a Large Language Model based on specified guidelines.
\n\n
Purpose
\n\n
The Clarity evaluation metric is used to assess how clear the prompts of a Large Language Model (LLM) are. This\nassessment is particularly important because clear prompts assist the LLM in more accurately interpreting and\nresponding to instructions.
\n\n
Test Mechanism
\n\n
The evaluation uses an LLM to scrutinize the clarity of prompts, factoring in considerations such as the inclusion\nof relevant details, persona adoption, step-by-step instructions, usage of examples, and specification of desired\noutput length. Each prompt is rated on a clarity scale of 1 to 10, and any prompt scoring at or above the preset\nthreshold (default of 7) will be marked as clear. It is important to note that this threshold can be adjusted via\ntest parameters, providing flexibility in the evaluation process.
\n\n
Signs of High Risk
\n\n
\n
Prompts that consistently score below the clarity threshold
\n
Repeated failure of prompts to adhere to guidelines for clarity, including detail inclusion, persona adoption,\nexplicit step-by-step instructions, use of examples, and specification of output length
\n
\n\n
Strengths
\n\n
\n
Encourages the development of more effective prompts that aid the LLM in interpreting instructions accurately
\n
Applies a quantifiable measure (a score from 1 to 10) to evaluate the clarity of prompts
\n
Threshold for clarity is adjustable, allowing for flexible evaluation depending on the context
\n
\n\n
Limitations
\n\n
\n
Scoring system is subjective and relies on the AI\u2019s interpretation of 'clarity'
\n
The test assumes that all required factors (detail inclusion, persona adoption, step-by-step instructions, use of\nexamples, and specification of output length) contribute equally to clarity, which might not always be the case
\n
The evaluation may not be as effective if used on non-textual models
Analyzes and grades the conciseness of prompts provided to a Large Language Model.
\n\n
Purpose
\n\n
The Conciseness Assessment is designed to evaluate the brevity and succinctness of prompts provided to a Language\nLearning Model (LLM). A concise prompt strikes a balance between offering clear instructions and eliminating\nredundant or unnecessary information, ensuring that the LLM receives relevant input without being overwhelmed.
\n\n
Test Mechanism
\n\n
Using an LLM, this test conducts a conciseness analysis on input prompts. The analysis grades the prompt on a scale\nfrom 1 to 10, where the grade reflects how well the prompt delivers clear instructions without being verbose.\nPrompts that score equal to or above a predefined threshold (default set to 7) are deemed successfully concise.\nThis threshold can be adjusted to meet specific requirements.
\n\n
Signs of High Risk
\n\n
\n
Prompts that consistently score below the predefined threshold.
\n
Prompts that are overly wordy or contain unnecessary information.
\n
Prompts that create confusion or ambiguity due to excess or unnecessary information.
\n
\n\n
Strengths
\n\n
\n
Ensures clarity and effectiveness of the prompts.
\n
Promotes brevity and preciseness in prompts without sacrificing essential information.
\n
Useful for models like LLMs, where input prompt length and clarity greatly influence model performance.
\n
Provides a quantifiable measure of prompt conciseness.
\n
\n\n
Limitations
\n\n
\n
The conciseness score is based on an AI's assessment, which might not fully capture human interpretation of\nconciseness.
\n
The predefined threshold for conciseness could be subjective and might need adjustment based on application.
\n
The test is dependent on the LLM\u2019s understanding of conciseness, which might vary from model to model.
Evaluates the proper use of delimiters in prompts provided to Large Language Models.
\n\n
Purpose
\n\n
The Delimitation Test aims to assess whether prompts provided to the Language Learning Model (LLM) correctly use\ndelimiters to mark different sections of the input. Well-delimited prompts help simplify the interpretation process\nfor the LLM, ensuring that the responses are precise and accurate.
\n\n
Test Mechanism
\n\n
The test employs an LLM to examine prompts for appropriate use of delimiters such as triple quotation marks, XML\ntags, and section titles. Each prompt is assigned a score from 1 to 10 based on its delimitation integrity. Prompts\nwith scores equal to or above the preset threshold (which is 7 by default, although it can be adjusted as\nnecessary) pass the test.
\n\n
Signs of High Risk
\n\n
\n
Prompts missing, improperly placed, or incorrectly used delimiters, leading to misinterpretation by the LLM.
\n
High-risk scenarios with complex prompts involving multiple tasks or diverse data where correct delimitation is\ncrucial.
\n
Scores below the threshold, indicating a high risk.
\n
\n\n
Strengths
\n\n
\n
Ensures clarity in demarcating different components of given prompts.
\n
Reduces ambiguity in understanding prompts, especially for complex tasks.
\n
Provides a quantified insight into the appropriateness of delimiter usage, aiding continuous improvement.
\n
\n\n
Limitations
\n\n
\n
Only checks for the presence and placement of delimiters, not whether the correct delimiter type is used for the\nspecific data or task.
\n
May not fully reveal the impacts of poor delimitation on the LLM's final performance.
\n
The preset score threshold may not be refined enough for complex tasks and prompts, requiring regular manual\nadjustment.
Evaluates and grades the use of affirmative, proactive language over negative instructions in LLM prompts.
\n\n
Purpose
\n\n
The Negative Instruction test is utilized to scrutinize the prompts given to a Large Language Model (LLM). The\nobjective is to ensure these prompts are expressed using proactive, affirmative language. The focus is on\ninstructions indicating what needs to be done rather than what needs to be avoided, thereby guiding the LLM more\nefficiently towards the desired output.
\n\n
Test Mechanism
\n\n
An LLM is employed to evaluate each prompt. The prompt is graded based on its use of positive instructions with\nscores ranging between 1-10. This grade reflects how effectively the prompt leverages affirmative language while\nshying away from negative or restrictive instructions. A prompt that attains a grade equal to or above a\npredetermined threshold (7 by default) is regarded as adhering effectively to the best practices of positive\ninstruction. This threshold can be custom-tailored through the test parameters.
\n\n
Signs of High Risk
\n\n
\n
Low score obtained from the LLM analysis, indicating heavy reliance on negative instructions in the prompts.
\n
Failure to surpass the preset minimum threshold.
\n
The LLM generates ambiguous or undesirable outputs as a consequence of the negative instructions used in the\nprompt.
\n
\n\n
Strengths
\n\n
\n
Encourages the usage of affirmative, proactive language in prompts, aiding in more accurate and advantageous\nmodel responses.
\n
The test result provides a comprehensible score, helping to understand how well a prompt follows the positive\ninstruction best practices.
\n
\n\n
Limitations
\n\n
\n
Despite an adequate score, a prompt could still be misleading or could lead to undesired responses due to factors\nnot covered by this test.
\n
The test necessitates an LLM for evaluation, which might not be available or feasible in certain scenarios.
\n
A numeric scoring system, while straightforward, may oversimplify complex issues related to prompt designing and\ninstruction clarity.
\n
The effectiveness of the test hinges significantly on the predetermined threshold level, which can be subjective\nand may need to be adjusted according to specific use-cases.
Assesses the robustness of prompts provided to a Large Language Model under varying conditions and contexts. This test\nspecifically measures the model's ability to generate correct classifications with the given prompt even when the\ninputs are edge cases or otherwise difficult to classify.
\n\n
Purpose
\n\n
The Robustness test is meant to evaluate the resilience and reliability of prompts provided to a Language Learning\nModel (LLM). The aim of this test is to guarantee that the prompts consistently generate accurate and expected\noutputs, even in diverse or challenging scenarios. This test is only applicable to LLM-powered text classification\ntasks where the prompt has a single input variable.
\n\n
Test Mechanism
\n\n
The Robustness test appraises prompts under various conditions, alterations, and contexts to ascertain their\nstability in producing consistent responses from the LLM. Factors evaluated include different phrasings, inclusion\nof potential distracting elements, and various input complexities. By default, the test generates 10 inputs for a\nprompt but can be adjusted according to test parameters.
\n\n
Signs of High Risk
\n\n
\n
If the output from the tests diverges extensively from the expected results, this indicates high risk.
\n
When the prompt doesn't give a consistent performance across various tests.
\n
A high risk is indicated when the prompt is susceptible to breaking, especially when the output is expected to be\nof a specific type.
\n
\n\n
Strengths
\n\n
\n
The robustness test helps to ensure stable performance of the LLM prompts and lowers the chances of generating\nunexpected or off-target outputs.
\n
This test is vital for applications where predictability and reliability of the LLM\u2019s output are crucial.
\n
\n\n
Limitations
\n\n
\n
Currently, the test only supports single-variable prompts, which restricts its application to more complex models.
\n
When there are too many target classes (over 10), the test is skipped, which can leave potential vulnerabilities\nunchecked in complex multi-class models.
\n
The test may not account for all potential conditions or alterations that could show up in practical use\nscenarios.
Evaluates and scores the specificity of prompts provided to a Large Language Model (LLM), based on clarity, detail,\nand relevance.
\n\n
Purpose
\n\n
The Specificity Test evaluates the clarity, precision, and effectiveness of the prompts provided to a Language\nModel (LLM). It aims to ensure that the instructions embedded in a prompt are indisputably clear and relevant,\nthereby helping to remove ambiguity and steer the LLM towards desired outputs. This level of specificity\nsignificantly affects the accuracy and relevance of LLM outputs.
\n\n
Test Mechanism
\n\n
The Specificity Test employs an LLM to grade each prompt based on clarity, detail, and relevance parameters within\na specificity scale that extends from 1 to 10. On this scale, prompts scoring equal to or more than a predefined\nthreshold (set to 7 by default) pass the evaluation, while those scoring below this threshold fail it. Users can\nadjust this threshold as per their requirements.
\n\n
Signs of High Risk
\n\n
\n
Prompts scoring consistently below the established threshold
\n
Vague or ambiguous prompts that do not provide clear direction to the LLM
\n
Overly verbose prompts that may confuse the LLM instead of providing clear guidance
\n
\n\n
Strengths
\n\n
\n
Enables precise and clear communication with the LLM to achieve desired outputs
\n
Serves as a crucial means to measure the effectiveness of prompts
\n
Highly customizable, allowing users to set their threshold based on specific use cases
\n
\n\n
Limitations
\n\n
\n
This test doesn't consider the content comprehension capability of the LLM
\n
High specificity score doesn't guarantee a high-quality response from the LLM, as the model's performance is also\ndependent on various other factors
\n
Striking a balance between specificity and verbosity can be challenging, as overly detailed prompts might confuse\nor mislead the model
Allows for setting options on the input object that are passed by the user\nwhen using the input to run a test or set of tests
\n\n
To allow options, just override this method in the subclass (see VMDataset)\nand ensure that it returns a new instance of the input with the specified options\nset.
\n\n
Arguments:
\n\n
\n
**kwargs: Arbitrary keyword arguments that will be passed to the input object
\n
\n\n
Returns:
\n\n
\n
VMInput: A new instance of the input with the specified options set
Child classes should be used to support new dataset types (tensor, polars etc)\nby converting the user's dataset into a numpy array collecting metadata like\ncolumn names and then call this (parent) class __init__ method.
\n\n
This way we can support multiple dataset types but under the hood we only\nneed to work with numpy arrays and pandas dataframes in this class.
\n\n
Attributes:
\n\n
\n
raw_dataset (np.ndarray): The raw dataset as a NumPy array.
\n
input_id (str): Identifier for the dataset.
\n
index (np.ndarray): The raw dataset index as a NumPy array.
\n
columns (Set[str]): The column names of the dataset.
\n
target_column (str): The target column name of the dataset.
\n
feature_columns (List[str]): The feature column names of the dataset.
\n
feature_columns_numeric (List[str]): The numeric feature column names of the dataset.
\n
feature_columns_categorical (List[str]): The categorical feature column names of the dataset.
\n
text_column (str): The text column name of the dataset for NLP tasks.
\n
target_class_labels (Dict): The class labels for the target columns.
\n
df (pd.DataFrame): The dataset as a pandas DataFrame.
\n
extra_columns (Dict): Extra columns to include in the dataset.
Support options provided when passing an input to run_test or run_test_suite
\n\n
Example:
\n\n
\n
# to only use a certain subset of columns in the dataset:\nrun_test(\n "validmind.SomeTestID",\n inputs={\n "dataset":{\n "input_id":"my_dataset_id",\n "columns":["col1","col2"],\n }\n }\n)\n\n# behind the scenes, this retrieves the dataset object (VMDataset) from the registry\n# and then calls the `with_options()` method and passes `{"columns": ...}`\n
\n
\n\n
Arguments:
\n\n
\n
**kwargs: Options:\n
\n
columns: Filter columns in the dataset
\n
\n
\n\n
Returns:
\n\n
\n
VMDataset: A new instance of the dataset with only the specified columns
Returns the ipywidget compatible representation of the figure. Ideally\nwe would render images as-is, but Plotly FigureWidgets don't work well\non Google Colab when they are combined with ipywidgets.
figure (Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget,\nbytes, Figure]): The figure to add (can be either a VM Figure object,\na raw figure object from the supported libraries, or a png image as\nraw bytes)
Base class for test suites. Test suites are used to define a grouping of tests that\ncan be run as a suite against datasets and models. Test Suites can be defined by\ninheriting from this base class and defining the list of tests as a class variable.
\n\n
Tests can be a flat list of strings or may be nested into sections by using a dict
Returns the default configuration for the test suite
\n\n
Each test in a test suite can accept parameters and those parameters can have\ndefault values. Both the parameters and their defaults are set in the test\nclass and a config object can be passed to the test suite's run method to\noverride the defaults. This function returns a dictionary containing the\nparameters and their default values for every test to allow users to view\nand set values
\n\n
Returns:
\n\n
\n
dict: A dictionary of test names and their default parameters
The ValidMind Library is a suite of developer tools and methods designed to automate the documentation and validation of your models.
-
-
Designed to be model agnostic, the ValidMind Library provides all the standard functionality without requiring you to rewrite any functions as long as your model is built in Python.
-
-
With a rich array of documentation tools and test suites, from documenting descriptions of your datasets to testing your models for weak spots and overfit areas, the ValidMind Library helps you automate model documentation by feeding the ValidMind Platform with documentation artifacts and test results.
-
-
To install the ValidMind Library:
-
-
-
pipinstallvalidmind
-
-
-
-
To initialize the ValidMind Library, paste the code snippet with the model identifier credentials directly into your development source code, replacing this example with your own:
After you have pasted the code snippet into your development source code and executed the code, the Python Library API will register with ValidMind. You can now use the ValidMind Library to document and test your models, and to upload to the ValidMind Platform.
Initializes the API client instances and calls the /ping endpoint to ensure
-the provided credentials are valid and we can connect to the ValidMind API.
-
-
If the API key and secret are not provided, the client will attempt to
-retrieve them from the environment variables VM_API_KEY and VM_API_SECRET.
-
-
Arguments:
-
-
-
project (str, optional): The project CUID. Alias for model. Defaults to None. [DEPRECATED]
-
model (str, optional): The model CUID. Defaults to None.
-
api_key (str, optional): The API key. Defaults to None.
-
api_secret (str, optional): The API secret. Defaults to None.
-
api_host (str, optional): The API host. Defaults to None.
-
monitoring (bool): The ongoing monitoring flag. Defaults to False.
-
generate_descriptions (bool): Whether to use GenAI to generate test result descriptions. Defaults to True.
-
-
-
Raises:
-
-
-
ValueError: If the API key and secret are not provided
-
-
-
-
-
-
-
-
- def
- reload():
-
-
-
-
-
-
Reconnect to the ValidMind API and reload the project configuration
Initializes a VM Dataset, which can then be passed to other functions
-that can perform additional analysis and tests on the data. This function
-also ensures we are reading a valid dataset type.
-
-
The following dataset types are supported:
-
-
-
Pandas DataFrame
-
Polars DataFrame
-
Numpy ndarray
-
Torch TensorDataset
-
-
-
Arguments:
-
-
-
dataset : dataset from various python libraries
-
model (VMModel): ValidMind model object
-
targets (vm.vm.DatasetTargets): A list of target variables
-
target_column (str): The name of the target column in the dataset
-
feature_columns (list): A list of names of feature columns in the dataset
-
extra_columns (dictionary): A dictionary containing the names of the
-
prediction_column and group_by_columns in the dataset
-
class_labels (dict): A list of class labels for classification problems
-
type (str): The type of dataset (one of DATASET_TYPES)
-
input_id (str): The input ID for the dataset (e.g. "my_dataset"). By default,
-this will be set to dataset but if you are passing this dataset as a
-test input using some other key than dataset, then you should set
-this to the same key.
Initializes a VM Model, which can then be passed to other functions
-that can perform additional analysis and tests on the data. This function
-also ensures we are creating a model supported libraries.
-
-
Arguments:
-
-
-
model: A trained model or VMModel instance
-
input_id (str): The input ID for the model (e.g. "my_model"). By default,
-this will be set to model but if you are passing this model as a
-test input using some other key than model, then you should set
-this to the same key.
-
attributes (dict): A dictionary of model attributes
-
predict_fn (callable): A function that takes an input and returns a prediction
-
**kwargs: Additional arguments to pass to the model
R models must be saved to disk and the filetype depends on the model type...
-Currently we support the following model types:
-
-
-
LogisticRegression glm model in R: saved as an RDS file with saveRDS
-
LinearRegression lm model in R: saved as an RDS file with saveRDS
-
XGBClassifier: saved as a .json or .bin file with xgb.save
-
XGBRegressor: saved as a .json or .bin file with xgb.save
-
-
-
LogisticRegression and LinearRegression models are converted to sklearn models by extracting
-the coefficients and intercept from the R model. XGB models are loaded using the xgboost
-since xgb models saved in .json or .bin format can be loaded directly with either Python or R
-
-
Arguments:
-
-
-
model_path (str): The path to the R model saved as an RDS or XGB file
-
model_type (str): The type of the model (one of R_MODEL_TYPES)
-
-
-
Returns:
-
-
-
vm.vm.Model: A VM Model instance
-
-
-
-
-
-
-
-
- def
- preview_template():
-
-
-
-
-
-
Preview the documentation template for the current project
-
-
This function will display the documentation template for the current project. If
-the project has not been initialized, then an error will be raised.
-
-
Raises:
-
-
-
ValueError: If the project has not been initialized
Collect and run all the tests associated with a template
-
-
This function will analyze the current project's documentation template and collect
-all the tests associated with it into a test suite. It will then run the test
-suite, log the results to the ValidMind API, and display them to the user.
-
-
Arguments:
-
-
-
section (str or list, optional): The section(s) to preview. Defaults to None.
-
send (bool, optional): Whether to send the results to the ValidMind API. Defaults to True.
-
fail_fast (bool, optional): Whether to stop running tests after the first failure. Defaults to False.
-
inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite
-
config: A dictionary of test parameters to override the defaults
-
**kwargs: backwards compatibility for passing in test inputs using keyword arguments
-
-
-
Returns:
-
-
-
TestSuite or dict: The completed TestSuite instance or a dictionary of TestSuites if section is a list.
-
-
-
Raises:
-
-
-
ValueError: If the project has not been initialized
Unit metrics are key-value pairs where the key is the metric name and the value is
-a scalar (int or float). These key-value pairs are associated with the currently
-selected model (inventory model in the ValidMind Platform) and keys can be logged
-to over time to create a history of the metric. On the ValidMind Platform, these metrics
-will be used to create plots/visualizations for documentation and dashboards etc.
-
-
Arguments:
-
-
-
key (str): The metric key
-
value (float): The metric value
-
inputs (list, optional): A list of input IDs that were used to compute the metric.
-
params (dict, optional): Dictionary of parameters used to compute the metric.
-
recorded_at (str, optional): The timestamp of the metric. Server will use
-current time if not provided.
-
thresholds (dict, optional): Dictionary of thresholds for the metric.
Gets a TestSuite object for the current project or a specific test suite
-
-
This function provides an interface to retrieve the TestSuite instance for the
-current project or a specific TestSuite instance identified by test_suite_id.
-The project Test Suite will contain sections for every section in the project's
-documentation template and these Test Suite Sections will contain all the tests
-associated with that template section.
-
-
Arguments:
-
-
-
test_suite_id (str, optional): The test suite name. If not passed, then the
-project's test suite will be returned. Defaults to None.
-
section (str, optional): The section of the documentation template from which
-to retrieve the test suite. This only applies if test_suite_id is None.
-Defaults to None.
-
args: Additional arguments to pass to the TestSuite
-
kwargs: Additional keyword arguments to pass to the TestSuite
This function provides a high level interface for running a test suite. A test suite is
-a collection of tests. This function will automatically find the correct test suite
-class based on the test_suite_id, initialize each of the tests, and run them.
-
-
Arguments:
-
-
-
test_suite_id (str): The test suite name (e.g. 'classifier_full_suite')
-
config (dict, optional): A dictionary of parameters to pass to the tests in the
-test suite. Defaults to None.
-
send (bool, optional): Whether to post the test results to the API. send=False
-is useful for testing. Defaults to True.
-
fail_fast (bool, optional): Whether to stop running tests after the first failure. Defaults to False.
-
inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite e.g. model, dataset
-models etc. These inputs will be accessible by any test in the test suite. See the test
-documentation or vm.describe_test() for more details on the inputs required for each.
-
**kwargs: backwards compatibility for passing in test inputs using keyword arguments
-
-
-
Raises:
-
-
-
ValueError: If the test suite name is not found or if there is an error initializing the test suite
-
-
-
Returns:
-
-
-
TestSuite: the TestSuite instance
-
-
-
-
-
-
-
-
- def
- print_env():
-
-
-
-
-
-
Prints a log of the running environment for debugging.
-
-
Output includes: ValidMind Library version, operating system details, installed dependencies, and the ISO 8601 timestamp at log creation.
-
-
-
-
-
-
-
- def
- tags(*tags):
-
-
-
-
-
-
Decorator for specifying tags for a test.
-
-
Arguments:
-
-
-
*tags: The tags to apply to the test.
-
-
-
-
-
-
-
-
- def
- tasks(*tasks):
-
-
-
-
-
-
Decorator for specifying the task types that a test is designed for.
-
-
Arguments:
-
-
-
*tasks: The task types that the test is designed for.
-
-
-
-
-
-
-
-
- def
- test(func_or_id):
-
-
-
-
-
-
Decorator for creating and registering custom tests
-
-
This decorator registers the function it wraps as a test function within ValidMind
-under the provided ID. Once decorated, the function can be run using the
-validmind.tests.run_test function.
-
-
The function can take two different types of arguments:
-
-
-
Inputs: ValidMind model or dataset (or list of models/datasets). These arguments
-must use the following names: model, models, dataset, datasets.
-
Parameters: Any additional keyword arguments of any type (must have a default
-value) that can have any name.
-
-
-
The function should return one of the following types:
-
-
-
Table: Either a list of dictionaries or a pandas DataFrame
-
Plot: Either a matplotlib figure or a plotly figure
-
Scalar: A single number (int or float)
-
Boolean: A single boolean value indicating whether the test passed or failed
-
-
-
The function may also include a docstring. This docstring will be used and logged
-as the metric's description.
-
-
Arguments:
-
-
-
func: The function to decorate
-
test_id: The identifier for the metric. If not provided, the function name is used.
-
-
-
Returns:
-
-
-
The decorated function.
-
-
-
-
-
-
-
-
- class
- RawData:
-
-
-
-
-
-
Holds raw data for a test result
-
-
-
-
-
-
- RawData(log:bool=False, **kwargs)
-
-
-
-
-
-
Create a new RawData object
-
-
Arguments:
-
-
-
log (bool): If True, log the raw data to ValidMind
-
**kwargs: Keyword arguments to set as attributes e.g.
-RawData(log=True, dataset_duplicates=df_duplicates)
-
-
-
-
-
-
-
-
- def
- inspect(self, show:bool=True):
-
-
-
-
-
-
Inspect the raw data
-
-
-
-
-
-
-
- def
- serialize(self):
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/__version__.html b/docs/_build/validmind/__version__.html
deleted file mode 100644
index 588b7c178..000000000
--- a/docs/_build/validmind/__version__.html
+++ /dev/null
@@ -1,239 +0,0 @@
-
-
-
-
-
-
- validmind.__version__ API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Returns input configuration for the default documentation
-template assigned to this demo model
-
-
The default documentation template uses the following inputs:
-
-
-
raw_dataset
-
train_dataset
-
test_dataset
-
model
-
-
-
We assign the following inputs depending on the input config expected
-by each test:
-
-
-
When a test expects a "dataset" we use the raw_dataset
-
When a tets expects "datasets" we use the train_dataset and test_dataset
-
When a test expects a "model" we use the model
-
When a test expects "model" and "dataset" we use the model and test_dataset
-
The only exception is ClassifierPerformance since that runs twice: once
-with the train_dataset (in sample) and once with the test_dataset (out of sample)
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/datasets/classification/taiwan_credit.html b/docs/_build/validmind/datasets/classification/taiwan_credit.html
deleted file mode 100644
index 7cdba8403..000000000
--- a/docs/_build/validmind/datasets/classification/taiwan_credit.html
+++ /dev/null
@@ -1,271 +0,0 @@
-
-
-
-
-
-
- validmind.datasets.classification.taiwan_credit API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Split a time series DataFrame into train, validation, and test sets.
-
-
Arguments:
-
-
-
df (pandas.DataFrame): The time series DataFrame to be split.
-
split_option (str): The split option to choose from: 'train_test_val' (default) or 'train_test'.
-
train_size (float): The proportion of the dataset to include in the training set. Default is 0.6.
-
test_size (float): The proportion of the dataset to include in the test set. Default is 0.2.
-
-
-
Returns:
-
-
-
train_df (pandas.DataFrame): The training set.
- validation_df (pandas.DataFrame): The validation set (only returned if split_option is 'train_test_val').
- test_df (pandas.DataFrame): The test set.
Split a time series DataFrame into train, validation, and test sets.
-
-
Arguments:
-
-
-
df (pandas.DataFrame): The time series DataFrame to be split.
-
split_option (str): The split option to choose from: 'train_test_val' (default) or 'train_test'.
-
train_size (float): The proportion of the dataset to include in the training set. Default is 0.6.
-
test_size (float): The proportion of the dataset to include in the test set. Default is 0.2.
-
-
-
Returns:
-
-
-
train_df (pandas.DataFrame): The training set.
- validation_df (pandas.DataFrame): The validation set (only returned if split_option is 'train_test_val').
- test_df (pandas.DataFrame): The test set.
-
- class
- TimeSeriesUnivariate(validmind.vm_models.test_suite.test_suite.TestSuite):
-
-
-
-
-
-
This test suite provides a preliminary understanding of the target variable(s)
-used in the time series dataset. It visualizations that present the raw time
-series data and a histogram of the target variable(s).
-
-
The raw time series data provides a visual inspection of the target variable's
-behavior over time. This helps to identify any patterns or trends in the data,
-as well as any potential outliers or anomalies. The histogram of the target
-variable displays the distribution of values, providing insight into the range
-and frequency of values observed in the data.
-
- class
- TimeSeriesMultivariate(validmind.vm_models.test_suite.test_suite.TestSuite):
-
-
-
-
-
-
This test suite provides a preliminary understanding of the features
-and relationship in multivariate dataset. It presents various
-multivariate visualizations that can help identify patterns, trends,
-and relationships between pairs of variables. The visualizations are
-designed to explore the relationships between multiple features
-simultaneously. They allow you to quickly identify any patterns or
-trends in the data, as well as any potential outliers or anomalies.
-The individual feature distribution can also be explored to provide
-insight into the range and frequency of values observed in the data.
-This multivariate analysis test suite aims to provide an overview of
-the data structure and guide further exploration and modeling.
Test IDs are in the format namespace.path_to_module.TestClassOrFuncName[:tag].
-The tag is optional and is used to distinguish between multiple results from the
-same test.
-
-
Arguments:
-
-
-
test_id (str): The test ID in the format namespace.path_to_module.TestName[:tag]
-
test_func (callable, optional): The test function to load. If not provided, the
-test will be loaded from the test provider. Defaults to None.
This function can be used to see test details including the test name, description,
-required inputs and default params. It can also be used to get a dictionary of the
-above information for programmatic use.
-
-
Arguments:
-
-
-
test_id (str, optional): The test ID. Defaults to None.
-
raw (bool, optional): If True, returns a dictionary with the test details.
-Defaults to False.
This function is the main entry point for running tests. It can run simple unit metrics,
-ValidMind and custom tests, composite tests made up of multiple unit metrics and comparison
-tests made up of multiple tests.
-
-
Arguments:
-
-
-
test_id (TestID, optional): Test ID to run. Not required if name and unit_metrics provided.
-
params (dict, optional): Parameters to customize test behavior. See test details for available parameters.
Test providers in ValidMind are responsible for loading tests from different sources,
-such as local files, databases, or remote services. The LocalTestProvider specifically
-loads tests from the local file system.
-
-
To use the LocalTestProvider, you need to provide the root_folder, which is the
-root directory for local tests. The test_id is a combination of the namespace (set
-when registering the test provider) and the path to the test class module, where
-slashes are replaced by dots and the .py extension is left out.
-
-
Example usage:
-
-
# Create an instance of LocalTestProvider with the root folder
-test_provider = LocalTestProvider("/path/to/tests/folder")
-
-# Register the test provider with a namespace
-register_test_provider("my_namespace", test_provider)
-
-# List all tests in the namespace (returns a list of test IDs)
-test_provider.list_tests()
-# this is used by the list_tests() function to aggregate all tests
-# from all test providers
-
-# Load a test using the test_id (namespace + path to test class module)
-test = test_provider.load_test("my_namespace.my_test_class")
-# full path to the test class module is /path/to/tests/folder/my_test_class.py
-
-
-
Attributes:
-
-
-
root_folder (str): The root directory for local tests.
-
-
-
-
-
-
-
- LocalTestProvider(root_folder:str)
-
-
-
-
-
-
Initialize the LocalTestProvider with the given root_folder
-(see class docstring for details)
-
-
Arguments:
-
-
-
root_folder (str): The root directory for local tests.
-
-
-
-
-
-
-
-
- def
- list_tests(self):
-
-
-
-
-
-
List all tests in the given namespace
-
-
Returns:
-
-
-
list: A list of test IDs
-
-
-
-
-
-
-
-
- def
- load_test(self, test_id:str):
-
-
-
-
-
-
Load the test identified by the given test_id.
-
-
Arguments:
-
-
-
test_id (str): The identifier of the test. This corresponds to the relative
-
path of the python file from the root folder, with slashes replaced by dots
-
-
-
Returns:
-
-
-
The test class that matches the last part of the test_id.
-
-
-
Raises:
-
-
-
LocalTestProviderLoadModuleError: If the test module cannot be imported
-
LocalTestProviderLoadTestError: If the test class cannot be found in the module
List all task types and their associated tags, with one row per task type and
-all tags for a task type in one row.
-
-
Returns:
-
-
-
pandas.DataFrame: A DataFrame with 'Task Type' and concatenated 'Tags'.
-
-
-
-
-
-
-
-
- def
- test(func_or_id):
-
-
-
-
-
-
Decorator for creating and registering custom tests
-
-
This decorator registers the function it wraps as a test function within ValidMind
-under the provided ID. Once decorated, the function can be run using the
-run_test function.
-
-
The function can take two different types of arguments:
-
-
-
Inputs: ValidMind model or dataset (or list of models/datasets). These arguments
-must use the following names: model, models, dataset, datasets.
-
Parameters: Any additional keyword arguments of any type (must have a default
-value) that can have any name.
-
-
-
The function should return one of the following types:
-
-
-
Table: Either a list of dictionaries or a pandas DataFrame
-
Plot: Either a matplotlib figure or a plotly figure
-
Scalar: A single number (int or float)
-
Boolean: A single boolean value indicating whether the test passed or failed
-
-
-
The function may also include a docstring. This docstring will be used and logged
-as the metric's description.
-
-
Arguments:
-
-
-
func: The function to decorate
-
test_id: The identifier for the metric. If not provided, the function name is used.
-
-
-
Returns:
-
-
-
The decorated function.
-
-
-
-
-
-
-
-
- def
- tags(*tags):
-
-
-
-
-
-
Decorator for specifying tags for a test.
-
-
Arguments:
-
-
-
*tags: The tags to apply to the test.
-
-
-
-
-
-
-
-
- def
- tasks(*tasks):
-
-
-
-
-
-
Decorator for specifying the task types that a test is designed for.
-
-
Arguments:
-
-
-
*tasks: The task types that the test is designed for.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation.html b/docs/_build/validmind/tests/data_validation.html
deleted file mode 100644
index 59954c50a..000000000
--- a/docs/_build/validmind/tests/data_validation.html
+++ /dev/null
@@ -1,298 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Analyzes time series data using Autocorrelation Function (ACF) and Partial Autocorrelation Function (PACF) plots to
-reveal trends and correlations.
-
-
Purpose
-
-
The ACF (Autocorrelation Function) and PACF (Partial Autocorrelation Function) plot test is employed to analyze
-time series data in machine learning models. It illuminates the correlation of the data over time by plotting the
-correlation of the series with its own lags (ACF), and the correlations after removing effects already accounted
-for by earlier lags (PACF). This information can identify trends, such as seasonality, degrees of autocorrelation,
-and inform the selection of order parameters for AutoRegressive Integrated Moving Average (ARIMA) models.
-
-
Test Mechanism
-
-
The ACFandPACFPlot test accepts a dataset with a time-based index. It first confirms the index is of a datetime
-type, then handles any NaN values. The test subsequently generates ACF and PACF plots for each column in the
-dataset, producing a subplot for each. If the dataset doesn't include key columns, an error is returned.
-
-
Signs of High Risk
-
-
-
Sudden drops in the correlation at a specific lag might signal a model at high risk.
-
Consistent high correlation across multiple lags could also indicate non-stationarity in the data, which may
-suggest that a model estimated on this data won't generalize well to future, unknown data.
-
-
-
Strengths
-
-
-
ACF and PACF plots offer clear graphical representations of the correlations in time series data.
-
These plots are effective at revealing important data characteristics such as seasonality, trends, and
-correlation patterns.
-
The insights from these plots aid in better model configuration, particularly in the selection of ARIMA model
-parameters.
-
-
-
Limitations
-
-
-
ACF and PACF plots are exclusively for time series data and hence, can't be applied to all ML models.
-
These plots require large, consistent datasets as gaps could lead to misleading results.
-
The plots can only represent linear correlations and fail to capture any non-linear relationships within the data.
-
The plots might be difficult for non-experts to interpret and should not replace more advanced analyses.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ADF.html b/docs/_build/validmind/tests/data_validation/ADF.html
deleted file mode 100644
index 8700cb5cd..000000000
--- a/docs/_build/validmind/tests/data_validation/ADF.html
+++ /dev/null
@@ -1,298 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ADF API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the stationarity of a time series dataset using the Augmented Dickey-Fuller (ADF) test.
-
-
Purpose
-
-
The Augmented Dickey-Fuller (ADF) test metric is used to determine the order of integration, i.e., the stationarity
-of a given time series dataset. The stationary property of data is pivotal in many machine learning models as it
-impacts the reliability and effectiveness of predictions and forecasts.
-
-
Test Mechanism
-
-
The ADF test is executed using the adfuller function from the statsmodels library on each feature of the
-dataset. Multiple outputs are generated for each run, including the ADF test statistic and p-value, count of lags
-used, the number of observations considered in the test, critical values at various confidence levels, and the
-information criterion. These results are stored for each feature for subsequent analysis.
-
-
Signs of High Risk
-
-
-
An inflated ADF statistic and high p-value (generally above 0.05) indicate a high risk to the model's performance
-due to the presence of a unit root indicating non-stationarity.
-
Non-stationarity might result in untrustworthy or insufficient forecasts.
-
-
-
Strengths
-
-
-
The ADF test is robust to sophisticated correlations within the data, making it suitable for settings where data
-displays complex stochastic behavior.
-
It provides explicit outputs like test statistics, critical values, and information criterion, enhancing
-understanding and transparency in the model validation process.
-
-
-
Limitations
-
-
-
The ADF test might demonstrate low statistical power, making it challenging to differentiate between a unit root
-and near-unit-root processes, potentially causing false negatives.
-
It assumes the data follows an autoregressive process, which might not always be the case.
-
The test struggles with time series data that have structural breaks.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/AutoAR.html b/docs/_build/validmind/tests/data_validation/AutoAR.html
deleted file mode 100644
index 29f5eb0ad..000000000
--- a/docs/_build/validmind/tests/data_validation/AutoAR.html
+++ /dev/null
@@ -1,305 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.AutoAR API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Automatically identifies the optimal Autoregressive (AR) order for a time series using BIC and AIC criteria.
-
-
Purpose
-
-
The AutoAR test is intended to automatically identify the Autoregressive (AR) order of a time series by utilizing
-the Bayesian Information Criterion (BIC) and Akaike Information Criterion (AIC). AR order is crucial in forecasting
-tasks as it dictates the quantity of prior terms in the sequence to use for predicting the current term. The
-objective is to select the most fitting AR model that encapsulates the trend and seasonality in the time series
-data.
-
-
Test Mechanism
-
-
The test mechanism operates by iterating through a possible range of AR orders up to a defined maximum. An AR model
-is fitted for each order, and the corresponding BIC and AIC are computed. BIC and AIC statistical measures are
-designed to penalize models for complexity, preferring simpler models that fit the data proficiently. To verify the
-stationarity of the time series, the Augmented Dickey-Fuller test is executed. The AR order, BIC, and AIC findings
-are compiled into a dataframe for effortless comparison. Then, the AR order with the smallest BIC is established as
-the desirable order for each variable.
-
-
Signs of High Risk
-
-
-
An augmented Dickey Fuller test p-value > 0.05, indicating the time series isn't stationary, may lead to
-inaccurate results.
-
Problems with the model fitting procedure, such as computational or convergence issues.
-
Continuous selection of the maximum specified AR order may suggest an insufficient set limit.
-
-
-
Strengths
-
-
-
The test independently pinpoints the optimal AR order, thereby reducing potential human bias.
-
It strikes a balance between model simplicity and goodness-of-fit to avoid overfitting.
-
Has the capability to account for stationarity in a time series, an essential aspect for dependable AR modeling.
-
The results are aggregated into a comprehensive table, enabling an easy interpretation.
-
-
-
Limitations
-
-
-
The tests need a stationary time series input.
-
They presume a linear relationship between the series and its lags.
-
The search for the best model is constrained by the maximum AR order supplied in the parameters. Therefore, a low
-max_ar_order could result in subpar outcomes.
-
AIC and BIC may not always agree on the selection of the best model. This potentially requires the user to juggle
-interpretational choices.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/AutoMA.html b/docs/_build/validmind/tests/data_validation/AutoMA.html
deleted file mode 100644
index 7269611d4..000000000
--- a/docs/_build/validmind/tests/data_validation/AutoMA.html
+++ /dev/null
@@ -1,308 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.AutoMA API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Automatically selects the optimal Moving Average (MA) order for each variable in a time series dataset based on
-minimal BIC and AIC values.
-
-
Purpose
-
-
The AutoMA metric serves an essential role of automated decision-making for selecting the optimal Moving Average
-(MA) order for every variable in a given time series dataset. The selection is dependent on the minimalization of
-BIC (Bayesian Information Criterion) and AIC (Akaike Information Criterion); these are established statistical
-tools used for model selection. Furthermore, prior to the commencement of the model fitting process, the algorithm
-conducts a stationarity test (Augmented Dickey-Fuller test) on each series.
-
-
Test Mechanism
-
-
Starting off, the AutoMA algorithm checks whether the max_ma_order parameter has been provided. It consequently
-loops through all variables in the dataset, carrying out the Dickey-Fuller test for stationarity. For each
-stationary variable, it fits an ARIMA model for orders running from 0 to max_ma_order. The result is a list
-showcasing the BIC and AIC values of the ARIMA models based on different orders. The MA order, which yields the
-smallest BIC, is chosen as the 'best MA order' for every single variable. The final results include a table
-summarizing the auto MA analysis and another table listing the best MA order for each variable.
-
-
Signs of High Risk
-
-
-
When a series is non-stationary (p-value>0.05 in the Dickey-Fuller test), the produced result could be inaccurate.
-
Any error that arises in the process of fitting the ARIMA models, especially with a higher MA order, can
-potentially indicate risks and might need further investigation.
-
-
-
Strengths
-
-
-
The metric facilitates automation in the process of selecting the MA order for time series forecasting. This
-significantly saves time and reduces efforts conventionally necessary for manual hyperparameter tuning.
-
The use of both BIC and AIC enhances the likelihood of selecting the most suitable model.
-
The metric ascertains the stationarity of the series prior to model fitting, thus ensuring that the underlying
-assumptions of the MA model are fulfilled.
-
-
-
Limitations
-
-
-
If the time series fails to be stationary, the metric may yield inaccurate results. Consequently, it necessitates
-pre-processing steps to stabilize the series before fitting the ARIMA model.
-
The metric adopts a rudimentary model selection process based on BIC and doesn't consider other potential model
-selection strategies. Depending on the specific dataset, other strategies could be more appropriate.
-
The 'max_ma_order' parameter must be manually input which doesn't always guarantee optimal performance,
-especially when configured too low.
-
The computation time increases with the rise in max_ma_order, hence, the metric may become computationally
-costly for larger values.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/AutoStationarity.html b/docs/_build/validmind/tests/data_validation/AutoStationarity.html
deleted file mode 100644
index cd2b43534..000000000
--- a/docs/_build/validmind/tests/data_validation/AutoStationarity.html
+++ /dev/null
@@ -1,306 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.AutoStationarity API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Automates Augmented Dickey-Fuller test to assess stationarity across multiple time series in a DataFrame.
-
-
Purpose
-
-
The AutoStationarity metric is intended to automatically detect and evaluate the stationary nature of each time
-series in a DataFrame. It incorporates the Augmented Dickey-Fuller (ADF) test, a statistical approach used to
-assess stationarity. Stationarity is a fundamental property suggesting that statistic features like mean and
-variance remain unchanged over time. This is necessary for many time-series models.
-
-
Test Mechanism
-
-
The mechanism for the AutoStationarity test involves applying the Augmented Dicky-Fuller test to each time series
-within the given dataframe to assess if they are stationary. Every series in the dataframe is looped, using the ADF
-test up to a defined maximum order (configurable and by default set to 5). The p-value resulting from the ADF test
-is compared against a predetermined threshold (also configurable and by default set to 0.05). The time series is
-deemed stationary at its current differencing order if the p-value is less than the threshold.
-
-
Signs of High Risk
-
-
-
A significant number of series not achieving stationarity even at the maximum order of differencing can indicate
-high risk or potential failure in the model.
-
This could suggest the series may not be appropriately modeled by a stationary process, hence other modeling
-approaches might be required.
-
-
-
Strengths
-
-
-
The key strength in this metric lies in the automation of the ADF test, enabling mass stationarity analysis
-across various time series and boosting the efficiency and credibility of the analysis.
-
The utilization of the ADF test, a widely accepted method for testing stationarity, lends authenticity to the
-results derived.
-
The introduction of the max order and threshold parameters give users the autonomy to determine their preferred
-levels of stringency in the tests.
-
-
-
Limitations
-
-
-
The Augmented Dickey-Fuller test and the stationarity test are not without their limitations. These tests are
-premised on the assumption that the series can be modeled by an autoregressive process, which may not always hold
-true.
-
The stationarity check is highly sensitive to the choice of threshold for the significance level; an extremely
-high or low threshold could lead to incorrect results regarding the stationarity properties.
-
There's also a risk of over-differencing if the maximum order is set too high, which could induce unnecessary
-cycles.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/BivariateScatterPlots.html b/docs/_build/validmind/tests/data_validation/BivariateScatterPlots.html
deleted file mode 100644
index 6c0a52e5c..000000000
--- a/docs/_build/validmind/tests/data_validation/BivariateScatterPlots.html
+++ /dev/null
@@ -1,300 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.BivariateScatterPlots API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Generates bivariate scatterplots to visually inspect relationships between pairs of numerical predictor variables
-in machine learning classification tasks.
-
-
Purpose
-
-
This function is intended for visual inspection and monitoring of relationships between pairs of numerical
-variables in a machine learning model targeting classification tasks. It helps in understanding how predictor
-variables (features) interact with each other, which can inform feature selection, model-building strategies, and
-identify potential biases or irregularities in the data.
-
-
Test Mechanism
-
-
The function creates scatter plots for each pair of numerical features in the dataset. It first filters out
-non-numerical and binary features, ensuring the plots focus on meaningful numerical relationships. The resulting
-scatterplots are color-coded uniformly to avoid visual distraction, and the function returns a tuple of Plotly
-figure objects, each representing a scatter plot for a pair of features.
-
-
Signs of High Risk
-
-
-
Visual patterns suggesting non-linear relationships, multicollinearity, clustering, or outlier points in the
-scatter plots.
-
Such issues could affect the assumptions and performance of certain models, especially those assuming linearity,
-like logistic regression.
-
-
-
Strengths
-
-
-
Scatterplots provide an intuitive and visual tool to explore relationships between two variables.
-
They are useful for identifying outliers, variable associations, and trends, including non-linear patterns.
-
Supports visualization of binary or multi-class classification datasets, focusing on numerical features.
-
-
-
Limitations
-
-
-
Scatterplots are limited to bivariate analysis, showing relationships between only two variables at a time.
-
Not ideal for very large datasets where overlapping points can reduce the clarity of the visualization.
-
Scatterplots are exploratory tools and do not provide quantitative measures of model quality or performance.
-
Interpretation is subjective and relies on the domain knowledge and judgment of the viewer.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/BoxPierce.html b/docs/_build/validmind/tests/data_validation/BoxPierce.html
deleted file mode 100644
index 3de013fb5..000000000
--- a/docs/_build/validmind/tests/data_validation/BoxPierce.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.BoxPierce API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Detects autocorrelation in time-series data through the Box-Pierce test to validate model performance.
-
-
Purpose
-
-
The Box-Pierce test is utilized to detect the presence of autocorrelation in a time-series dataset.
-Autocorrelation, or serial correlation, refers to the degree of similarity between observations based on the
-temporal spacing between them. This test is essential for affirming the quality of a time-series model by ensuring
-that the error terms in the model are random and do not adhere to a specific pattern.
-
-
Test Mechanism
-
-
The implementation of the Box-Pierce test involves calculating a test statistic along with a corresponding p-value
-derived from the dataset features. These quantities are used to test the null hypothesis that posits the data to be
-independently distributed. This is achieved by iterating over every feature column in the time-series data and
-applying the acorr_ljungbox function of the statsmodels library. The function yields the Box-Pierce test
-statistic as well as the respective p-value, all of which are cached as test results.
-
-
Signs of High Risk
-
-
-
A low p-value, typically under 0.05 as per statistical convention, throws the null hypothesis of independence
-into question. This implies that the dataset potentially houses autocorrelations, thus indicating a high-risk
-scenario concerning model performance.
-
Large Box-Pierce test statistic values may indicate the presence of autocorrelation.
-
-
-
Strengths
-
-
-
Detects patterns in data that are supposed to be random, thereby ensuring no underlying autocorrelation.
-
Can be computed efficiently given its low computational complexity.
-
Can be widely applied to most regression problems, making it very versatile.
-
-
-
Limitations
-
-
-
Assumes homoscedasticity (constant variance) and normality of residuals, which may not always be the case in
-real-world datasets.
-
May exhibit reduced power for detecting complex autocorrelation schemes such as higher-order or negative
-correlations.
-
It only provides a general indication of the existence of autocorrelation, without providing specific insights
-into the nature or patterns of the detected autocorrelation.
-
In the presence of trends or seasonal patterns, the Box-Pierce test may yield misleading results.
-
Applicability is limited to time-series data, which limits its overall utility.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ChiSquaredFeaturesTable.html b/docs/_build/validmind/tests/data_validation/ChiSquaredFeaturesTable.html
deleted file mode 100644
index 527675294..000000000
--- a/docs/_build/validmind/tests/data_validation/ChiSquaredFeaturesTable.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ChiSquaredFeaturesTable API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the statistical association between categorical features and a target variable using the Chi-Squared test.
-
-
Purpose
-
-
The ChiSquaredFeaturesTable function is designed to evaluate the relationship between categorical features and a
-target variable in a dataset. It performs a Chi-Squared test of independence for each categorical feature to
-determine whether a statistically significant association exists with the target variable. This is particularly
-useful in Model Risk Management for understanding the relevance of features and identifying potential biases in a
-classification model.
-
-
Test Mechanism
-
-
The function creates a contingency table for each categorical feature and the target variable, then applies the
-Chi-Squared test to compute the Chi-squared statistic and the p-value. The results for each feature include the
-variable name, Chi-squared statistic, p-value, p-value threshold, and a pass/fail status based on whether the
-p-value is below the specified threshold. The output is a DataFrame summarizing these results, sorted by p-value to
-highlight the most statistically significant associations.
-
-
Signs of High Risk
-
-
-
High p-values (greater than the set threshold) indicate a lack of significant association between a feature and
-the target variable, resulting in a 'Fail' status.
-
Features with a 'Fail' status might not be relevant for the model, which could negatively impact model
-performance.
-
-
-
Strengths
-
-
-
Provides a clear, statistical assessment of the relationship between categorical features and the target variable.
-
Produces an easily interpretable summary with a 'Pass/Fail' outcome for each feature, helping in feature
-selection.
-
The p-value threshold is adjustable, allowing for flexibility in statistical rigor.
-
-
-
Limitations
-
-
-
Assumes the dataset is tabular and consists of categorical variables, which may not be suitable for all datasets.
-
The test is designed for classification tasks and is not applicable to regression problems.
-
As with all hypothesis tests, the Chi-Squared test can only detect associations, not causal relationships.
-
The choice of p-value threshold can affect the interpretation of feature relevance, and different thresholds may
-lead to different conclusions.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ClassImbalance.html b/docs/_build/validmind/tests/data_validation/ClassImbalance.html
deleted file mode 100644
index 134c2334f..000000000
--- a/docs/_build/validmind/tests/data_validation/ClassImbalance.html
+++ /dev/null
@@ -1,310 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ClassImbalance API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates and quantifies class distribution imbalance in a dataset used by a machine learning model.
-
-
Purpose
-
-
The Class Imbalance test is designed to evaluate the distribution of target classes in a dataset that's utilized by
-a machine learning model. Specifically, it aims to ensure that the classes aren't overly skewed, which could lead
-to bias in the model's predictions. It's crucial to have a balanced training dataset to avoid creating a model
-that's biased with high accuracy for the majority class and low accuracy for the minority class.
-
-
Test Mechanism
-
-
This Class Imbalance test operates by calculating the frequency (expressed as a percentage) of each class in the
-target column of the dataset. It then checks whether each class appears in at least a set minimum percentage of the
-total records. This minimum percentage is a modifiable parameter, but the default value is set to 10%.
-
-
Signs of High Risk
-
-
-
Any class that represents less than the pre-set minimum percentage threshold is marked as high risk, implying a
-potential class imbalance.
-
The function provides a pass/fail outcome for each class based on this criterion.
-
Fundamentally, if any class fails this test, it's highly likely that the dataset possesses imbalanced class
-distribution.
-
-
-
Strengths
-
-
-
The test can spot under-represented classes that could affect the efficiency of a machine learning model.
-
The calculation is straightforward and swift.
-
The test is highly informative because it not only spots imbalance, but it also quantifies the degree of
-imbalance.
-
The adjustable threshold enables flexibility and adaptation to differing use-cases or domain-specific needs.
-
The test creates a visually insightful plot showing the classes and their corresponding proportions, enhancing
-interpretability and comprehension of the data.
-
-
-
Limitations
-
-
-
The test might struggle to perform well or provide vital insights for datasets with a high number of classes. In
-such cases, the imbalance could be inevitable due to the inherent class distribution.
-
Sensitivity to the threshold value might result in faulty detection of imbalance if the threshold is set
-excessively high.
-
Regardless of the percentage threshold, it doesn't account for varying costs or impacts of misclassifying
-different classes, which might fluctuate based on specific applications or domains.
-
While it can identify imbalances in class distribution, it doesn't provide direct methods to address or correct
-these imbalances.
-
The test is only applicable for classification operations and unsuitable for regression or clustering tasks.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/DatasetDescription.html b/docs/_build/validmind/tests/data_validation/DatasetDescription.html
deleted file mode 100644
index 603f21b99..000000000
--- a/docs/_build/validmind/tests/data_validation/DatasetDescription.html
+++ /dev/null
@@ -1,386 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.DatasetDescription API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Returns a collection of histograms for a numerical or categorical column.
-We store different combinations of bin sizes to allow analyzing the data better
-
-
Will be used in favor of _get_histogram in the future
-
-
-
-
-
-
-
- def
- describe_column(df, column):
-
-
-
-
-
-
Gets descriptive statistics for a single column in a Pandas DataFrame.
Provides comprehensive analysis and statistical summaries of each column in a machine learning model's dataset.
-
-
Purpose
-
-
The test depicted in the script is meant to run a comprehensive analysis on a Machine Learning model's datasets.
-The test or metric is implemented to obtain a complete summary of the columns in the dataset, including vital
-statistics of each column such as count, distinct values, missing values, histograms for numerical, categorical,
-boolean, and text columns. This summary gives a comprehensive overview of the dataset to better understand the
-characteristics of the data that the model is trained on or evaluates.
-
-
Test Mechanism
-
-
The DatasetDescription class accomplishes the purpose as follows: firstly, the test method "run" infers the data
-type of each column in the dataset and stores the details (id, column type). For each column, the
-"describe_column" method is invoked to collect statistical information about the column, including count,
-missing value count and its proportion to the total, unique value count, and its proportion to the total. Depending
-on the data type of a column, histograms are generated that reflect the distribution of data within the column.
-Numerical columns use the "get_numerical_histograms" method to calculate histogram distribution, whereas for
-categorical, boolean and text columns, a histogram is computed with frequencies of each unique value in the
-datasets. For unsupported types, an error is raised. Lastly, a summary table is built to aggregate all the
-statistical insights and histograms of the columns in a dataset.
-
-
Signs of High Risk
-
-
-
High ratio of missing values to total values in one or more columns which may impact the quality of the
-predictions.
-
Unsupported data types in dataset columns.
-
Large number of unique values in the dataset's columns which might make it harder for the model to establish
-patterns.
-
Extreme skewness or irregular distribution of data as reflected in the histograms.
-
-
-
Strengths
-
-
-
Provides a detailed analysis of the dataset with versatile summaries like count, unique values, histograms, etc.
-
Flexibility in handling different types of data: numerical, categorical, boolean, and text.
-
Useful in detecting problems in the dataset like missing values, unsupported data types, irregular data
-distribution, etc.
-
The summary gives a comprehensive understanding of dataset features allowing developers to make informed
-decisions.
-
-
-
Limitations
-
-
-
The computation can be expensive from a resource standpoint, particularly for large datasets with numerous columns.
-
The histograms use an arbitrary number of bins which may not be the optimal number of bins for specific data
-distribution.
-
Unsupported data types for columns will raise an error which may limit evaluating the dataset.
-
Columns with all null or missing values are not included in histogram computation.
-
This test only validates the quality of the dataset but doesn't address the model's performance directly.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/DatasetSplit.html b/docs/_build/validmind/tests/data_validation/DatasetSplit.html
deleted file mode 100644
index 54aa70a42..000000000
--- a/docs/_build/validmind/tests/data_validation/DatasetSplit.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.DatasetSplit API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates and visualizes the distribution proportions among training, testing, and validation datasets of an ML
-model.
-
-
Purpose
-
-
The DatasetSplit test is designed to evaluate and visualize the distribution of data among training, testing, and
-validation datasets, if available, within a given machine learning model. The main purpose is to assess whether the
-model's datasets are split appropriately, as an imbalanced split might affect the model's ability to learn from the
-data and generalize to unseen data.
-
-
Test Mechanism
-
-
The DatasetSplit test first calculates the total size of all available datasets in the model. Then, for each
-individual dataset, the methodology involves determining the size of the dataset and its proportion relative to the
-total size. The results are then conveniently summarized in a table that shows dataset names, sizes, and
-proportions. Absolute size and proportion of the total dataset size are displayed for each individual dataset.
-
-
Signs of High Risk
-
-
-
A very small training dataset, which may result in the model not learning enough from the data.
-
A very large training dataset and a small test dataset, which may lead to model overfitting and poor
-generalization to unseen data.
-
A small or non-existent validation dataset, which might complicate the model's performance assessment.
-
-
-
Strengths
-
-
-
The DatasetSplit test provides a clear, understandable visualization of dataset split proportions, which can
-highlight any potential imbalance in dataset splits quickly.
-
It covers a wide range of task types including classification, regression, and text-related tasks.
-
The metric is not tied to any specific data type and is applicable to tabular data, time series data, or text
-data.
-
-
-
Limitations
-
-
-
The DatasetSplit test does not provide any insight into the quality or diversity of the data within each split,
-just the size and proportion.
-
The test does not give any recommendations or adjustments for imbalanced datasets.
-
Potential lack of compatibility with more complex modes of data splitting (for example, stratified or time-based
-splits) could limit the applicability of this test.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/DescriptiveStatistics.html b/docs/_build/validmind/tests/data_validation/DescriptiveStatistics.html
deleted file mode 100644
index 1230fb2e1..000000000
--- a/docs/_build/validmind/tests/data_validation/DescriptiveStatistics.html
+++ /dev/null
@@ -1,339 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.DescriptiveStatistics API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Performs a detailed descriptive statistical analysis of both numerical and categorical data within a model's
-dataset.
-
-
Purpose
-
-
The purpose of the Descriptive Statistics metric is to provide a comprehensive summary of both numerical and
-categorical data within a dataset. This involves statistics such as count, mean, standard deviation, minimum and
-maximum values for numerical data. For categorical data, it calculates the count, number of unique values, most
-common value and its frequency, and the proportion of the most frequent value relative to the total. The goal is to
-visualize the overall distribution of the variables in the dataset, aiding in understanding the model's behavior
-and predicting its performance.
-
-
Test Mechanism
-
-
The testing mechanism utilizes two in-built functions of pandas dataframes: describe() for numerical fields and
-value_counts() for categorical fields. The describe() function pulls out several summary statistics, while
-value_counts() accounts for unique values. The resulting data is formatted into two distinct tables, one for
-numerical and another for categorical variable summaries. These tables provide a clear summary of the main
-characteristics of the variables, which can be instrumental in assessing the model's performance.
-
-
Signs of High Risk
-
-
-
Skewed data or significant outliers can represent high risk. For numerical data, this may be reflected via a
-significant difference between the mean and median (50% percentile).
-
For categorical data, a lack of diversity (low count of unique values), or overdominance of a single category
-(high frequency of the top value) can indicate high risk.
-
-
-
Strengths
-
-
-
Provides a comprehensive summary of the dataset, shedding light on the distribution and characteristics of the
-variables under consideration.
-
It is a versatile and robust method, applicable to both numerical and categorical data.
-
Helps highlight crucial anomalies such as outliers, extreme skewness, or lack of diversity, which are vital in
-understanding model behavior during testing and validation.
-
-
-
Limitations
-
-
-
While this metric offers a high-level overview of the data, it may fail to detect subtle correlations or complex
-patterns.
-
Does not offer any insights on the relationship between variables.
-
Alone, descriptive statistics cannot be used to infer properties about future unseen data.
-
Should be used in conjunction with other statistical tests to provide a comprehensive understanding of the
-model's data.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/DickeyFullerGLS.html b/docs/_build/validmind/tests/data_validation/DickeyFullerGLS.html
deleted file mode 100644
index d7f55b645..000000000
--- a/docs/_build/validmind/tests/data_validation/DickeyFullerGLS.html
+++ /dev/null
@@ -1,302 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.DickeyFullerGLS API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses stationarity in time series data using the Dickey-Fuller GLS test to determine the order of integration.
-
-
Purpose
-
-
The Dickey-Fuller GLS (DFGLS) test is utilized to determine the order of integration in time series data. For
-machine learning models dealing with time series and forecasting, this metric evaluates the existence of a unit
-root, thereby checking whether a time series is non-stationary. This analysis is a crucial initial step when
-dealing with time series data.
-
-
Test Mechanism
-
-
This code implements the Dickey-Fuller GLS unit root test on each attribute of the dataset. This process involves
-iterating through every column of the dataset and applying the DFGLS test to assess the presence of a unit root.
-The resulting information, including the test statistic ('stat'), the p-value ('pvalue'), the quantity of lagged
-differences utilized in the regression ('usedlag'), and the number of observations ('nobs'), is subsequently stored.
-
-
Signs of High Risk
-
-
-
A high p-value for the DFGLS test represents a high risk. Specifically, a p-value above a typical threshold of
-0.05 suggests that the time series data is quite likely to be non-stationary, thus presenting a high risk for
-generating unreliable forecasts.
-
-
-
Strengths
-
-
-
The Dickey-Fuller GLS test is a potent tool for checking the stationarity of time series data.
-
It helps to verify the assumptions of the models before the actual construction of the machine learning models
-proceeds.
-
The results produced by this metric offer a clear insight into whether the data is appropriate for specific
-machine learning models, especially those demanding the stationarity of time series data.
-
-
-
Limitations
-
-
-
Despite its benefits, the DFGLS test does present some drawbacks. It can potentially lead to inaccurate
-conclusions if the time series data incorporates a structural break.
-
If the time series tends to follow a trend while still being stationary, the test might misinterpret it,
-necessitating further detrending.
-
The test also presents challenges when dealing with shorter time series data or volatile data, not producing
-reliable results in these cases.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/Duplicates.html b/docs/_build/validmind/tests/data_validation/Duplicates.html
deleted file mode 100644
index 6603aa3ee..000000000
--- a/docs/_build/validmind/tests/data_validation/Duplicates.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.Duplicates API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Tests dataset for duplicate entries, ensuring model reliability via data quality verification.
-
-
Purpose
-
-
The 'Duplicates' test is designed to check for duplicate rows within the dataset provided to the model. It serves
-as a measure of data quality, ensuring that the model isn't merely memorizing duplicate entries or being swayed by
-redundant information. This is an important step in the pre-processing of data for both classification and
-regression tasks.
-
-
Test Mechanism
-
-
This test operates by checking each row for duplicates in the dataset. If a text column is specified in the
-dataset, the test is conducted on this column; if not, the test is run on all feature columns. The number and
-percentage of duplicates are calculated and returned in a DataFrame. Additionally, a test is passed if the total
-count of duplicates falls below a specified minimum threshold.
-
-
Signs of High Risk
-
-
-
A high number of duplicate rows in the dataset, which can lead to overfitting where the model performs well on
-the training data but poorly on unseen data.
-
A high percentage of duplicate rows in the dataset, indicating potential problems with data collection or
-processing.
-
-
-
Strengths
-
-
-
Assists in improving the reliability of the model's training process by ensuring the training data is not
-contaminated with duplicate entries, which can distort statistical analyses.
-
Provides both absolute numbers and percentage values of duplicate rows, giving a thorough overview of data
-quality.
-
Highly customizable as it allows for setting a user-defined minimum threshold to determine if the test has been
-passed.
-
-
-
Limitations
-
-
-
Does not distinguish between benign duplicates (i.e., coincidental identical entries in different rows) and
-problematic duplicates originating from data collection or processing errors.
-
The test becomes more computationally intensive as the size of the dataset increases, which might not be suitable
-for very large datasets.
-
Can only check for exact duplicates and may miss semantically similar information packaged differently.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/EngleGrangerCoint.html b/docs/_build/validmind/tests/data_validation/EngleGrangerCoint.html
deleted file mode 100644
index 7ef6a316f..000000000
--- a/docs/_build/validmind/tests/data_validation/EngleGrangerCoint.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.EngleGrangerCoint API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the degree of co-movement between pairs of time series data using the Engle-Granger cointegration test.
-
-
Purpose
-
-
The intent of this Engle-Granger cointegration test is to explore and quantify the degree of co-movement between
-pairs of time series variables in a dataset. This is particularly useful in enhancing the accuracy of predictive
-regressions whenever the underlying variables are co-integrated, i.e., they move together over time.
-
-
Test Mechanism
-
-
The test first drops any non-applicable values from the input dataset and then iterates over each pair of variables
-to apply the Engle-Granger cointegration test. The test generates a 'p' value, which is then compared against a
-pre-specified threshold (0.05 by default). The pair is labeled as 'Cointegrated' if the 'p' value is less than or
-equal to the threshold or 'Not cointegrated' otherwise. A summary table is returned by the metric showing
-cointegration results for each variable pair.
-
-
Signs of High Risk
-
-
-
A significant number of hypothesized cointegrated variables do not pass the test.
-
A considerable number of 'p' values are close to the threshold, indicating minor data fluctuations can switch the
-decision between 'Cointegrated' and 'Not cointegrated'.
-
-
-
Strengths
-
-
-
Provides an effective way to analyze relationships between time series, particularly in contexts where it's
-essential to check if variables move together in a statistically significant manner.
-
Useful in various domains, especially finance or economics, where predictive models often hinge on understanding
-how different variables move together over time.
-
-
-
Limitations
-
-
-
Assumes that the time series are integrated of the same order, which isn't always true in multivariate time
-series datasets.
-
The presence of non-stationary characteristics in the series or structural breaks can result in falsely positive
-or negative cointegration results.
-
May not perform well for small sample sizes due to lack of statistical power and should be supplemented with
-other predictive indicators for a more robust model evaluation.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/FeatureTargetCorrelationPlot.html b/docs/_build/validmind/tests/data_validation/FeatureTargetCorrelationPlot.html
deleted file mode 100644
index 28221aa44..000000000
--- a/docs/_build/validmind/tests/data_validation/FeatureTargetCorrelationPlot.html
+++ /dev/null
@@ -1,302 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.FeatureTargetCorrelationPlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Visualizes the correlation between input features and the model's target output in a color-coded horizontal bar
-plot.
-
-
Purpose
-
-
This test is designed to graphically illustrate the correlations between distinct input features and the target
-output of a Machine Learning model. Understanding how each feature influences the model's predictions is crucial—a
-higher correlation indicates a stronger influence of the feature on the target variable. This correlation study is
-especially advantageous during feature selection and for comprehending the model's operation.
-
-
Test Mechanism
-
-
This FeatureTargetCorrelationPlot test computes and presents the correlations between the features and the target
-variable using a specific dataset. These correlations are calculated and are then graphically represented in a
-horizontal bar plot, color-coded based on the strength of the correlation. A hovering template can also be utilized
-for informative tooltips. It is possible to specify the features to be analyzed and adjust the graph's height
-according to need.
-
-
Signs of High Risk
-
-
-
There are no strong correlations (either positive or negative) between features and the target variable. This
-could suggest high risk as the supplied features do not appear to significantly impact the prediction output.
-
The presence of duplicated correlation values might hint at redundancy in the feature set.
-
-
-
Strengths
-
-
-
Provides visual assistance to interpreting correlations more effectively.
-
Gives a clear and simple tour of how each feature affects the model's target variable.
-
Beneficial for feature selection and grasping the model's prediction nature.
-
Precise correlation values for each feature are offered by the hover template, contributing to a granular-level
-comprehension.
-
-
-
Limitations
-
-
-
The test only accepts numerical data, meaning variables of other types need to be prepared beforehand.
-
The plot assumes all correlations to be linear, thus non-linear relationships might not be captured effectively.
-
Not apt for models that employ complex feature interactions, like Decision Trees or Neural Networks, as the test
-may not accurately reflect their importance.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/HighCardinality.html b/docs/_build/validmind/tests/data_validation/HighCardinality.html
deleted file mode 100644
index d2f7e5b83..000000000
--- a/docs/_build/validmind/tests/data_validation/HighCardinality.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.HighCardinality API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the number of unique values in categorical columns to detect high cardinality and potential overfitting.
-
-
Purpose
-
-
The “High Cardinality” test is used to evaluate the number of unique values present in the categorical columns of a
-dataset. In this context, high cardinality implies the presence of a large number of unique, non-repetitive values
-in the dataset.
-
-
Test Mechanism
-
-
The test first infers the dataset's type and then calculates an initial numeric threshold based on the test
-parameters. It only considers columns classified as "Categorical". For each of these columns, the number of
-distinct values (n_distinct) and the percentage of distinct values (p_distinct) are calculated. The test will pass
-if n_distinct is less than the calculated numeric threshold. Lastly, the results, which include details such as
-column name, number of distinct values, and pass/fail status, are compiled into a table.
-
-
Signs of High Risk
-
-
-
A large number of distinct values (high cardinality) in one or more categorical columns implies a high risk.
-
A column failing the test (n_distinct >= num_threshold) is another indicator of high risk.
-
-
-
Strengths
-
-
-
The High Cardinality test is effective in early detection of potential overfitting and unwanted noise.
-
It aids in identifying potential outliers and inconsistencies, thereby improving data quality.
-
The test can be applied to both classification and regression task types, demonstrating its versatility.
-
-
-
Limitations
-
-
-
The test is restricted to only "Categorical" data types and is thus not suitable for numerical or continuous
-features, limiting its scope.
-
The test does not consider the relevance or importance of unique values in categorical features, potentially
-causing it to overlook critical data points.
-
The threshold (both number and percent) used for the test is static and may not be optimal for diverse datasets
-and varied applications. Further mechanisms to adjust and refine this threshold could enhance its effectiveness.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/HighPearsonCorrelation.html b/docs/_build/validmind/tests/data_validation/HighPearsonCorrelation.html
deleted file mode 100644
index 694c563b7..000000000
--- a/docs/_build/validmind/tests/data_validation/HighPearsonCorrelation.html
+++ /dev/null
@@ -1,302 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.HighPearsonCorrelation API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Identifies highly correlated feature pairs in a dataset suggesting feature redundancy or multicollinearity.
-
-
Purpose
-
-
The High Pearson Correlation test measures the linear relationship between features in a dataset, with the main
-goal of identifying high correlations that might indicate feature redundancy or multicollinearity. Identification
-of such issues allows developers and risk management teams to properly deal with potential impacts on the machine
-learning model's performance and interpretability.
-
-
Test Mechanism
-
-
The test works by generating pairwise Pearson correlations for all features in the dataset, then sorting and
-eliminating duplicate and self-correlations. It assigns a Pass or Fail based on whether the absolute value of the
-correlation coefficient surpasses a pre-set threshold (defaulted at 0.3). It lastly returns the top n strongest
-correlations regardless of passing or failing status (where n is 10 by default but can be configured by passing the
-top_n_correlations parameter).
-
-
Signs of High Risk
-
-
-
A high risk indication would be the presence of correlation coefficients exceeding the threshold.
-
If the features share a strong linear relationship, this could lead to potential multicollinearity and model
-overfitting.
-
Redundancy of variables can undermine the interpretability of the model due to uncertainty over the authenticity
-of individual variable's predictive power.
-
-
-
Strengths
-
-
-
Provides a quick and simple means of identifying relationships between feature pairs.
-
Generates a transparent output that displays pairs of correlated variables, the Pearson correlation coefficient,
-and a Pass or Fail status for each.
-
Aids in early identification of potential multicollinearity issues that may disrupt model training.
-
-
-
Limitations
-
-
-
Can only delineate linear relationships, failing to shed light on nonlinear relationships or dependencies.
-
Sensitive to outliers where a few outliers could notably affect the correlation coefficient.
-
Limited to identifying redundancy only within feature pairs; may fail to spot more complex relationships among
-three or more variables.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/IQROutliersBarPlot.html b/docs/_build/validmind/tests/data_validation/IQROutliersBarPlot.html
deleted file mode 100644
index 3d267e5b9..000000000
--- a/docs/_build/validmind/tests/data_validation/IQROutliersBarPlot.html
+++ /dev/null
@@ -1,328 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.IQROutliersBarPlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Visualizes outlier distribution across percentiles in numerical data using the Interquartile Range (IQR) method.
-
-
Purpose
-
-
The InterQuartile Range Outliers Bar Plot (IQROutliersBarPlot) metric aims to visually analyze and evaluate the
-extent of outliers in numeric variables based on percentiles. Its primary purpose is to clarify the dataset's
-distribution, flag possible abnormalities in it, and gauge potential risks associated with processing potentially
-skewed data, which can affect the machine learning model's predictive prowess.
-
-
Test Mechanism
-
-
The examination invokes a series of steps:
-
-
-
For every numeric feature in the dataset, the 25th percentile (Q1) and 75th percentile (Q3) are calculated
-before deriving the Interquartile Range (IQR), the difference between Q1 and Q3.
-
Subsequently, the metric calculates the lower and upper thresholds by subtracting Q1 from the threshold times
-IQR and adding Q3 to threshold times IQR, respectively. The default threshold is set at 1.5.
-
Any value in the feature that falls below the lower threshold or exceeds the upper threshold is labeled as an
-outlier.
-
The number of outliers are tallied for different percentiles, such as [0-25], [25-50], [50-75], and [75-100].
-
These counts are employed to construct a bar plot for the feature, showcasing the distribution of outliers
-across different percentiles.
-
-
-
Signs of High Risk
-
-
-
A prevalence of outliers in the data, potentially skewing its distribution.
-
Outliers dominating higher percentiles (75-100) which implies the presence of extreme values, capable of severely
-influencing the model's performance.
-
Certain features harboring most of their values as outliers, which signifies that these features might not
-contribute positively to the model's forecasting ability.
-
-
-
Strengths
-
-
-
Effectively identifies outliers in the data through visual means, facilitating easier comprehension and offering
-insights into the outliers' possible impact on the model.
-
Provides flexibility by accommodating all numeric features or a chosen subset.
-
Task-agnostic in nature; it is viable for both classification and regression tasks.
-
Can handle large datasets as its operation does not hinge on computationally heavy operations.
-
-
-
Limitations
-
-
-
Its application is limited to numerical variables and does not extend to categorical ones.
-
Only reveals the presence and distribution of outliers and does not provide insights into how these outliers
-might affect the model's predictive performance.
-
The assumption that data is unimodal and symmetric may not always hold true. In cases with non-normal
-distributions, the results can be misleading.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/IQROutliersTable.html b/docs/_build/validmind/tests/data_validation/IQROutliersTable.html
deleted file mode 100644
index f879f7a71..000000000
--- a/docs/_build/validmind/tests/data_validation/IQROutliersTable.html
+++ /dev/null
@@ -1,320 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.IQROutliersTable API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Determines and summarizes outliers in numerical features using the Interquartile Range method.
-
-
Purpose
-
-
The "Interquartile Range Outliers Table" (IQROutliersTable) metric is designed to identify and summarize outliers
-within numerical features of a dataset using the Interquartile Range (IQR) method. This exercise is crucial in the
-pre-processing of data because outliers can substantially distort statistical analysis and impact the performance
-of machine learning models.
-
-
Test Mechanism
-
-
The IQR, which is the range separating the first quartile (25th percentile) from the third quartile (75th
-percentile), is calculated for each numerical feature within the dataset. An outlier is defined as a data point
-falling below the "Q1 - 1.5 * IQR" or above "Q3 + 1.5 * IQR" range. The test computes the number of outliers and
-their summary statistics (minimum, 25th percentile, median, 75th percentile, and maximum values) for each numerical
-feature. If no specific features are chosen, the test applies to all numerical features in the dataset. The default
-outlier threshold is set to 1.5 but can be customized by the user.
-
-
Signs of High Risk
-
-
-
A large number of outliers in multiple features.
-
Outliers significantly distanced from the mean value of variables.
-
Extremely high or low outlier values indicative of data entry errors or other data quality issues.
-
-
-
Strengths
-
-
-
Provides a comprehensive summary of outliers for each numerical feature, helping pinpoint features with potential
-quality issues.
-
The IQR method is robust to extremely high or low outlier values as it is based on quartile calculations.
-
Can be customized to work on selected features and set thresholds for outliers.
-
-
-
Limitations
-
-
-
Might cause false positives if the variable deviates from a normal or near-normal distribution, especially for
-skewed distributions.
-
Does not provide interpretation or recommendations for addressing outliers, relying on further analysis by users
-or data scientists.
-
Only applicable to numerical features, not categorical data.
-
Default thresholds may not be optimal for data with heavy pre-processing, manipulation, or inherently high
-kurtosis (heavy tails).
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/IsolationForestOutliers.html b/docs/_build/validmind/tests/data_validation/IsolationForestOutliers.html
deleted file mode 100644
index 77af84442..000000000
--- a/docs/_build/validmind/tests/data_validation/IsolationForestOutliers.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.IsolationForestOutliers API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Detects outliers in a dataset using the Isolation Forest algorithm and visualizes results through scatter plots.
-
-
Purpose
-
-
The IsolationForestOutliers test is designed to identify anomalies or outliers in the model's dataset using the
-isolation forest algorithm. This algorithm assumes that anomalous data points can be isolated more quickly due to
-their distinctive properties. By creating isolation trees and identifying instances with shorter average path
-lengths, the test is able to pick out data points that differ from the majority.
-
-
Test Mechanism
-
-
The test uses the isolation forest algorithm, which builds an ensemble of isolation trees by randomly selecting
-features and splitting the data based on random thresholds. It isolates anomalies rather than focusing on normal
-data points. For each pair of variables, a scatter plot is generated which distinguishes the identified outliers
-from the inliers. The results of the test can be visualized using these scatter plots, illustrating the distinction
-between outliers and inliers.
-
-
Signs of High Risk
-
-
-
The presence of high contamination, indicating a large number of anomalies
-
Inability to detect clusters of anomalies that are close in the feature space
-
Misclassifying normal instances as anomalies
-
Failure to detect actual anomalies
-
-
-
Strengths
-
-
-
Ability to handle large, high-dimensional datasets
-
Efficiency in isolating anomalies instead of normal instances
-
Insensitivity to the underlying distribution of data
-
Ability to recognize anomalies even when they are not separated from the main data cloud through identifying
-distinctive properties
-
Visually presents the test results for better understanding and interpretability
-
-
-
Limitations
-
-
-
Difficult to detect anomalies that are close to each other or prevalent in datasets
-
Dependency on the contamination parameter which may need fine-tuning to be effective
-
Potential failure in detecting collective anomalies if they behave similarly to normal data
-
Potential lack of precision in identifying which features contribute most to the anomalous behavior
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/JarqueBera.html b/docs/_build/validmind/tests/data_validation/JarqueBera.html
deleted file mode 100644
index 738bfee55..000000000
--- a/docs/_build/validmind/tests/data_validation/JarqueBera.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.JarqueBera API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses normality of dataset features in an ML model using the Jarque-Bera test.
-
-
Purpose
-
-
The purpose of the Jarque-Bera test as implemented in this metric is to determine if the features in the dataset of
-a given Machine Learning model follow a normal distribution. This is crucial for understanding the distribution and
-behavior of the model's features, as numerous statistical methods assume normal distribution of the data.
-
-
Test Mechanism
-
-
The test mechanism involves computing the Jarque-Bera statistic, p-value, skew, and kurtosis for each feature in
-the dataset. It utilizes the 'jarque_bera' function from the 'statsmodels' library in Python, storing the results
-in a dictionary. The test evaluates the skewness and kurtosis to ascertain whether the dataset follows a normal
-distribution. A significant p-value (typically less than 0.05) implies that the data does not possess normal
-distribution.
-
-
Signs of High Risk
-
-
-
A high Jarque-Bera statistic and a low p-value (usually less than 0.05) indicate high-risk conditions.
-
Such results suggest the data significantly deviates from a normal distribution. If a machine learning model
-expects feature data to be normally distributed, these findings imply that it may not function as intended.
-
-
-
Strengths
-
-
-
Provides insights into the shape of the data distribution, helping determine whether a given set of data follows
-a normal distribution.
-
Particularly useful for risk assessment for models that assume a normal distribution of data.
-
By measuring skewness and kurtosis, it provides additional insights into the nature and magnitude of a
-distribution's deviation.
-
-
-
Limitations
-
-
-
Only checks for normality in the data distribution. It cannot provide insights into other types of distributions.
-
Datasets that aren't normally distributed but follow some other distribution might lead to inaccurate risk
-assessments.
-
Highly sensitive to large sample sizes, often rejecting the null hypothesis (that data is normally distributed)
-even for minor deviations in larger datasets.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/KPSS.html b/docs/_build/validmind/tests/data_validation/KPSS.html
deleted file mode 100644
index 65f0956e7..000000000
--- a/docs/_build/validmind/tests/data_validation/KPSS.html
+++ /dev/null
@@ -1,300 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.KPSS API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the stationarity of time-series data in a machine learning model using the KPSS unit root test.
-
-
Purpose
-
-
The KPSS (Kwiatkowski-Phillips-Schmidt-Shin) unit root test is utilized to ensure the stationarity of data within a
-machine learning model. It specifically works on time-series data to establish the order of integration, which is
-essential for accurate forecasting. A fundamental requirement for any time series model is that the series should
-be stationary.
-
-
Test Mechanism
-
-
This test calculates the KPSS score for each feature in the dataset. The KPSS score includes a statistic, a
-p-value, a used lag, and critical values. The core principle behind the KPSS test is to evaluate the hypothesis
-that an observable time series is stationary around a deterministic trend. If the computed statistic exceeds the
-critical value, the null hypothesis (that the series is stationary) is rejected, indicating that the series is
-non-stationary.
-
-
Signs of High Risk
-
-
-
High KPSS score, particularly if the calculated statistic is higher than the critical value.
-
Rejection of the null hypothesis, indicating that the series is recognized as non-stationary, can severely affect
-the model's forecasting capability.
-
-
-
Strengths
-
-
-
Directly measures the stationarity of a series, fulfilling a key prerequisite for many time-series models.
-
The underlying logic of the test is intuitive and simple, making it easy to understand and accessible for both
-developers and risk management teams.
-
-
-
Limitations
-
-
-
Assumes the absence of a unit root in the series and doesn't differentiate between series that are stationary and
-those border-lining stationarity.
-
The test may have restricted power against certain alternatives.
-
The reliability of the test is contingent on the number of lags selected, which introduces potential bias in the
-measurement.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/LJungBox.html b/docs/_build/validmind/tests/data_validation/LJungBox.html
deleted file mode 100644
index 370d7c001..000000000
--- a/docs/_build/validmind/tests/data_validation/LJungBox.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.LJungBox API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses autocorrelations in dataset features by performing a Ljung-Box test on each feature.
-
-
Purpose
-
-
The Ljung-Box test is a type of statistical test utilized to ascertain whether there are autocorrelations within a
-given dataset that differ significantly from zero. In the context of a machine learning model, this test is
-primarily used to evaluate data utilized in regression tasks, especially those involving time series and
-forecasting.
-
-
Test Mechanism
-
-
The test operates by iterating over each feature within the dataset and applying the acorr_ljungbox
-function from the statsmodels.stats.diagnostic library. This function calculates the Ljung-Box statistic and
-p-value for each feature. These results are then stored in a pandas DataFrame where the columns are the feature names,
-statistic, and p-value respectively. Generally, a lower p-value indicates a higher likelihood of significant
-autocorrelations within the feature.
-
-
Signs of High Risk
-
-
-
High Ljung-Box statistic values or low p-values.
-
Presence of significant autocorrelations in the respective features.
-
Potential for negative impact on model performance or bias if autocorrelations are not properly handled.
-
-
-
Strengths
-
-
-
Powerful tool for detecting autocorrelations within datasets, especially in time series data.
-
Provides quantitative measures (statistic and p-value) for precise evaluation.
-
Helps avoid issues related to autoregressive residuals and other challenges in regression models.
-
-
-
Limitations
-
-
-
Cannot detect all types of non-linearity or complex interrelationships among variables.
-
Testing individual features may not fully encapsulate the dynamics of the data if features interact with each other.
-
Designed more for traditional statistical models and may not be fully compatible with certain types of complex
-machine learning models.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/LaggedCorrelationHeatmap.html b/docs/_build/validmind/tests/data_validation/LaggedCorrelationHeatmap.html
deleted file mode 100644
index 87aef69db..000000000
--- a/docs/_build/validmind/tests/data_validation/LaggedCorrelationHeatmap.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.LaggedCorrelationHeatmap API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses and visualizes correlation between target variable and lagged independent variables in a time-series
-dataset.
-
-
Purpose
-
-
The LaggedCorrelationHeatmap metric is utilized to appraise and illustrate the correlation between the target
-variable and delayed copies (lags) of independent variables in a time-series dataset. It assists in revealing
-relationships in time-series data where the influence of an independent variable on the dependent variable is not
-immediate but occurs after a period (lags).
-
-
Test Mechanism
-
-
To execute this test, Python's Pandas library pairs with Plotly to perform computations and present the
-visualization in the form of a heatmap. The test begins by extracting the target variable and corresponding
-independent variables from the dataset. Then, generation of lags of independent variables takes place, followed by
-the calculation of correlation between these lagged variables and the target variable. The outcome is a correlation
-matrix that gets recorded and illustrated as a heatmap, where different color intensities represent the strength of
-the correlation, making patterns easier to identify.
-
-
Signs of High Risk
-
-
-
Insignificant correlations across the heatmap, indicating a lack of noteworthy relationships between variables.
-
Correlations that break intuition or previous understanding, suggesting potential issues with the dataset or the
-model.
-
-
-
Strengths
-
-
-
This metric serves as an exceptional tool for exploring and visualizing time-dependent relationships between
-features and the target variable in a time-series dataset.
-
It aids in identifying delayed effects that might go unnoticed with other correlation measures.
-
The heatmap offers an intuitive visual representation of time-dependent correlations and influences.
-
-
-
Limitations
-
-
-
The metric presumes linear relationships between variables, potentially ignoring non-linear relationships.
-
The correlation considered is linear; therefore, intricate non-linear interactions might be overlooked.
-
The metric is only applicable for time-series data, limiting its utility outside of this context.
-
The number of lags chosen can significantly influence the results; too many lags can render the heatmap difficult
-to interpret, while too few might overlook delayed effects.
-
This metric does not take into account any causal relationships, but merely demonstrates correlation.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/MissingValues.html b/docs/_build/validmind/tests/data_validation/MissingValues.html
deleted file mode 100644
index dbb5de5a9..000000000
--- a/docs/_build/validmind/tests/data_validation/MissingValues.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.MissingValues API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates dataset quality by ensuring missing value ratio across all features does not exceed a set threshold.
-
-
Purpose
-
-
The Missing Values test is designed to evaluate the quality of a dataset by measuring the number of missing values
-across all features. The objective is to ensure that the ratio of missing data to total data is less than a
-predefined threshold, defaulting to 1, in order to maintain the data quality necessary for reliable predictive
-strength in a machine learning model.
-
-
Test Mechanism
-
-
The mechanism for this test involves iterating through each column of the dataset, counting missing values
-(represented as NaNs), and calculating the percentage they represent against the total number of rows. The test
-then checks if these missing value counts are less than the predefined min_threshold. The results are shown in a
-table summarizing each column, the number of missing values, the percentage of missing values in each column, and a
-Pass/Fail status based on the threshold comparison.
-
-
Signs of High Risk
-
-
-
When the number of missing values in any column exceeds the min_threshold value.
-
Presence of missing values across many columns, leading to multiple instances of failing the threshold.
-
-
-
Strengths
-
-
-
Quick and granular identification of missing data across each feature in the dataset.
-
Provides an effective and straightforward means of maintaining data quality, essential for constructing efficient
-machine learning models.
-
-
-
Limitations
-
-
-
Does not suggest the root causes of the missing values or recommend ways to impute or handle them.
-
May overlook features with significant missing data but still less than the min_threshold, potentially
-impacting the model.
-
Does not account for data encoded as values like "-999" or "None," which might not technically classify as
-missing but could bear similar implications.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/MissingValuesBarPlot.html b/docs/_build/validmind/tests/data_validation/MissingValuesBarPlot.html
deleted file mode 100644
index 89a214568..000000000
--- a/docs/_build/validmind/tests/data_validation/MissingValuesBarPlot.html
+++ /dev/null
@@ -1,306 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.MissingValuesBarPlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the percentage and distribution of missing values in the dataset via a bar plot, with emphasis on
-identifying high-risk columns based on a user-defined threshold.
-
-
Purpose
-
-
The 'MissingValuesBarPlot' metric provides a color-coded visual representation of the percentage of missing values
-for each column in an ML model's dataset. The primary purpose of this metric is to easily identify and quantify
-missing data, which are essential steps in data preprocessing. The presence of missing data can potentially skew
-the model's predictions and decrease its accuracy. Additionally, this metric uses a pre-set threshold to categorize
-various columns into ones that contain missing data above the threshold (high risk) and below the threshold (less
-risky).
-
-
Test Mechanism
-
-
The test mechanism involves scanning each column in the input dataset and calculating the percentage of missing
-values. It then compares each column's missing data percentage with the predefined threshold, categorizing columns
-with missing data above the threshold as high-risk. The test generates a bar plot in which columns with missing
-data are represented on the y-axis and their corresponding missing data percentages are displayed on the x-axis.
-The color of each bar reflects the missing data percentage in relation to the threshold: grey for values below the
-threshold and light coral for those exceeding it. The user-defined threshold is represented by a red dashed line on
-the plot.
-
-
Signs of High Risk
-
-
-
Columns with higher percentages of missing values beyond the threshold are high-risk. These are visually
-represented by light coral bars on the bar plot.
-
-
-
Strengths
-
-
-
Helps in quickly identifying and quantifying missing data across all columns of the dataset.
-
Facilitates pattern recognition through visual representation.
-
Enables customization of the level of risk tolerance via a user-defined threshold.
-
Supports both classification and regression tasks, sharing its versatility.
-
-
-
Limitations
-
-
-
It only considers the quantity of missing values, not differentiating between different types of missingness
-(Missing completely at random - MCAR, Missing at random - MAR, Not Missing at random - NMAR).
-
It doesn't offer insights into potential approaches for handling missing entries, such as various imputation
-strategies.
-
The metric does not consider possible impacts of the missing data on the model's accuracy or precision.
-
Interpretation of the findings and the next steps might require an expert understanding of the field.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/MutualInformation.html b/docs/_build/validmind/tests/data_validation/MutualInformation.html
deleted file mode 100644
index d0d0d8b29..000000000
--- a/docs/_build/validmind/tests/data_validation/MutualInformation.html
+++ /dev/null
@@ -1,312 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.MutualInformation API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Calculates mutual information scores between features and target variable to evaluate feature relevance.
-
-
Purpose
-
-
The Mutual Information test quantifies the predictive power of each feature by measuring its statistical
-dependency with the target variable. This helps identify relevant features for model training and
-detect potential redundant or irrelevant variables, supporting feature selection decisions and model
-interpretability.
-
-
Test Mechanism
-
-
The test employs sklearn's mutual_info_classif/mutual_info_regression functions to compute mutual
-information between each feature and the target. It produces a normalized score (0 to 1) for each
-feature, where higher scores indicate stronger relationships. Results are presented in both tabular
-format and visualized through a bar plot with a configurable threshold line.
-
-
Signs of High Risk
-
-
-
Many features showing very low mutual information scores
-
Key business features exhibiting unexpectedly low scores
-
All features showing similar, low information content
-
Large discrepancy between business importance and MI scores
-
Highly skewed distribution of MI scores
-
Critical features below the minimum threshold
-
Unexpected zero or near-zero scores for known important features
-
Inconsistent scores across different data samples
-
-
-
Strengths
-
-
-
Captures non-linear relationships between features and target
-
Scale-invariant measurement of feature relevance
-
Works for both classification and regression tasks
-
Provides interpretable scores (0 to 1 scale)
-
Supports automated feature selection
-
No assumptions about data distribution
-
Handles numerical and categorical features
-
Computationally efficient for most datasets
-
-
-
Limitations
-
-
-
Requires sufficient data for reliable estimates
-
May be computationally intensive for very large datasets
-
Cannot detect redundant features (pairwise relationships)
-
Sensitive to feature discretization for continuous variables
-
Does not account for feature interactions
-
May underestimate importance of rare but crucial events
-
Cannot handle missing values directly
-
May be affected by extreme class imbalance
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/PearsonCorrelationMatrix.html b/docs/_build/validmind/tests/data_validation/PearsonCorrelationMatrix.html
deleted file mode 100644
index 2a624745e..000000000
--- a/docs/_build/validmind/tests/data_validation/PearsonCorrelationMatrix.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.PearsonCorrelationMatrix API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates linear dependency between numerical variables in a dataset via a Pearson Correlation coefficient heat map.
-
-
Purpose
-
-
This test is intended to evaluate the extent of linear dependency between all pairs of numerical variables in the
-given dataset. It provides the Pearson Correlation coefficient, which reveals any high correlations present. The
-purpose of doing this is to identify potential redundancy, as variables that are highly correlated can often be
-removed to reduce the dimensionality of the dataset without significantly impacting the model's performance.
-
-
Test Mechanism
-
-
This metric test generates a correlation matrix for all numerical variables in the dataset using the Pearson
-correlation formula. A heat map is subsequently created to visualize this matrix effectively. The color of each
-point on the heat map corresponds to the magnitude and direction (positive or negative) of the correlation, with a
-range from -1 (perfect negative correlation) to 1 (perfect positive correlation). Any correlation coefficients
-higher than 0.7 (in absolute terms) are indicated in white in the heat map, suggesting a high degree of correlation.
-
-
Signs of High Risk
-
-
-
A large number of variables in the dataset showing a high degree of correlation (coefficients approaching ±1).
-This indicates redundancy within the dataset, suggesting that some variables may not be contributing new
-information to the model.
-
Potential risk of overfitting.
-
-
-
Strengths
-
-
-
Detects and quantifies the linearity of relationships between variables, aiding in identifying redundant
-variables to simplify models and potentially improve performance.
-
The heatmap visualization provides an easy-to-understand overview of correlations, beneficial for users not
-comfortable with numerical matrices.
-
-
-
Limitations
-
-
-
Limited to detecting linear relationships, potentially missing non-linear relationships which impede
-opportunities for dimensionality reduction.
-
Measures only the degree of linear relationship, not the strength of one variable's effect on another.
-
The 0.7 correlation threshold is arbitrary and might exclude valid dependencies with lower coefficients.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/PhillipsPerronArch.html b/docs/_build/validmind/tests/data_validation/PhillipsPerronArch.html
deleted file mode 100644
index 21847dbc6..000000000
--- a/docs/_build/validmind/tests/data_validation/PhillipsPerronArch.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.PhillipsPerronArch API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the stationarity of time series data in each feature of the ML model using the Phillips-Perron test.
-
-
Purpose
-
-
The Phillips-Perron (PP) test is used to determine the stationarity of time series data for each feature in a
-dataset, which is crucial for forecasting tasks. It tests the null hypothesis that a time series is unit-root
-non-stationary. This is vital for understanding the stochastic behavior of the data and ensuring the robustness and
-validity of predictions generated by regression analysis models.
-
-
Test Mechanism
-
-
The PP test is conducted for each feature in the dataset as follows:
-
-
-
A data frame is created from the dataset.
-
For each column, the Phillips-Perron method calculates the test statistic, p-value, lags used, and number of
-observations.
-
The results are then stored for each feature, providing a metric that indicates the stationarity of the time
-series data.
-
-
-
Signs of High Risk
-
-
-
A high p-value, indicating that the series has a unit root and is non-stationary.
-
Test statistic values exceeding critical values, suggesting non-stationarity.
-
High 'usedlag' value, pointing towards autocorrelation issues that may degrade model performance.
-
-
-
Strengths
-
-
-
Resilience against heteroskedasticity in the error term.
-
Effective for long time series data.
-
Helps in determining whether the time series is stationary, aiding in the selection of suitable forecasting
-models.
-
-
-
Limitations
-
-
-
Applicable only within a univariate time series framework.
-
Relies on asymptotic theory, which may reduce the test’s power for small sample sizes.
-
Non-stationary time series must be converted to stationary series through differencing, potentially leading to
-loss of important data points.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ProtectedClassesCombination.html b/docs/_build/validmind/tests/data_validation/ProtectedClassesCombination.html
deleted file mode 100644
index c311b3449..000000000
--- a/docs/_build/validmind/tests/data_validation/ProtectedClassesCombination.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ProtectedClassesCombination API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Visualizes combinations of protected classes and their corresponding error metric differences.
-
-
Purpose
-
-
This test aims to provide insights into how different combinations of protected classes affect various error metrics,
-particularly the false negative rate (FNR) and false positive rate (FPR). By visualizing these combinations,
-it helps identify potential biases or disparities in model performance across different intersectional groups.
-
-
Test Mechanism
-
-
The test performs the following steps:
-
-
-
Combines the specified protected class columns to create a single multi-class category.
-
Calculates error metrics (FNR, FPR, etc.) for each combination of protected classes.
-
Generates visualizations showing the distribution of these metrics across all class combinations.
-
-
-
Signs of High Risk
-
-
-
Large disparities in FNR or FPR across different protected class combinations.
-
Consistent patterns of higher error rates for specific combinations of protected attributes.
-
Unexpected or unexplainable variations in error metrics between similar group combinations.
-
-
-
Strengths
-
-
-
Provides a comprehensive view of intersectional fairness across multiple protected attributes.
-
Allows for easy identification of potentially problematic combinations of protected classes.
-
Visualizations make it easier to spot patterns or outliers in model performance across groups.
-
-
-
Limitations
-
-
-
May become complex and difficult to interpret with a large number of protected classes or combinations.
-
Does not provide statistical significance of observed differences.
-
Visualization alone may not capture all nuances of intersectional fairness.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ProtectedClassesDescription.html b/docs/_build/validmind/tests/data_validation/ProtectedClassesDescription.html
deleted file mode 100644
index 1854c2c65..000000000
--- a/docs/_build/validmind/tests/data_validation/ProtectedClassesDescription.html
+++ /dev/null
@@ -1,308 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ProtectedClassesDescription API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Visualizes the distribution of protected classes in the dataset relative to the target variable
-and provides descriptive statistics.
-
-
Purpose
-
-
The ProtectedClassesDescription test aims to identify potential biases or significant differences in the
-distribution of target outcomes across different protected classes. This visualization and statistical summary
-help in understanding the relationship between protected attributes and the target variable, which is crucial
-for assessing fairness in machine learning models.
-
-
Test Mechanism
-
-
The function creates interactive stacked bar charts for each specified protected class using Plotly.
-Additionally, it generates a single table of descriptive statistics for all protected classes, including:
-
-
-
Protected class and category
-
Count and percentage of each category within the protected class
-
Mean, median, and mode of the target variable for each category
-
Standard deviation of the target variable for each category
-
Minimum and maximum values of the target variable for each category
-
-
-
Signs of High Risk
-
-
-
Significant imbalances in the distribution of target outcomes across different categories of a protected class.
-
Large disparities in mean, median, or mode of the target variable across categories.
-
Underrepresentation or overrepresentation of certain groups within protected classes.
-
High standard deviations in certain categories, indicating potential volatility or outliers.
-
-
-
Strengths
-
-
-
Provides both visual and statistical representation of potential biases in the dataset.
-
Allows for easy identification of imbalances in target variable distribution across protected classes.
-
Interactive plots enable detailed exploration of the data.
-
Consolidated statistical summary provides quantitative measures to complement visual analysis.
-
Applicable to both classification and regression tasks.
-
-
-
Limitations
-
-
-
Does not provide advanced statistical measures of bias or fairness.
-
May become cluttered if there are many categories within a protected class or many unique target values.
-
Interpretation may require domain expertise to understand the implications of observed disparities.
-
Does not account for intersectionality or complex interactions between multiple protected attributes.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ProtectedClassesDisparity.html b/docs/_build/validmind/tests/data_validation/ProtectedClassesDisparity.html
deleted file mode 100644
index 1a6ac579a..000000000
--- a/docs/_build/validmind/tests/data_validation/ProtectedClassesDisparity.html
+++ /dev/null
@@ -1,302 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ProtectedClassesDisparity API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Investigates disparities in model performance across different protected class segments.
-
-
Purpose
-
-
This test aims to identify and quantify potential biases in model outcomes by comparing various performance metrics
-across different segments of protected classes. It helps in assessing whether the model produces discriminatory
-outcomes for certain groups, which is crucial for ensuring fairness in machine learning models.
-
-
Test Mechanism
-
-
The test performs the following steps:
-
-
-
Calculates performance metrics (e.g., false negative rate, false positive rate, true positive rate) for each segment
-of the specified protected classes.
-
Computes disparity ratios by comparing these metrics between different segments and a reference group.
-
Generates visualizations showing the disparities and their relation to a user-defined disparity tolerance threshold.
-
Produces a comprehensive table with various disparity metrics for detailed analysis.
-
-
-
Signs of High Risk
-
-
-
Disparity ratios exceeding the specified disparity tolerance threshold.
-
Consistent patterns of higher error rates or lower performance for specific protected class segments.
-
Statistically significant differences in performance metrics across segments.
-
-
-
Strengths
-
-
-
Provides a comprehensive view of model fairness across multiple protected attributes and metrics.
-
Allows for easy identification of problematic disparities through visual and tabular representations.
-
Customizable disparity tolerance threshold to align with specific use-case requirements.
-
Applicable to various performance metrics, offering a multi-faceted analysis of model fairness.
-
-
-
Limitations
-
-
-
Relies on a predefined reference group for each protected class, which may not always be the most appropriate choice.
-
Does not account for intersectionality between different protected attributes.
-
The interpretation of results may require domain expertise to understand the implications of observed disparities.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.html b/docs/_build/validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.html
deleted file mode 100644
index 389aa50b5..000000000
--- a/docs/_build/validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.html
+++ /dev/null
@@ -1,397 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ProtectedClassesThresholdOptimizer API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Obtains a classifier by applying group-specific thresholds to the provided estimator.
-
-
Purpose
-
-
This test aims to optimize the fairness of a machine learning model by applying different
-classification thresholds for different protected groups. It helps in mitigating bias and
-achieving more equitable outcomes across different demographic groups.
-
-
Test Mechanism
-
-
The test uses Fairlearn's ThresholdOptimizer to:
-
-
-
Fit an optimizer on the training data, considering protected classes.
-
Apply optimized thresholds to make predictions on the test data.
-
Calculate and report various fairness metrics.
-
Visualize the optimized thresholds.
-
-
-
Signs of High Risk
-
-
-
Large disparities in fairness metrics (e.g., Demographic Parity Ratio, Equalized Odds Ratio)
-across different protected groups.
-
Significant differences in False Positive Rates (FPR) or True Positive Rates (TPR) between groups.
-
Thresholds that vary widely across different protected groups.
-
-
-
Strengths
-
-
-
Provides a post-processing method to improve model fairness without modifying the original model.
-
Allows for balancing multiple fairness criteria simultaneously.
-
Offers visual insights into the threshold optimization process.
-
-
-
Limitations
-
-
-
May lead to a decrease in overall model performance while improving fairness.
-
Requires access to protected attribute information at prediction time.
-
The effectiveness can vary depending on the chosen fairness constraint and objective.
Evaluates the stationarity of time series data by plotting its rolling mean and standard deviation over a specified
-window.
-
-
Purpose
-
-
The RollingStatsPlot metric is employed to gauge the stationarity of time series data in a given dataset. This
-metric specifically evaluates the rolling mean and rolling standard deviation of the dataset over a pre-specified
-window size. The rolling mean provides an understanding of the average trend in the data, while the rolling
-standard deviation gauges the volatility of the data within the window. It is critical in preparing time series
-data for modeling as it reveals key insights into data behavior across time.
-
-
Test Mechanism
-
-
This mechanism is comprised of two steps. Initially, the rolling mean and standard deviation for each of the
-dataset's columns are calculated over a window size, which can be user-specified or by default set to 12 data
-points. Then, the calculated rolling mean and standard deviation are visualized via separate plots, illustrating
-the trends and volatility in the dataset. A straightforward check is conducted to ensure the existence of columns
-in the dataset, and to verify that the given dataset has been indexed by its date and time—a necessary prerequisite
-for time series analysis.
-
-
Signs of High Risk
-
-
-
The presence of non-stationary patterns in either the rolling mean or the rolling standard deviation plots, which
-could indicate trends or seasonality in the data that may affect the performance of time series models.
-
Missing columns in the dataset, which would prevent the execution of this metric correctly.
-
The detection of NaN values in the dataset, which may need to be addressed before the metric can proceed
-successfully.
-
-
-
Strengths
-
-
-
Offers visualizations of trending behavior and volatility within the data, facilitating a broader understanding
-of the dataset's inherent characteristics.
-
Checks of the dataset's integrity, such as the existence of all required columns and the availability of a
-datetime index.
-
Adjusts to accommodate various window sizes, thus allowing accurate analysis of data with differing temporal
-granularities.
-
Considers each column of the data individually, thereby accommodating multi-feature datasets.
-
-
-
Limitations
-
-
-
For all columns, a fixed-size window is utilized. This may not accurately capture patterns in datasets where
-different features may require different optimal window sizes.
-
Requires the dataset to be indexed by date and time, hence it may not be usable for datasets without a timestamp
-index.
-
Primarily serves for data visualization as it does not facilitate any quantitative measures for stationarity,
-such as through statistical tests. Therefore, the interpretation is subjective and depends heavily on modeler
-discretion.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/RunsTest.html b/docs/_build/validmind/tests/data_validation/RunsTest.html
deleted file mode 100644
index 13bce1013..000000000
--- a/docs/_build/validmind/tests/data_validation/RunsTest.html
+++ /dev/null
@@ -1,305 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.RunsTest API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Executes Runs Test on ML model to detect non-random patterns in output data sequence.
-
-
Purpose
-
-
The Runs Test is a statistical procedure used to determine whether the sequence of data extracted from the ML model
-behaves randomly or not. Specifically, it analyzes runs, sequences of consecutive positives or negatives, in the
-data to check if there are more or fewer runs than expected under the assumption of randomness. This can be an
-indication of some pattern, trend, or cycle in the model's output which may need attention.
-
-
Test Mechanism
-
-
The testing mechanism applies the Runs Test from the statsmodels module on each column of the training dataset. For
-every feature in the dataset, a Runs Test is executed, whose output includes a Runs Statistic and P-value. A low
-P-value suggests that data arrangement in the feature is not likely to be random. The results are stored in a
-dictionary where the keys are the feature names, and the values are another dictionary storing the test statistic
-and the P-value for each feature.
-
-
Signs of High Risk
-
-
-
High risk is indicated when the P-value is close to zero.
-
If the P-value is less than a predefined significance level (like 0.05), it suggests that the runs (series of
-positive or negative values) in the model's output are not random and are longer or shorter than what is expected
-under a random scenario.
-
This would mean there's a high risk of non-random distribution of errors or model outcomes, suggesting potential
-issues with the model.
-
-
-
Strengths
-
-
-
Straightforward and fast for detecting non-random patterns in data sequence.
-
Validates assumptions of randomness, which is valuable for checking error distributions in regression models,
-trendless time series data, and ensuring a classifier doesn't favor one class over another.
-
Can be applied to both classification and regression tasks, making it versatile.
-
-
-
Limitations
-
-
-
Assumes that the data is independently and identically distributed (i.i.d.), which might not be the case for many
-real-world datasets.
-
The conclusion drawn from the low P-value indicating non-randomness does not provide information about the type
-or the source of the detected pattern.
-
Sensitive to extreme values (outliers), and overly large or small run sequences can influence the results.
-
Does not provide model performance evaluation; it is used to detect patterns in the sequence of outputs only.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ScatterPlot.html b/docs/_build/validmind/tests/data_validation/ScatterPlot.html
deleted file mode 100644
index 9d2e8bf1a..000000000
--- a/docs/_build/validmind/tests/data_validation/ScatterPlot.html
+++ /dev/null
@@ -1,306 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ScatterPlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses visual relationships, patterns, and outliers among features in a dataset through scatter plot matrices.
-
-
Purpose
-
-
The ScatterPlot test aims to visually analyze a given dataset by constructing a scatter plot matrix of its
-numerical features. The primary goal is to uncover relationships, patterns, and outliers across different features
-to provide both quantitative and qualitative insights into multidimensional relationships within the dataset. This
-visual assessment aids in understanding the efficacy of the chosen features for model training and their
-suitability.
-
-
Test Mechanism
-
-
Using the Seaborn library, the ScatterPlot function creates the scatter plot matrix. The process involves
-retrieving all numerical columns from the dataset and generating a scatter matrix for these columns. The resulting
-scatter plot provides visual representations of feature relationships. The function also adjusts axis labels for
-readability and returns the final plot as a Matplotlib Figure object for further analysis and visualization.
-
-
Signs of High Risk
-
-
-
The emergence of non-linear or random patterns across different feature pairs, suggesting complex relationships
-unsuitable for linear assumptions.
-
Lack of clear patterns or clusters, indicating weak or non-existent correlations among features, which could
-challenge certain model types.
-
Presence of outliers, as visual outliers can adversely influence the model's performance.
-
-
-
Strengths
-
-
-
Provides insight into the multidimensional relationships among multiple features.
-
Assists in identifying trends, correlations, and outliers that could affect model performance.
-
Validates assumptions made during model creation, such as linearity.
-
Versatile for application in both regression and classification tasks.
-
Using Seaborn facilitates an intuitive and detailed visual exploration of data.
-
-
-
Limitations
-
-
-
Scatter plot matrices may become cluttered and hard to decipher as the number of features increases.
-
Primarily reveals pairwise relationships and may fail to illuminate complex interactions involving three or more
-features.
-
Being a visual tool, precision in quantitative analysis might be compromised.
-
Outliers not clearly visible in plots can be missed, affecting model performance.
-
Assumes that the dataset can fit into the computer's memory, which might not be valid for extremely large
-datasets.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ScoreBandDefaultRates.html b/docs/_build/validmind/tests/data_validation/ScoreBandDefaultRates.html
deleted file mode 100644
index e42be0748..000000000
--- a/docs/_build/validmind/tests/data_validation/ScoreBandDefaultRates.html
+++ /dev/null
@@ -1,315 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ScoreBandDefaultRates API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Analyzes default rates and population distribution across credit score bands.
-
-
Purpose
-
-
The Score Band Default Rates test evaluates the discriminatory power of credit scores by analyzing
-default rates across different score bands. This helps validate score effectiveness, supports
-policy decisions, and provides insights into portfolio risk distribution.
-
-
Test Mechanism
-
-
The test segments the score distribution into bands and calculates key metrics for each band:
-
-
-
Population count and percentage in each band
-
Default rate within each band
-
Cumulative statistics across bands
-The results show how well the scores separate good and bad accounts.
-
-
-
Signs of High Risk
-
-
-
Non-monotonic default rates across score bands
-
Insufficient population in critical score bands
-
Unexpected default rates for score ranges
-
High concentration in specific score bands
-
Similar default rates across adjacent bands
-
Unstable default rates in key decision bands
-
Extreme population skewness
-
Poor risk separation between bands
-
-
-
Strengths
-
-
-
Clear view of score effectiveness
-
Supports policy threshold decisions
-
Easy to interpret and communicate
-
Directly links to business decisions
-
Shows risk segmentation power
-
Identifies potential score issues
-
Helps validate scoring model
-
Supports portfolio monitoring
-
-
-
Limitations
-
-
-
Sensitive to band definition choices
-
May mask within-band variations
-
Requires sufficient data in each band
-
Cannot capture non-linear patterns
-
Point-in-time analysis only
-
No temporal trend information
-
Assumes band boundaries are appropriate
-
May oversimplify risk patterns
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/SeasonalDecompose.html b/docs/_build/validmind/tests/data_validation/SeasonalDecompose.html
deleted file mode 100644
index 74af78bc4..000000000
--- a/docs/_build/validmind/tests/data_validation/SeasonalDecompose.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.SeasonalDecompose API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses patterns and seasonality in a time series dataset by decomposing its features into foundational components.
-
-
Purpose
-
-
The Seasonal Decompose test aims to decompose the features of a time series dataset into their fundamental
-components: observed, trend, seasonal, and residuals. By utilizing the Seasonal Decomposition of Time Series by
-Loess (STL) method, the test identifies underlying patterns, predominantly seasonality, in the dataset's features.
-This aids in developing a more comprehensive understanding of the dataset, which in turn facilitates more effective
-model validation.
-
-
Test Mechanism
-
-
The testing process leverages the seasonal_decompose function from the statsmodels.tsa.seasonal library to
-evaluate each feature in the dataset. It isolates each feature into four components—observed, trend, seasonal, and
-residuals—and generates six subplot graphs per feature for visual interpretation. Prior to decomposition, the test
-scrutinizes and removes any non-finite values, ensuring the reliability of the analysis.
-
-
Signs of High Risk
-
-
-
Non-Finiteness: Datasets with a high number of non-finite values may flag as high risk since these values are
-omitted before conducting the seasonal decomposition.
-
Frequent Warnings: Chronic failure to infer the frequency for a scrutinized feature indicates high risk.
-
High Seasonality: A significant seasonal component could potentially render forecasts unreliable due to
-overwhelming seasonal variation.
-
-
-
Strengths
-
-
-
Seasonality Detection: Accurately discerns hidden seasonality patterns in dataset features.
-
Visualization: Facilitates interpretation and comprehension through graphical representations.
-
Unrestricted Usage: Not confined to any specific regression model, promoting wide-ranging applicability.
-
-
-
Limitations
-
-
-
Dependence on Assumptions: Assumes that dataset features are periodically distributed. Features with no
-inferable frequency are excluded from the test.
-
Handling Non-Finite Values: Disregards non-finite values during analysis, potentially resulting in an
-incomplete understanding of the dataset.
-
Unreliability with Noisy Datasets: Produces unreliable results when used with datasets that contain heavy
-noise.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ShapiroWilk.html b/docs/_build/validmind/tests/data_validation/ShapiroWilk.html
deleted file mode 100644
index 5335984af..000000000
--- a/docs/_build/validmind/tests/data_validation/ShapiroWilk.html
+++ /dev/null
@@ -1,302 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ShapiroWilk API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates feature-wise normality of training data using the Shapiro-Wilk test.
-
-
Purpose
-
-
The Shapiro-Wilk test is utilized to investigate whether a particular dataset conforms to the standard normal
-distribution. This analysis is crucial in machine learning modeling because the normality of the data can
-profoundly impact the performance of the model. This metric is especially useful in evaluating various features of
-the dataset in both classification and regression tasks.
-
-
Test Mechanism
-
-
The Shapiro-Wilk test is conducted on each feature column of the training dataset to determine if the data
-contained fall within the normal distribution. The test presents a statistic and a p-value, with the p-value
-serving to validate or repudiate the null hypothesis, which is that the tested data is normally distributed.
-
-
Signs of High Risk
-
-
-
A p-value that falls below 0.05 signifies a high risk as it discards the null hypothesis, indicating that the
-data does not adhere to the normal distribution.
-
For machine learning models built on the presumption of data normality, such an outcome could result in subpar
-performance or incorrect predictions.
-
-
-
Strengths
-
-
-
The Shapiro-Wilk test is esteemed for its level of accuracy, thereby making it particularly well-suited to
-datasets of small to moderate sizes.
-
It proves its versatility through its efficient functioning in both classification and regression tasks.
-
By separately testing each feature column, the Shapiro-Wilk test can raise an alarm if a specific feature does
-not comply with the normality.
-
-
-
Limitations
-
-
-
The Shapiro-Wilk test's sensitivity can be a disadvantage as it often rejects the null hypothesis (i.e., data is
-normally distributed), even for minor deviations, especially in large datasets. This may lead to unwarranted 'false
-alarms' of high risk by deeming the data as not normally distributed even if it approximates normal distribution.
-
Exceptional care must be taken in managing missing data or outliers prior to testing as these can greatly skew
-the results.
-
Lastly, the Shapiro-Wilk test is not optimally suited for processing data with pronounced skewness or kurtosis.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/Skewness.html b/docs/_build/validmind/tests/data_validation/Skewness.html
deleted file mode 100644
index 6dc0717d9..000000000
--- a/docs/_build/validmind/tests/data_validation/Skewness.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.Skewness API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the skewness of numerical data in a dataset to check against a defined threshold, aiming to ensure data
-quality and optimize model performance.
-
-
Purpose
-
-
The purpose of the Skewness test is to measure the asymmetry in the distribution of data within a predictive
-machine learning model. Specifically, it evaluates the divergence of said distribution from a normal distribution.
-Understanding the level of skewness helps identify data quality issues, which are crucial for optimizing the
-performance of traditional machine learning models in both classification and regression settings.
-
-
Test Mechanism
-
-
This test calculates the skewness of numerical columns in the dataset, focusing specifically on numerical data
-types. The calculated skewness value is then compared against a predetermined maximum threshold, which is set by
-default to 1. If the skewness value is less than this maximum threshold, the test passes; otherwise, it fails. The
-test results, along with the skewness values and column names, are then recorded for further analysis.
-
-
Signs of High Risk
-
-
-
Substantial skewness levels that significantly exceed the maximum threshold.
-
Persistent skewness in the data, indicating potential issues with the foundational assumptions of the machine
-learning model.
-
Subpar model performance, erroneous predictions, or biased inferences due to skewed data distributions.
-
-
-
Strengths
-
-
-
Fast and efficient identification of unequal data distributions within a machine learning model.
-
Adjustable maximum threshold parameter, allowing for customization based on user needs.
-
Provides a clear quantitative measure to mitigate model risks related to data skewness.
-
-
-
Limitations
-
-
-
Only evaluates numeric columns, potentially missing skewness or bias in non-numeric data.
-
Assumes that data should follow a normal distribution, which may not always be applicable to real-world data.
-
Subjective threshold for risk grading, requiring expert input and recurrent iterations for refinement.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/SpreadPlot.html b/docs/_build/validmind/tests/data_validation/SpreadPlot.html
deleted file mode 100644
index a85454a53..000000000
--- a/docs/_build/validmind/tests/data_validation/SpreadPlot.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.SpreadPlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses potential correlations between pairs of time series variables through visualization to enhance
-understanding of their relationships.
-
-
Purpose
-
-
The SpreadPlot test aims to graphically illustrate and analyze the relationships between pairs of time series
-variables within a given dataset. This facilitated understanding helps in identifying and assessing potential time
-series correlations, such as cointegration, between the variables.
-
-
Test Mechanism
-
-
The SpreadPlot test computes and represents the spread between each pair of time series variables in the dataset.
-Specifically, the difference between two variables is calculated and presented as a line graph. This process is
-iterated for each unique pair of variables in the dataset, allowing for comprehensive visualization of their
-relationships.
-
-
Signs of High Risk
-
-
-
Large fluctuations in the spread over a given timespan.
-
Unexpected patterns or trends that may signal potential risks in the underlying correlations between the
-variables.
-
Presence of significant missing data or extreme outlier values, which could potentially skew the spread and
-indicate high risk.
-
-
-
Strengths
-
-
-
Allows for thorough visual examination and interpretation of the correlations between time-series pairs.
-
Aids in revealing complex relationships like cointegration.
-
Enhances interpretability by visualizing the relationships, thereby helping in spotting outliers and trends.
-
Capable of handling numerous variable pairs from the dataset through a versatile and adaptable process.
-
-
-
Limitations
-
-
-
Primarily serves as a visualization tool and does not offer quantitative measurements or statistics to
-objectively determine relationships.
-
Heavily relies on the quality and granularity of the data—missing data or outliers can notably disturb the
-interpretation of relationships.
-
Can become inefficient or difficult to interpret with a high number of variables due to the profuse number of
-plots.
-
Might not completely capture intricate non-linear relationships between the variables.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TabularCategoricalBarPlots.html b/docs/_build/validmind/tests/data_validation/TabularCategoricalBarPlots.html
deleted file mode 100644
index 12b8c3832..000000000
--- a/docs/_build/validmind/tests/data_validation/TabularCategoricalBarPlots.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TabularCategoricalBarPlots API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Generates and visualizes bar plots for each category in categorical features to evaluate the dataset's composition.
-
-
Purpose
-
-
The purpose of this metric is to visually analyze categorical data using bar plots. It is intended to evaluate the
-dataset's composition by displaying the counts of each category in each categorical feature.
-
-
Test Mechanism
-
-
The provided dataset is first checked to determine if it contains any categorical variables. If no categorical
-columns are found, the tool raises a ValueError. For each categorical variable in the dataset, a separate bar plot
-is generated. The number of occurrences for each category is calculated and displayed on the plot. If a dataset
-contains multiple categorical columns, multiple bar plots are produced.
-
-
Signs of High Risk
-
-
-
High risk could occur if the categorical variables exhibit an extreme imbalance, with categories having very few
-instances possibly being underrepresented in the model, which could affect the model's performance and its ability
-to generalize.
-
Another sign of risk is if there are too many categories in a single variable, which could lead to overfitting
-and make the model complex.
-
-
-
Strengths
-
-
-
Provides a visual and intuitively understandable representation of categorical data.
-
Aids in the analysis of variable distributions.
-
Helps in easily identifying imbalances or rare categories that could affect the model's performance.
-
-
-
Limitations
-
-
-
This method only works with categorical data and won't apply to numerical variables.
-
It does not provide informative value when there are too many categories, as the bar chart could become cluttered
-and hard to interpret.
-
Offers no insights into the model's performance or precision, but rather provides a descriptive analysis of the
-input.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TabularDateTimeHistograms.html b/docs/_build/validmind/tests/data_validation/TabularDateTimeHistograms.html
deleted file mode 100644
index a0834996b..000000000
--- a/docs/_build/validmind/tests/data_validation/TabularDateTimeHistograms.html
+++ /dev/null
@@ -1,305 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TabularDateTimeHistograms API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Generates histograms to provide graphical insight into the distribution of time intervals in a model's datetime
-data.
-
-
Purpose
-
-
The TabularDateTimeHistograms metric is designed to provide graphical insight into the distribution of time
-intervals in a machine learning model's datetime data. By plotting histograms of differences between consecutive
-date entries in all datetime variables, it enables an examination of the underlying pattern of time series data and
-identification of anomalies.
-
-
Test Mechanism
-
-
This test operates by first identifying all datetime columns and extracting them from the dataset. For each
-datetime column, it next computes the differences (in days) between consecutive dates, excluding zero values, and
-visualizes these differences in a histogram. The Plotly library's histogram function is used to generate
-histograms, which are labeled appropriately and provide a graphical representation of the frequency of different
-day intervals in the dataset.
-
-
Signs of High Risk
-
-
-
If no datetime columns are detected in the dataset, this would lead to a ValueError. Hence, the absence of
-datetime columns signifies a high risk.
-
A severely skewed or irregular distribution depicted in the histogram may indicate possible complications with
-the data, such as faulty timestamps or abnormalities.
-
-
-
Strengths
-
-
-
The metric offers a visual overview of time interval frequencies within the dataset, supporting the recognition
-of inherent patterns.
-
Histogram plots can aid in the detection of potential outliers and data anomalies, contributing to an assessment
-of data quality.
-
The metric is versatile, compatible with a range of task types, including classification and regression, and can
-work with multiple datetime variables if present.
-
-
-
Limitations
-
-
-
A major weakness of this metric is its dependence on the visual examination of data, as it does not provide a
-measurable evaluation of the model.
-
The metric might overlook complex or multi-dimensional trends in the data.
-
The test is only applicable to datasets containing datetime columns and will fail if such columns are unavailable.
-
The interpretation of the histograms relies heavily on the domain expertise and experience of the reviewer.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TabularDescriptionTables.html b/docs/_build/validmind/tests/data_validation/TabularDescriptionTables.html
deleted file mode 100644
index 5458c96d5..000000000
--- a/docs/_build/validmind/tests/data_validation/TabularDescriptionTables.html
+++ /dev/null
@@ -1,408 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TabularDescriptionTables API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Summarizes key descriptive statistics for numerical, categorical, and datetime variables in a dataset.
-
-
Purpose
-
-
The main purpose of this metric is to gather and present the descriptive statistics of numerical, categorical, and
-datetime variables present in a dataset. The attributes it measures include the count, mean, minimum and maximum
-values, percentage of missing values, data types of fields, and unique values for categorical fields, among others.
-
-
Test Mechanism
-
-
The test first segregates the variables in the dataset according to their data types (numerical, categorical, or
-datetime). Then, it compiles summary statistics for each type of variable. The specifics of these statistics vary
-depending on the type of variable:
-
-
-
For numerical variables, the metric extracts descriptors like count, mean, minimum and maximum values, count of
-missing values, and data types.
-
For categorical variables, it counts the number of unique values, displays unique values, counts missing values,
-and identifies data types.
-
For datetime variables, it counts the number of unique values, identifies the earliest and latest dates, counts
-missing values, and identifies data types.
-
-
-
Signs of High Risk
-
-
-
Masses of missing values in the descriptive statistics results could hint at high risk or failure, indicating
-potential data collection, integrity, and quality issues.
-
Detection of inappropriate distributions for numerical variables, like having negative values for variables that
-are always supposed to be positive.
-
Identifying inappropriate data types, like a continuous variable being encoded as a categorical type.
-
-
-
Strengths
-
-
-
Provides a comprehensive overview of the dataset.
-
Gives a snapshot into the essence of the numerical, categorical, and datetime fields.
-
Identifies potential data quality issues such as missing values or inconsistencies crucial for building credible
-machine learning models.
-
The metadata, including the data type and missing value information, are vital for anyone including data
-scientists dealing with the dataset before the modeling process.
-
-
-
Limitations
-
-
-
It does not perform any deeper statistical analysis or tests on the data.
-
It does not handle issues such as outliers, or relationships between variables.
-
It offers no insights into potential correlations or possible interactions between variables.
-
It does not investigate the potential impact of missing values on the performance of the machine learning models.
-
It does not explore potential transformation requirements that may be necessary to enhance the performance of the
-chosen algorithm.
Generates histograms for each numerical feature in a dataset to provide visual insights into data distribution and
-detect potential issues.
-
-
Purpose
-
-
The purpose of this test is to provide visual analysis of numerical data through the generation of histograms for
-each numerical feature in the dataset. Histograms aid in the exploratory analysis of data, offering insight into
-the distribution of the data, skewness, presence of outliers, and central tendencies. It helps in understanding if
-the inputs to the model are normally distributed, which is a common assumption in many machine learning algorithms.
-
-
Test Mechanism
-
-
This test scans the provided dataset and extracts all the numerical columns. For each numerical column, it
-constructs a histogram using plotly, with 50 bins. The deployment of histograms offers a robust visual aid,
-ensuring unruffled identification and understanding of numerical data distribution patterns.
-
-
Signs of High Risk
-
-
-
A high degree of skewness
-
Unexpected data distributions
-
Existence of extreme outliers in the histograms
-
-
-
These may indicate issues with the data that the model is receiving. If data for a numerical feature is expected to
-follow a certain distribution (like a normal distribution) but does not, it could lead to sub-par performance by
-the model. As such these instances should be treated as high-risk indicators.
-
-
Strengths
-
-
-
Provides a simple, easy-to-interpret visualization of how data for each numerical attribute is distributed.
-
Helps detect skewed values and outliers that could potentially harm the AI model's performance.
-
Can be applied to large datasets and multiple numerical variables conveniently.
-
-
-
Limitations
-
-
-
Only works with numerical data, thus ignoring non-numerical or categorical data.
-
Does not analyze relationships between different features, only the individual feature distributions.
-
Is a univariate analysis and may miss patterns or anomalies that only appear when considering multiple variables
-together.
-
Does not provide any insight into how these features affect the output of the model; it is purely an input
-analysis tool.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TargetRateBarPlots.html b/docs/_build/validmind/tests/data_validation/TargetRateBarPlots.html
deleted file mode 100644
index 139503584..000000000
--- a/docs/_build/validmind/tests/data_validation/TargetRateBarPlots.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TargetRateBarPlots API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Generates bar plots visualizing the default rates of categorical features for a classification machine learning
-model.
-
-
Purpose
-
-
This test, implemented as a metric, is designed to provide an intuitive, graphical summary of the decision-making
-patterns exhibited by a categorical classification machine learning model. The model's performance is evaluated
-using bar plots depicting the ratio of target rates—meaning the proportion of positive classes—for different
-categorical inputs. This allows for an easy, at-a-glance understanding of the model's accuracy.
-
-
Test Mechanism
-
-
The test involves creating a pair of bar plots for each categorical feature in the dataset. The first plot depicts
-the frequency of each category in the dataset, with each category visually distinguished by its unique color. The
-second plot shows the mean target rate of each category (sourced from the "default_column"). Plotly, a Python
-library, is used to generate these plots, with distinct plots created for each feature. If no specific columns are
-selected, the test will generate plots for each categorical column in the dataset.
-
-
Signs of High Risk
-
-
-
Inconsistent or non-binary values in the "default_column" could complicate or render impossible the calculation
-of average target rates.
-
Particularly low or high target rates for a specific category might suggest that the model is misclassifying
-instances of that category.
-
-
-
Strengths
-
-
-
This test offers a visually interpretable breakdown of the model's decisions, providing an easy way to spot
-irregularities, inconsistencies, or patterns.
-
Its flexibility allows for the inspection of one or multiple columns, as needed.
-
-
-
Limitations
-
-
-
The readability of the bar plots drops as the number of distinct categories increases in the dataset, which can
-make them harder to understand and less useful.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TimeSeriesDescription.html b/docs/_build/validmind/tests/data_validation/TimeSeriesDescription.html
deleted file mode 100644
index c9b2c8381..000000000
--- a/docs/_build/validmind/tests/data_validation/TimeSeriesDescription.html
+++ /dev/null
@@ -1,296 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TimeSeriesDescription API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Generates a detailed analysis for the provided time series dataset, summarizing key statistics to identify trends,
-patterns, and data quality issues.
-
-
Purpose
-
-
The TimeSeriesDescription function aims to analyze an individual time series by providing a summary of key
-statistics. This helps in understanding trends, patterns, and data quality issues within the time series.
-
-
Test Mechanism
-
-
The function extracts the time series data and provides a summary of key statistics. The dataset is expected to
-have a datetime index. The function checks this and raises an error if the index is not in datetime format. For
-each variable (column) in the dataset, appropriate statistics including start date, end date, frequency, number of
-missing values, count, min, and max values are calculated.
-
-
Signs of High Risk
-
-
-
If the index of the dataset is not in datetime format, it could lead to errors in time-series analysis.
-
Inconsistent or missing data within the dataset might affect the analysis of trends and patterns.
-
-
-
Strengths
-
-
-
Provides a comprehensive summary of key statistics for each variable, helping to identify data quality issues
-such as missing values.
-
Helps in understanding the distribution and range of the data by including min and max values.
-
-
-
Limitations
-
-
-
Assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access the pandas
-DataFrame.
-
Only analyzes datasets with a datetime index and will raise an error for other types of indices.
-
Does not handle large datasets efficiently; performance may degrade with very large datasets.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.html b/docs/_build/validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.html
deleted file mode 100644
index 6716d0ef3..000000000
--- a/docs/_build/validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.html
+++ /dev/null
@@ -1,294 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TimeSeriesDescriptiveStatistics API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the descriptive statistics of a time series dataset to identify trends, patterns, and data quality issues.
-
-
Purpose
-
-
The purpose of the TimeSeriesDescriptiveStatistics function is to analyze an individual time series by providing a
-summary of key descriptive statistics. This analysis helps in understanding trends, patterns, and data quality
-issues within the time series dataset.
-
-
Test Mechanism
-
-
The function extracts the time series data and provides a summary of key descriptive statistics. The dataset is
-expected to have a datetime index, and the function will check this and raise an error if the index is not in a
-datetime format. For each variable (column) in the dataset, appropriate statistics, including start date, end date,
-min, mean, max, skewness, kurtosis, and count, are calculated.
-
-
Signs of High Risk
-
-
-
If the index of the dataset is not in datetime format, it could lead to errors in time-series analysis.
-
Inconsistent or missing data within the dataset might affect the analysis of trends and patterns.
-
-
-
Strengths
-
-
-
Provides a comprehensive summary of key descriptive statistics for each variable.
-
Helps identify data quality issues and understand the distribution of the data.
-
-
-
Limitations
-
-
-
Assumes the dataset is provided as a DataFrameDataset object with a .df attribute to access the pandas DataFrame.
-
Only analyzes datasets with a datetime index and will raise an error for other types of indices.
-
Does not handle large datasets efficiently, and performance may degrade with very large datasets.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TimeSeriesFrequency.html b/docs/_build/validmind/tests/data_validation/TimeSeriesFrequency.html
deleted file mode 100644
index 603beef4d..000000000
--- a/docs/_build/validmind/tests/data_validation/TimeSeriesFrequency.html
+++ /dev/null
@@ -1,306 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TimeSeriesFrequency API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates consistency of time series data frequency and generates a frequency plot.
-
-
Purpose
-
-
The purpose of the TimeSeriesFrequency test is to evaluate the consistency in the frequency of data points in a
-time-series dataset. This test inspects the intervals or duration between each data point to determine if a fixed
-pattern (such as daily, weekly, or monthly) exists. The identification of such patterns is crucial to time-series
-analysis as any irregularities could lead to erroneous results and hinder the model's capacity for identifying
-trends and patterns.
-
-
Test Mechanism
-
-
Initially, the test checks if the dataframe index is in datetime format. Subsequently, it utilizes pandas'
-infer_freq method to identify the frequency of each data series within the dataframe. The infer_freq method
-attempts to establish the frequency of a time series and returns both the frequency string and a dictionary
-relating these strings to their respective labels. The test compares the frequencies of all datasets. If they share
-a common frequency, the test passes, but it fails if they do not. Additionally, Plotly is used to create a
-frequency plot, offering a visual depiction of the time differences between consecutive entries in the dataframe
-index.
-
-
Signs of High Risk
-
-
-
The test fails, indicating multiple unique frequencies within the dataset. This failure could suggest irregular
-intervals between observations, potentially interrupting pattern recognition or trend analysis.
-
The presence of missing or null frequencies could be an indication of inconsistencies in data or gaps within the
-data collection process.
-
-
-
Strengths
-
-
-
This test uses a systematic approach to checking the consistency of data frequency within a time-series dataset.
-
It increases the model's reliability by asserting the consistency of observations over time, an essential factor
-in time-series analysis.
-
The test generates a visual plot, providing an intuitive representation of the dataset's frequency distribution,
-which caters to visual learners and aids in interpretation and explanation.
-
-
-
Limitations
-
-
-
This test is only applicable to time-series datasets and hence not suitable for other types of datasets.
-
The infer_freq method might not always correctly infer frequency when faced with missing or irregular data
-points.
-
Depending on context or the model under development, mixed frequencies might sometimes be acceptable, but this
-test considers them a failing condition.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TimeSeriesHistogram.html b/docs/_build/validmind/tests/data_validation/TimeSeriesHistogram.html
deleted file mode 100644
index 36d91406c..000000000
--- a/docs/_build/validmind/tests/data_validation/TimeSeriesHistogram.html
+++ /dev/null
@@ -1,300 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TimeSeriesHistogram API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Visualizes distribution of time-series data using histograms and Kernel Density Estimation (KDE) lines.
-
-
Purpose
-
-
The TimeSeriesHistogram test aims to perform a histogram analysis on time-series data to assess the distribution of
-values within a dataset over time. This test is useful for regression tasks and can be applied to various types of
-data, such as internet traffic, stock prices, and weather data, providing insights into the probability
-distribution, skewness, and kurtosis of the dataset.
-
-
Test Mechanism
-
-
This test operates on a specific column within the dataset that must have a datetime type index. For each column in
-the dataset, a histogram is created using Plotly's histplot function. If the dataset includes more than one
-time-series, a distinct histogram is plotted for each series. Additionally, a Kernel Density Estimate (KDE) line is
-drawn for each histogram, visualizing the data's underlying probability distribution. The x and y-axis labels are
-hidden to focus solely on the data distribution.
-
-
Signs of High Risk
-
-
-
The dataset lacks a column with a datetime type index.
-
The specified columns do not exist within the dataset.
-
High skewness or kurtosis in the data distribution, indicating potential bias.
-
Presence of significant outliers in the data distribution.
-
-
-
Strengths
-
-
-
Serves as a visual diagnostic tool for understanding data behavior and distribution trends.
-
Effective for analyzing both single and multiple time-series data.
-
KDE line provides a smooth estimate of the overall trend in data distribution.
-
-
-
Limitations
-
-
-
Provides a high-level view without specific numeric measures such as skewness or kurtosis.
-
The histogram loses some detail due to binning of data values.
-
Cannot handle non-numeric data columns.
-
Histogram shape may be sensitive to the number of bins used.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TimeSeriesLinePlot.html b/docs/_build/validmind/tests/data_validation/TimeSeriesLinePlot.html
deleted file mode 100644
index 802c493aa..000000000
--- a/docs/_build/validmind/tests/data_validation/TimeSeriesLinePlot.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TimeSeriesLinePlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Generates and analyses time-series data through line plots revealing trends, patterns, anomalies over time.
-
-
Purpose
-
-
The TimeSeriesLinePlot metric is designed to generate and analyze time series data through the creation of line
-plots. This assists in the initial inspection of the data by providing a visual representation of patterns, trends,
-seasonality, irregularity, and anomalies that may be present in the dataset over a period of time.
-
-
Test Mechanism
-
-
The mechanism for this Python class involves extracting the column names from the provided dataset and subsequently
-generating line plots for each column using the Plotly Python library. For every column in the dataset, a
-time-series line plot is created where the values are plotted against the dataset's datetime index. It is important
-to note that indexes that are not of datetime type will result in a ValueError.
-
-
Signs of High Risk
-
-
-
Presence of time-series data that does not have datetime indices.
-
Provided columns do not exist in the provided dataset.
-
The detection of anomalous patterns or irregularities in the time-series plots, indicating potential high model
-instability or probable predictive error.
-
-
-
Strengths
-
-
-
The visual representation of complex time series data, which simplifies understanding and helps in recognizing
-temporal trends, patterns, and anomalies.
-
The adaptability of the metric, which allows it to effectively work with multiple time series within the same
-dataset.
-
Enables the identification of anomalies and irregular patterns through visual inspection, assisting in spotting
-potential data or model performance problems.
-
-
-
Limitations
-
-
-
The effectiveness of the metric is heavily reliant on the quality and patterns of the provided time series data.
-
Exclusively a visual tool, it lacks the capability to provide quantitative measurements, making it less effective
-for comparing and ranking multiple models or when specific numerical diagnostics are needed.
-
The metric necessitates that the time-specific data has been transformed into a datetime index, with the data
-formatted correctly.
-
The metric has an inherent limitation in that it cannot extract deeper statistical insights from the time series
-data, which can limit its efficacy with complex data structures and phenomena.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TimeSeriesMissingValues.html b/docs/_build/validmind/tests/data_validation/TimeSeriesMissingValues.html
deleted file mode 100644
index 154232b9f..000000000
--- a/docs/_build/validmind/tests/data_validation/TimeSeriesMissingValues.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TimeSeriesMissingValues API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Validates time-series data quality by confirming the count of missing values is below a certain threshold.
-
-
Purpose
-
-
This test is designed to validate the quality of a historical time-series dataset by verifying that the number of
-missing values is below a specified threshold. As time-series models greatly depend on the continuity and
-temporality of data points, missing values could compromise the model's performance. Consequently, this test aims
-to ensure data quality and readiness for the machine learning model, safeguarding its predictive capacity.
-
-
Test Mechanism
-
-
The test method commences by validating if the dataset has a datetime index; if not, an error is raised. It
-establishes a lower limit threshold for missing values and performs a missing values check on each column of the
-dataset. An object for the test result is created stating whether the number of missing values is within the
-specified threshold. Additionally, the test calculates the percentage of missing values alongside the raw count.
-
-
Signs of High Risk
-
-
-
The number of missing values in any column of the dataset surpasses the threshold, marking a failure and a
-high-risk scenario. The reasons could range from incomplete data collection, faulty sensors to data preprocessing
-errors.
-
-
-
Strengths
-
-
-
Effectively identifies missing values which could adversely affect the model’s performance.
-
Applicable and customizable through the threshold parameter across different data sets.
-
Goes beyond raw numbers by calculating the percentage of missing values, offering a more relative understanding
-of data scarcity.
-
-
-
Limitations
-
-
-
Although it identifies missing values, the test does not provide solutions to handle them.
-
The test demands that the dataset should have a datetime index, hence limiting its use only to time series
-analysis.
-
The test's sensitivity to the 'min_threshold' parameter may raise false alarms if set too strictly or may
-overlook problematic data if set too loosely.
-
Solely focuses on the 'missingness' of the data and might fall short in addressing other aspects of data quality.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TimeSeriesOutliers.html b/docs/_build/validmind/tests/data_validation/TimeSeriesOutliers.html
deleted file mode 100644
index 7a5441361..000000000
--- a/docs/_build/validmind/tests/data_validation/TimeSeriesOutliers.html
+++ /dev/null
@@ -1,305 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TimeSeriesOutliers API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Identifies and visualizes outliers in time-series data using the z-score method.
-
-
Purpose
-
-
This test is designed to identify outliers in time-series data using the z-score method. It's vital for ensuring
-data quality before modeling, as outliers can skew predictive models and significantly impact their overall
-performance.
-
-
Test Mechanism
-
-
The test processes a given dataset which must have datetime indexing, checks if a 'zscore_threshold' parameter has
-been supplied, and identifies columns with numeric data types. After finding numeric columns, the implementer then
-applies the z-score method to each numeric column, identifying outliers based on the threshold provided. Each
-outlier is listed together with their variable name, z-score, timestamp, and relative threshold in a dictionary and
-converted to a DataFrame for convenient output. Additionally, it produces visual plots for each time series
-illustrating outliers in the context of the broader dataset. The 'zscore_threshold' parameter sets the limit beyond
-which a data point will be labeled as an outlier. The default threshold is set at 3, indicating that any data point
-that falls 3 standard deviations away from the mean will be marked as an outlier.
-
-
Signs of High Risk
-
-
-
Many or substantial outliers are present within the dataset, indicating significant anomalies.
-
Data points with z-scores higher than the set threshold.
-
Potential impact on the performance of machine learning models if outliers are not properly addressed.
-
-
-
Strengths
-
-
-
The z-score method is a popular and robust method for identifying outliers in a dataset.
-
Simplifies time series maintenance by requiring a datetime index.
-
Identifies outliers for each numeric feature individually.
-
Provides an elaborate report showing variables, dates, z-scores, and pass/fail tests.
-
Offers visual inspection for detected outliers through plots.
-
-
-
Limitations
-
-
-
The test only identifies outliers in numeric columns, not in categorical variables.
-
The utility and accuracy of z-scores can be limited if the data doesn't follow a normal distribution.
-
The method relies on a subjective z-score threshold for deciding what constitutes an outlier, which might not
-always be suitable depending on the dataset and use case.
-
It does not address possible ways to handle identified outliers in the data.
-
The requirement for a datetime index could limit its application.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/TooManyZeroValues.html b/docs/_build/validmind/tests/data_validation/TooManyZeroValues.html
deleted file mode 100644
index bc9b03cbb..000000000
--- a/docs/_build/validmind/tests/data_validation/TooManyZeroValues.html
+++ /dev/null
@@ -1,313 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.TooManyZeroValues API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Identifies numerical columns in a dataset that contain an excessive number of zero values, defined by a threshold
-percentage.
-
-
Purpose
-
-
The 'TooManyZeroValues' test is utilized to identify numerical columns in the dataset that may present a quantity
-of zero values considered excessive. The aim is to detect situations where these may implicate data sparsity or a
-lack of variation, limiting their effectiveness within a machine learning model. The definition of 'too many' is
-quantified as a percentage of total values, with a default set to 3%.
-
-
Test Mechanism
-
-
This test is conducted by looping through each column in the dataset and categorizing those that pertain to
-numerical data. On identifying a numerical column, the function computes the total quantity of zero values and
-their ratio to the total row count. Should the proportion exceed a pre-set threshold parameter, set by default at
-0.03 or 3%, the column is considered to have failed the test. The results for each column are summarized and
-reported, indicating the count and percentage of zero values for each numerical column, alongside a status
-indicating whether the column has passed or failed the test.
-
-
Signs of High Risk
-
-
-
Numerical columns showing a high ratio of zero values when compared to the total count of rows (exceeding the
-predetermined threshold).
-
Columns characterized by zero values across the board suggest a complete lack of data variation, signifying high
-risk.
-
-
-
Strengths
-
-
-
Assists in highlighting columns featuring an excess of zero values that could otherwise go unnoticed within a
-large dataset.
-
Provides the flexibility to alter the threshold that determines when the quantity of zero values becomes 'too
-many', thus catering to specific needs of a particular analysis or model.
-
Offers feedback in the form of both counts and percentages of zero values, which allows a closer inspection of
-the distribution and proportion of zeros within a column.
-
Targets specifically numerical data, thereby avoiding inappropriate application to non-numerical columns and
-mitigating the risk of false test failures.
-
-
-
Limitations
-
-
-
Is exclusively designed to check for zero values and doesn’t assess the potential impact of other values that
-could affect the dataset, such as extremely high or low figures, missing values, or outliers.
-
Lacks the ability to detect a repetitive pattern of zeros, which could be significant in time-series or
-longitudinal data.
-
Zero values can actually be meaningful in some contexts; therefore, tagging them as 'too many' could potentially
-misinterpret the data to some extent.
-
This test does not take into consideration the context of the dataset, and fails to recognize that within certain
-columns, a high number of zero values could be quite normal and not necessarily an indicator of poor data quality.
-
Cannot evaluate non-numerical or categorical columns, which might bring with them different types of concerns or
-issues.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/UniqueRows.html b/docs/_build/validmind/tests/data_validation/UniqueRows.html
deleted file mode 100644
index 6ce28be9b..000000000
--- a/docs/_build/validmind/tests/data_validation/UniqueRows.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.UniqueRows API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Verifies the diversity of the dataset by ensuring that the count of unique rows exceeds a prescribed threshold.
-
-
Purpose
-
-
The UniqueRows test is designed to gauge the quality of the data supplied to the machine learning model by
-verifying that the count of distinct rows in the dataset exceeds a specific threshold, thereby ensuring a varied
-collection of data. Diversity in data is essential for training an unbiased and robust model that excels when faced
-with novel data.
-
-
Test Mechanism
-
-
The testing process starts with calculating the total number of rows in the dataset. Subsequently, the count of
-unique rows is determined for each column in the dataset. If the percentage of unique rows (calculated as the ratio
-of unique rows to the overall row count) is less than the prescribed minimum percentage threshold given as a
-function parameter, the test passes. The results are cached and a final pass or fail verdict is given based on
-whether all columns have successfully passed the test.
-
-
Signs of High Risk
-
-
-
A lack of diversity in data columns, demonstrated by a count of unique rows that falls short of the preset
-minimum percentage threshold, is indicative of high risk.
-
This lack of variety in the data signals potential issues with data quality, possibly leading to overfitting in
-the model and issues with generalization, thus posing a significant risk.
-
-
-
Strengths
-
-
-
The UniqueRows test is efficient in evaluating the data's diversity across each information column in the dataset.
-
This test provides a quick, systematic method to assess data quality based on uniqueness, which can be pivotal in
-developing effective and unbiased machine learning models.
-
-
-
Limitations
-
-
-
A limitation of the UniqueRows test is its assumption that the data's quality is directly proportionate to its
-uniqueness, which may not always hold true. There might be contexts where certain non-unique rows are essential and
-should not be overlooked.
-
The test does not consider the relative 'importance' of each column in predicting the output, treating all
-columns equally.
-
This test may not be suitable or useful for categorical variables, where the count of unique categories is
-inherently limited.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/WOEBinPlots.html b/docs/_build/validmind/tests/data_validation/WOEBinPlots.html
deleted file mode 100644
index 0ec9aeb2a..000000000
--- a/docs/_build/validmind/tests/data_validation/WOEBinPlots.html
+++ /dev/null
@@ -1,309 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.WOEBinPlots API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Generates visualizations of Weight of Evidence (WoE) and Information Value (IV) for understanding predictive power
-of categorical variables in a data set.
-
-
Purpose
-
-
This test is designed to visualize the Weight of Evidence (WoE) and Information Value (IV) for categorical
-variables in a provided dataset. By showcasing the data distribution across different categories of each feature,
-it aids in understanding each variable's predictive power in the context of a classification-based machine learning
-model. Commonly used in credit scoring models, WoE and IV are robust statistical methods for evaluating a
-variable's predictive power.
-
-
Test Mechanism
-
-
The test implementation follows defined steps. Initially, it selects non-numeric columns from the dataset and
-changes them to string type, paving the way for accurate binning. It then performs an automated WoE binning
-operation on these selected features, effectively categorizing the potential values of a variable into distinct
-bins. After the binning process, the function generates two separate visualizations (a scatter chart for WoE values
-and a bar chart for IV) for each variable. These visual presentations are formed according to the spread of each
-metric across various categories of each feature.
-
-
Signs of High Risk
-
-
-
Errors occurring during the binning process.
-
Challenges in converting non-numeric columns into string data type.
-
Misbalance in the distribution of WoE and IV, with certain bins overtaking others conspicuously. This could
-denote that the model is disproportionately dependent on certain variables or categories for predictions, an
-indication of potential risks to its robustness and generalizability.
-
-
-
Strengths
-
-
-
Provides a detailed visual representation of the relationship between feature categories and the target variable.
-This grants an intuitive understanding of each feature's contribution to the model.
-
Allows for easy identification of features with high impact, facilitating feature selection and enhancing
-comprehension of the model's decision logic.
-
WoE conversions are monotonic, upholding the rank ordering of the original data points, which simplifies analysis.
-
-
-
Limitations
-
-
-
The method is largely reliant on the binning process, and an inappropriate binning threshold or bin number choice
-might result in a misrepresentation of the variable's distribution.
-
While excellent for categorical data, the encoding of continuous variables into categorical can sometimes lead to
-information loss.
-
Extreme or outlier values can dramatically affect the computation of WoE and IV, skewing results.
-
The method requires a sufficient number of events per bin to generate a reliable information value and weight of
-evidence.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/WOEBinTable.html b/docs/_build/validmind/tests/data_validation/WOEBinTable.html
deleted file mode 100644
index f64e74375..000000000
--- a/docs/_build/validmind/tests/data_validation/WOEBinTable.html
+++ /dev/null
@@ -1,300 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.WOEBinTable API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the Weight of Evidence (WoE) and Information Value (IV) of each feature to evaluate its predictive power
-in a binary classification model.
-
-
Purpose
-
-
The Weight of Evidence (WoE) and Information Value (IV) test is designed to evaluate the predictive power of each
-feature in a machine learning model. This test generates binned groups of values from each feature, computes the
-WoE and IV for each bin, and provides insights into the relationship between each feature and the target variable,
-illustrating their contribution to the model's predictive capabilities.
-
-
Test Mechanism
-
-
The test uses the scorecardpy.woebin method to perform automatic binning of the dataset based on WoE. The method
-accepts a list of break points for binning numeric variables through the parameter breaks_adj. If no breaks are
-provided, it uses default binning. The bins are then used to calculate the WoE and IV values, effectively creating
-a dataframe that includes the bin boundaries, WoE, and IV values for each feature. A target variable is required
-in the dataset to perform this analysis.
-
-
Signs of High Risk
-
-
-
High IV values, indicating variables with excessive predictive power which might lead to overfitting.
-
Errors during the binning process, potentially due to inappropriate data types or poorly defined bins.
-
-
-
Strengths
-
-
-
Highly effective for feature selection in binary classification problems, as it quantifies the predictive
-information within each feature concerning the binary outcome.
-
The WoE transformation creates a monotonic relationship between the target and independent variables.
-
-
-
Limitations
-
-
-
Primarily designed for binary classification tasks, making it less applicable or reliable for multi-class
-classification or regression tasks.
-
Potential difficulties if the dataset has many features, non-binnable features, or non-numeric features.
-
The metric does not help in distinguishing whether the observed predictive factor is due to data randomness or a
-true phenomenon.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/ZivotAndrewsArch.html b/docs/_build/validmind/tests/data_validation/ZivotAndrewsArch.html
deleted file mode 100644
index 6f70bc649..000000000
--- a/docs/_build/validmind/tests/data_validation/ZivotAndrewsArch.html
+++ /dev/null
@@ -1,300 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.ZivotAndrewsArch API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the order of integration and stationarity of time series data using the Zivot-Andrews unit root test.
-
-
Purpose
-
-
The Zivot-Andrews Arch metric is used to evaluate the order of integration for time series data in a machine
-learning model. It's designed to test for stationarity, a crucial aspect of time series analysis, where data points
-are independent of time. Stationarity means that the statistical properties such as mean, variance, and
-autocorrelation are constant over time.
-
-
Test Mechanism
-
-
The Zivot-Andrews unit root test is performed on each feature in the dataset using the ZivotAndrews function from
-the arch.unitroot module. This function returns several metrics for each feature, including the statistical
-value, p-value (probability value), the number of lags used, and the number of observations. The p-value is used to
-decide on the null hypothesis (the time series has a unit root and is non-stationary) based on a chosen level of
-significance.
-
-
Signs of High Risk
-
-
-
A high p-value suggests high risk, indicating insufficient evidence to reject the null hypothesis, implying that
-the time series has a unit root and is non-stationary.
-
Non-stationary time series data can lead to misleading statistics and unreliable machine learning models.
-
-
-
Strengths
-
-
-
Dynamically tests for stationarity against structural breaks in time series data, offering robust evaluation of
-stationarity in features.
-
Especially beneficial with financial, economic, or other time-series data where data observations lack a
-consistent pattern and structural breaks may occur.
-
-
-
Limitations
-
-
-
Assumes data is derived from a single-equation, autoregressive model, making it less appropriate for multivariate
-time series data or data not aligning with this model.
-
May not account for unexpected shocks or changes in the series trend, both of which can significantly impact data
-stationarity.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/nlp.html b/docs/_build/validmind/tests/data_validation/nlp.html
deleted file mode 100644
index 81e3b938d..000000000
--- a/docs/_build/validmind/tests/data_validation/nlp.html
+++ /dev/null
@@ -1,248 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.nlp API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the most frequent non-stopwords in a text column for identifying prevalent language patterns.
-
-
Purpose
-
-
The CommonWords metric is used to identify and visualize the most prevalent words within a specified text column of
-a dataset. This provides insights into the prevalent language patterns and vocabulary, especially useful in Natural
-Language Processing (NLP) tasks such as text classification and text summarization.
-
-
Test Mechanism
-
-
The test methodology involves splitting the specified text column's entries into words, collating them into a
-corpus, and then counting the frequency of each word using the Counter. The forty most frequently occurring
-non-stopwords are then visualized in an interactive bar chart using Plotly, where the x-axis represents the words,
-and the y-axis indicates their frequency of occurrence.
-
-
Signs of High Risk
-
-
-
A lack of distinct words within the list, or the most common words being stopwords.
-
Frequent occurrence of irrelevant or inappropriate words could point out a poorly curated or noisy dataset.
-
An error returned due to the absence of a valid Dataset object, indicating high risk as the metric cannot be
-effectively implemented without it.
-
-
-
Strengths
-
-
-
The metric provides clear insights into the language features – specifically word frequency – of unstructured
-text data.
-
It can reveal prominent vocabulary and language patterns, which prove vital for feature extraction in NLP tasks.
-
The interactive visualization helps in quickly capturing the patterns and understanding the data intuitively.
-
-
-
Limitations
-
-
-
The test disregards semantic or context-related information as it solely focuses on word frequency.
-
It intentionally ignores stopwords, which might carry necessary significance in certain scenarios.
-
The applicability is limited to English-language text data as English stopwords are used for filtering, hence
-cannot account for data in other languages.
-
The metric requires a valid Dataset object, indicating a dependency condition that limits its broader
-applicability.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/nlp/Hashtags.html b/docs/_build/validmind/tests/data_validation/nlp/Hashtags.html
deleted file mode 100644
index 66787a387..000000000
--- a/docs/_build/validmind/tests/data_validation/nlp/Hashtags.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.nlp.Hashtags API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses hashtag frequency in a text column, highlighting usage trends and potential dataset bias or spam.
-
-
Purpose
-
-
The Hashtags test is designed to measure the frequency of hashtags used within a given text column in a dataset. It
-is particularly useful for natural language processing tasks such as text classification and text summarization.
-The goal is to identify common trends and patterns in the use of hashtags, which can serve as critical indicators
-or features within a machine learning model.
-
-
Test Mechanism
-
-
The test implements a regular expression (regex) to extract all hashtags from the specified text column. For each
-hashtag found, it makes a tally of its occurrences. It then outputs a list of the top N hashtags (default is 25,
-but customizable), sorted by their counts in descending order. The results are also visualized in a bar plot, with
-frequency counts on the y-axis and the corresponding hashtags on the x-axis.
-
-
Signs of High Risk
-
-
-
A low diversity in the usage of hashtags, as indicated by a few hashtags being used disproportionately more than
-others.
-
Repeated usage of one or few hashtags can be indicative of spam or a biased dataset.
-
If there are no or extremely few hashtags found in the dataset, it perhaps signifies that the text data does not
-contain structured social media data.
-
-
-
Strengths
-
-
-
Provides a concise visual representation of the frequency of hashtags, which can be critical for understanding
-trends about a particular topic in text data.
-
Instrumental in tasks specifically related to social media text analytics, such as opinion analysis and trend
-discovery.
-
Adaptable, allowing the flexibility to determine the number of top hashtags to be analyzed.
-
-
-
Limitations
-
-
-
Assumes the presence of hashtags and therefore may not be applicable for text datasets that do not contain
-hashtags (e.g., formal documents, scientific literature).
-
Language-specific limitations of hashtag formulations are not taken into account.
-
Does not account for typographical errors, variations, or synonyms in hashtags.
-
Does not provide context or sentiment associated with the hashtags, so the information provided may have limited
-utility on its own.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/nlp/LanguageDetection.html b/docs/_build/validmind/tests/data_validation/nlp/LanguageDetection.html
deleted file mode 100644
index 8cbc89ad7..000000000
--- a/docs/_build/validmind/tests/data_validation/nlp/LanguageDetection.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.nlp.LanguageDetection API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the diversity of languages in a textual dataset by detecting and visualizing the distribution of languages.
-
-
Purpose
-
-
The Language Detection test aims to identify and visualize the distribution of languages present within a textual
-dataset. This test helps in understanding the diversity of languages in the data, which is crucial for developing
-and validating multilingual models.
-
-
Test Mechanism
-
-
This test operates by:
-
-
-
Checking if the dataset has a specified text column.
-
Using a language detection library to determine the language of each text entry in the dataset.
-
Generating a histogram plot of the language distribution, with language codes on the x-axis and their frequencies
-on the y-axis.
-
-
-
If the text column is not specified, a ValueError is raised to ensure proper dataset configuration.
-
-
Signs of High Risk
-
-
-
A high proportion of entries returning "Unknown" language codes.
-
Detection of unexpectedly diverse or incorrect language codes, indicating potential data quality issues.
-
Significant imbalance in language distribution, which might indicate potential biases in the dataset.
-
-
-
Strengths
-
-
-
Provides a visual representation of language diversity within the dataset.
-
Helps identify data quality issues related to incorrect or unknown language detection.
-
Useful for ensuring that multilingual models have adequate and appropriate representation from various languages.
-
-
-
Limitations
-
-
-
Dependency on the accuracy of the language detection library, which may not be perfect.
-
Languages with similar structures or limited text length may be incorrectly classified.
-
The test returns "Unknown" for entries where language detection fails, which might mask underlying issues with
-certain languages or text formats.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/nlp/Mentions.html b/docs/_build/validmind/tests/data_validation/nlp/Mentions.html
deleted file mode 100644
index 0111fd643..000000000
--- a/docs/_build/validmind/tests/data_validation/nlp/Mentions.html
+++ /dev/null
@@ -1,302 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.nlp.Mentions API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Calculates and visualizes frequencies of '@' prefixed mentions in a text-based dataset for NLP model analysis.
-
-
Purpose
-
-
The "Mentions" test is designed to gauge the quality of data in a Natural Language Processing (NLP) or text-focused
-Machine Learning model. The primary objective is to identify and calculate the frequency of 'mentions' within a
-chosen text column of a dataset. A 'mention' in this context refers to individual text elements that are prefixed
-by '@'. The output of this test reveals the most frequently mentioned entities or usernames, which can be integral
-for applications such as social media analyses or customer sentiment analyses.
-
-
Test Mechanism
-
-
The test first verifies the existence of a text column in the provided dataset. It then employs a regular
-expression pattern to extract mentions from the text. Subsequently, the frequency of each unique mention is
-calculated. The test selects the most frequent mentions based on default or user-defined parameters, the default
-being the top 25, for representation. This process of thresholding forms the core of the test. A treemap plot
-visualizes the test results, where the size of each rectangle corresponds to the frequency of a particular mention.
-
-
Signs of High Risk
-
-
-
The lack of a valid text column in the dataset, which would result in the failure of the test execution.
-
The absence of any mentions within the text data, indicating that there might not be any text associated with
-'@'. This situation could point toward sparse or poor-quality data, thereby hampering the model's generalization or
-learning capabilities.
-
-
-
Strengths
-
-
-
The test is specifically optimized for text-based datasets which gives it distinct power in the context of NLP.
-
It enables quick identification and visually appealing representation of the predominant elements or mentions.
-
It can provide crucial insights about the most frequently mentioned entities or usernames.
-
-
-
Limitations
-
-
-
The test only recognizes mentions that are prefixed by '@', hence useful textual aspects not preceded by '@'
-might be ignored.
-
This test isn't suited for datasets devoid of textual data.
-
It does not provide insights on less frequently occurring data or outliers, which means potentially significant
-patterns could be overlooked.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/nlp/PolarityAndSubjectivity.html b/docs/_build/validmind/tests/data_validation/nlp/PolarityAndSubjectivity.html
deleted file mode 100644
index 469537011..000000000
--- a/docs/_build/validmind/tests/data_validation/nlp/PolarityAndSubjectivity.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.nlp.PolarityAndSubjectivity API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Analyzes the polarity and subjectivity of text data within a given dataset to visualize the sentiment distribution.
-
-
Purpose
-
-
The Polarity and Subjectivity test is designed to evaluate the sentiment expressed in textual data. By analyzing
-these aspects, it helps to identify the emotional tone and subjectivity of the dataset, which could be crucial in
-understanding customer feedback, social media sentiments, or other text-related data.
-
-
Test Mechanism
-
-
This test uses TextBlob to compute the polarity and subjectivity scores of textual data in a given dataset. The
-mechanism includes:
-
-
-
Iterating through each text entry in the specified column of the dataset.
-
Applying the TextBlob library to compute the polarity (ranging from -1 for negative sentiment to +1 for positive
-sentiment) and subjectivity (ranging from 0 for objective to 1 for subjective) for each entry.
-
Creating a scatter plot using Plotly to visualize the relationship between polarity and subjectivity.
-
-
-
Signs of High Risk
-
-
-
High concentration of negative polarity values indicating prevalent negative sentiments.
-
High subjectivity scores suggesting the text data is largely opinion-based rather than factual.
-
Disproportionate clusters of extreme scores (e.g., many points near -1 or +1 polarity).
-
-
-
Strengths
-
-
-
Quantifies sentiment and subjectivity which can provide actionable insights.
-
Visualizes sentiment distribution, aiding in easy interpretation.
-
Utilizes well-established TextBlob library for sentiment analysis.
-
-
-
Limitations
-
-
-
Polarity and subjectivity calculations may oversimplify nuanced text sentiments.
-
Reliance on TextBlob which may not be accurate for all domains or contexts.
-
Visualization could become cluttered with very large datasets, making interpretation difficult.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/nlp/Punctuations.html b/docs/_build/validmind/tests/data_validation/nlp/Punctuations.html
deleted file mode 100644
index 057b4eaf5..000000000
--- a/docs/_build/validmind/tests/data_validation/nlp/Punctuations.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.nlp.Punctuations API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Analyzes and visualizes the frequency distribution of punctuation usage in a given text dataset.
-
-
Purpose
-
-
The Punctuations Metric's primary purpose is to analyze the frequency of punctuation usage within a given text
-dataset. This is often used in Natural Language Processing tasks, such as text classification and text
-summarization.
-
-
Test Mechanism
-
-
The test begins by verifying that the input "dataset" is of the type VMDataset. The count_mode parameter must be
-either "token" (counts punctuation marks as individual tokens) or "word" (counts punctuation marks within words).
-Following that, a corpus is created from the dataset by splitting its text on spaces. Each unique punctuation
-character in the text corpus is then tallied. The frequency distribution of each punctuation symbol is visualized
-as a bar graph, with these results being stored as Figures and associated with the main Punctuations object.
-
-
Signs of High Risk
-
-
-
Excessive or unusual frequency of specific punctuation marks, potentially denoting dubious quality, data
-corruption, or skewed data.
-
-
-
Strengths
-
-
-
Provides valuable insights into the distribution of punctuation usage in a text dataset.
-
Important in validating the quality, consistency, and nature of the data.
-
Can provide hints about the style or tonality of the text corpus, such as informal and emotional context
-indicated by frequent exclamation marks.
-
-
-
Limitations
-
-
-
Focuses solely on punctuation usage, potentially missing other important textual characteristics.
-
General cultural or tonality assumptions based on punctuation distribution can be misguiding, as these vary
-across different languages and contexts.
-
Less effective with languages that use non-standard or different punctuation.
-
Visualization may lack interpretability when there are many unique punctuation marks in the dataset.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/nlp/Sentiment.html b/docs/_build/validmind/tests/data_validation/nlp/Sentiment.html
deleted file mode 100644
index 1457538d9..000000000
--- a/docs/_build/validmind/tests/data_validation/nlp/Sentiment.html
+++ /dev/null
@@ -1,295 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.nlp.Sentiment API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Analyzes the sentiment of text data within a dataset using the VADER sentiment analysis tool.
-
-
Purpose
-
-
The Sentiment test evaluates the overall sentiment of text data within a dataset. By analyzing sentiment scores, it
-aims to ensure that the model is interpreting text data accurately and is not biased towards a particular sentiment.
-
-
Test Mechanism
-
-
This test uses the VADER (Valence Aware Dictionary and sEntiment Reasoner) SentimentIntensityAnalyzer. It processes
-each text entry in a specified column of the dataset to calculate the compound sentiment score, which represents
-the overall sentiment polarity. The distribution of these sentiment scores is then visualized using a KDE (Kernel
-Density Estimation) plot, highlighting any skewness or concentration in sentiment.
-
-
Signs of High Risk
-
-
-
Extreme polarity in sentiment scores, indicating potential bias.
-
Unusual concentration of sentiment scores in a specific range.
-
Significant deviation from expected sentiment distribution for the given text data.
-
-
-
Strengths
-
-
-
Provides a clear visual representation of sentiment distribution.
-
Uses a well-established sentiment analysis tool (VADER).
-
Can handle a wide range of text data, making it flexible for various applications.
-
-
-
Limitations
-
-
-
May not capture nuanced or context-specific sentiments.
-
Relies heavily on the accuracy of the VADER sentiment analysis tool.
-
Visualization alone may not provide comprehensive insights into underlying causes of sentiment distribution.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/nlp/StopWords.html b/docs/_build/validmind/tests/data_validation/nlp/StopWords.html
deleted file mode 100644
index 97e63f9b5..000000000
--- a/docs/_build/validmind/tests/data_validation/nlp/StopWords.html
+++ /dev/null
@@ -1,311 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.nlp.StopWords API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates and visualizes the frequency of English stop words in a text dataset against a defined threshold.
-
-
Purpose
-
-
The StopWords threshold test is a tool designed for assessing the quality of text data in an ML model. It focuses
-on the identification and analysis of "stop words" in a given dataset. Stop words are frequent, common, yet
-semantically insignificant words (for example: "the", "and", "is") in a language. This test evaluates the
-proportion of stop words to the total word count in the dataset, in essence, scrutinizing the frequency of stop
-word usage. The core objective is to highlight the prevalent stop words based on their usage frequency, which can
-be instrumental in cleaning the data from noise and improving ML model performance.
-
-
Test Mechanism
-
-
The StopWords test initiates on receiving an input of a 'VMDataset' object. Absence of such an object will trigger
-an error. The methodology involves inspection of the text column of the VMDataset to create a 'corpus' (a
-collection of written texts). Leveraging the Natural Language Toolkit's (NLTK) stop word repository, the test
-screens the corpus for any stop words and documents their frequency. It further calculates the percentage usage of
-each stop word compared to the total word count in the corpus. This percentage is evaluated against a predefined
-'min_percent_threshold'. If this threshold is breached, the test returns a failed output. Top prevailing stop words
-along with their usage percentages are returned, facilitated by a bar chart visualization of these stop words and
-their frequency.
-
-
Signs of High Risk
-
-
-
A percentage of any stop words exceeding the predefined 'min_percent_threshold'.
-
High frequency of stop words in the dataset which may adversely affect the application's analytical performance
-due to noise creation.
-
-
-
Strengths
-
-
-
The ability to scrutinize and quantify the usage of stop words.
-
Provides insights into potential noise in the text data due to stop words.
-
Directly aids in enhancing model training efficiency.
-
Includes a bar chart visualization feature to easily interpret and action upon the stop words frequency
-information.
-
-
-
Limitations
-
-
-
The test only supports English stop words, making it less effective with datasets of other languages.
-
The 'min_percent_threshold' parameter may require fine-tuning for different datasets, impacting the overall
-effectiveness of the test.
-
Contextual use of the stop words within the dataset is not considered, potentially overlooking their significance
-in certain contexts.
-
The test focuses specifically on the frequency of stop words, not providing direct measures of model performance
-or predictive accuracy.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/nlp/TextDescription.html b/docs/_build/validmind/tests/data_validation/nlp/TextDescription.html
deleted file mode 100644
index 9584383a9..000000000
--- a/docs/_build/validmind/tests/data_validation/nlp/TextDescription.html
+++ /dev/null
@@ -1,323 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.nlp.TextDescription API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Conducts comprehensive textual analysis on a dataset using NLTK to evaluate various parameters and generate
-visualizations.
-
-
Purpose
-
-
The TextDescription test aims to conduct a thorough textual analysis of a dataset using the NLTK (Natural Language
-Toolkit) library. It evaluates various metrics such as total words, total sentences, average sentence length, total
-paragraphs, total unique words, most common words, total punctuations, and lexical diversity. The goal is to
-understand the nature of the text and anticipate challenges machine learning models might face in text processing,
-language understanding, or summarization tasks.
-
-
Test Mechanism
-
-
The test works by:
-
-
-
Parsing the dataset and tokenizing the text into words, sentences, and paragraphs using NLTK.
-
Removing stopwords and unwanted tokens.
-
Calculating parameters like total words, total sentences, average sentence length, total paragraphs, total unique
-words, total punctuations, and lexical diversity.
-
Generating scatter plots to visualize correlations between various metrics (e.g., Total Words vs Total Sentences).
-
-
-
Signs of High Risk
-
-
-
Anomalies or increased complexity in lexical diversity.
-
Longer sentences and paragraphs.
-
High uniqueness of words.
-
Large number of unwanted tokens.
-
Missing or erroneous visualizations.
-
-
-
Strengths
-
-
-
Essential for pre-processing text data in machine learning models.
-
Provides a comprehensive breakdown of text data, aiding in understanding its complexity.
-
Generates visualizations to help comprehend text structure and complexity.
-
-
-
Limitations
-
-
-
Highly dependent on the NLTK library, limiting the test to supported languages.
-
Limited customization for removing undesirable tokens and stop words.
-
Does not consider semantic or grammatical complexities.
-
Assumes well-structured documents, which may result in inaccuracies with poorly formatted text.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/data_validation/nlp/Toxicity.html b/docs/_build/validmind/tests/data_validation/nlp/Toxicity.html
deleted file mode 100644
index 293b084bd..000000000
--- a/docs/_build/validmind/tests/data_validation/nlp/Toxicity.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.data_validation.nlp.Toxicity API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the toxicity of text data within a dataset to visualize the distribution of toxicity scores.
-
-
Purpose
-
-
The Toxicity test aims to evaluate the level of toxic content present in a text dataset by leveraging a pre-trained
-toxicity model. It helps in identifying potentially harmful or offensive language that may negatively impact users
-or stakeholders.
-
-
Test Mechanism
-
-
This test uses a pre-trained toxicity evaluation model and applies it to each text entry in the specified column of
-a dataset’s dataframe. The procedure involves:
-
-
-
Loading a pre-trained toxicity model.
-
Extracting the text from the specified column in the dataset.
-
Computing toxicity scores for each text entry.
-
Generating a KDE (Kernel Density Estimate) plot to visualize the distribution of these toxicity scores.
-
-
-
Signs of High Risk
-
-
-
High concentration of high toxicity scores in the KDE plot.
-
A significant proportion of text entries with toxicity scores above a predefined threshold.
-
Wide distribution of toxicity scores, indicating inconsistency in content quality.
-
-
-
Strengths
-
-
-
Provides a visual representation of toxicity distribution, making it easier to identify outliers.
-
Uses a robust pre-trained model for toxicity evaluation.
-
Can process large text datasets efficiently.
-
-
-
Limitations
-
-
-
Depends on the accuracy and bias of the pre-trained toxicity model.
-
Does not provide context-specific insights, which may be necessary for nuanced understanding.
-
May not capture all forms of subtle or indirect toxic language.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation.html b/docs/_build/validmind/tests/model_validation.html
deleted file mode 100644
index 76cf0d275..000000000
--- a/docs/_build/validmind/tests/model_validation.html
+++ /dev/null
@@ -1,256 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the quality of machine-generated text using BERTScore metrics and visualizes results through histograms
-and bar charts, alongside compiling a comprehensive table of descriptive statistics.
-
-
Purpose
-
-
This function is designed to assess the quality of text generated by machine learning models using BERTScore
-metrics. BERTScore evaluates text generation models' performance by calculating precision, recall, and F1 score
-based on BERT contextual embeddings.
-
-
Test Mechanism
-
-
The function starts by extracting the true and predicted values from the provided dataset and model. It then
-initializes the BERTScore evaluator. For each pair of true and predicted texts, the function calculates the
-BERTScore metrics and compiles them into a dataframe. Histograms and bar charts are generated for each BERTScore
-metric (Precision, Recall, and F1 Score) to visualize their distribution. Additionally, a table of descriptive
-statistics (mean, median, standard deviation, minimum, and maximum) is compiled for each metric, providing a
-comprehensive summary of the model's performance. The test uses the evaluation_model param to specify the
-huggingface model to use for evaluation. microsoft/deberta-xlarge-mnli is the best-performing model but is
-very large and may be slow without a GPU. microsoft/deberta-large-mnli is a smaller model that is faster to
-run and distilbert-base-uncased is much lighter and can run on a CPU but is less accurate.
-
-
Signs of High Risk
-
-
-
Consistently low scores across BERTScore metrics could indicate poor quality in the generated text, suggesting
-that the model fails to capture the essential content of the reference texts.
-
Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
-
Low recall scores may indicate that important information from the reference text is being omitted.
-
An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the
-model's ability to balance informativeness and conciseness.
-
-
-
Strengths
-
-
-
Provides a multifaceted evaluation of text quality through different BERTScore metrics, offering a detailed view
-of model performance.
-
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the
-scores.
-
Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
-
-
-
Limitations
-
-
-
BERTScore relies on the contextual embeddings from BERT models, which may not fully capture all nuances of text
-similarity.
-
The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
-
While useful for comparison, BERTScore metrics alone do not provide a complete assessment of a model's
-performance and should be supplemented with other metrics and qualitative analysis.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/BleuScore.html b/docs/_build/validmind/tests/model_validation/BleuScore.html
deleted file mode 100644
index 46614a2a0..000000000
--- a/docs/_build/validmind/tests/model_validation/BleuScore.html
+++ /dev/null
@@ -1,306 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.BleuScore API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the quality of machine-generated text using BLEU metrics and visualizes the results through histograms
-and bar charts, alongside compiling a comprehensive table of descriptive statistics for BLEU scores.
-
-
Purpose
-
-
This function is designed to assess the quality of text generated by machine learning models using the BLEU metric.
-BLEU, which stands for Bilingual Evaluation Understudy, is a metric used to evaluate the overlap of n-grams between
-the machine-generated text and reference texts. This evaluation is crucial for tasks such as text summarization,
-machine translation, and text generation, where the goal is to produce text that accurately reflects the content
-and meaning of human-crafted references.
-
-
Test Mechanism
-
-
The function starts by extracting the true and predicted values from the provided dataset and model. It then
-initializes the BLEU evaluator. For each pair of true and predicted texts, the function calculates the BLEU scores
-and compiles them into a dataframe. Histograms and bar charts are generated for the BLEU scores to visualize their
-distribution. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and
-maximum) is compiled for the BLEU scores, providing a comprehensive summary of the model's performance.
-
-
Signs of High Risk
-
-
-
Consistently low BLEU scores could indicate poor quality in the generated text, suggesting that the model fails
-to capture the essential content of the reference texts.
-
Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
-
Low recall scores may indicate that important information from the reference text is being omitted.
-
An imbalanced performance between precision and recall, reflected by a low BLEU score, could signal issues in the
-model's ability to balance informativeness and conciseness.
-
-
-
Strengths
-
-
-
Provides a straightforward and widely-used evaluation of text quality through BLEU scores.
-
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the
-scores.
-
Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
-
-
-
Limitations
-
-
-
BLEU metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or
-grammatical quality of the text.
-
The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
-
While useful for comparison, BLEU scores alone do not provide a complete assessment of a model's performance and
-should be supplemented with other metrics and qualitative analysis.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/ClusterSizeDistribution.html b/docs/_build/validmind/tests/model_validation/ClusterSizeDistribution.html
deleted file mode 100644
index 9a4a19535..000000000
--- a/docs/_build/validmind/tests/model_validation/ClusterSizeDistribution.html
+++ /dev/null
@@ -1,305 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.ClusterSizeDistribution API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the performance of clustering models by comparing the distribution of cluster sizes in model predictions
-with the actual data.
-
-
Purpose
-
-
The Cluster Size Distribution test aims to assess the performance of clustering models by comparing the
-distribution of cluster sizes in the model's predictions with the actual data. This comparison helps determine if
-the clustering model's output aligns well with the true cluster distribution, providing insights into the model's
-accuracy and performance.
-
-
Test Mechanism
-
-
The test mechanism involves the following steps:
-
-
-
Run the clustering model on the provided dataset to obtain predictions.
-
Convert both the actual and predicted outputs into pandas dataframes.
-
Use pandas built-in functions to derive the cluster size distributions from these dataframes.
-
Construct two histograms: one for the actual cluster size distribution and one for the predicted distribution.
-
Plot the histograms side-by-side for visual comparison.
-
-
-
Signs of High Risk
-
-
-
Discrepancies between the actual cluster size distribution and the predicted cluster size distribution.
-
Irregular distribution of data across clusters in the predicted outcomes.
-
High number of outlier clusters suggesting the model struggles to correctly group data.
-
-
-
Strengths
-
-
-
Provides a visual and intuitive way to compare the clustering model's performance against actual data.
-
Effectively reveals where the model may be over- or underestimating cluster sizes.
-
Versatile as it works well with any clustering model.
-
-
-
Limitations
-
-
-
Assumes that the actual cluster distribution is optimal, which may not always be the case.
-
Relies heavily on visual comparison, which could be subjective and may not offer a precise numerical measure of
-performance.
-
May not fully capture other important aspects of clustering, such as cluster density, distances between clusters,
-and the shape of clusters.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/ContextualRecall.html b/docs/_build/validmind/tests/model_validation/ContextualRecall.html
deleted file mode 100644
index afa8c7fc6..000000000
--- a/docs/_build/validmind/tests/model_validation/ContextualRecall.html
+++ /dev/null
@@ -1,307 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.ContextualRecall API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates a Natural Language Generation model's ability to generate contextually relevant and factually correct
-text, visualizing the results through histograms and bar charts, alongside compiling a comprehensive table of
-descriptive statistics for contextual recall scores.
-
-
Purpose
-
-
The Contextual Recall metric is used to evaluate the ability of a natural language generation (NLG) model to
-generate text that appropriately reflects the given context or prompt. It measures the model's capability to
-remember and reproduce the main context in its resulting output. This metric is critical in natural language
-processing tasks, as the coherency and contextuality of the generated text are essential.
-
-
Test Mechanism
-
-
The function starts by extracting the true and predicted values from the provided dataset and model. It then
-tokenizes the reference and candidate texts into discernible words or tokens using NLTK. The token overlap between
-the reference and candidate texts is identified, and the Contextual Recall score is computed by dividing the number
-of overlapping tokens by the total number of tokens in the reference text. Scores are calculated for each test
-dataset instance, resulting in an array of scores. These scores are visualized using a histogram and a bar chart to
-show score variations across different rows. Additionally, a table of descriptive statistics (mean, median,
-standard deviation, minimum, and maximum) is compiled for the contextual recall scores, providing a comprehensive
-summary of the model's performance.
-
-
Signs of High Risk
-
-
-
Low contextual recall scores could indicate that the model is not effectively reflecting the original context in
-its output, leading to incoherent or contextually misaligned text.
-
A consistent trend of low recall scores could suggest underperformance of the model.
-
-
-
Strengths
-
-
-
Provides a quantifiable measure of a model's adherence to the context and factual elements of the generated
-narrative.
-
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of
-contextual recall scores.
-
Descriptive statistics offer a concise summary of the model's performance in generating contextually relevant
-texts.
-
-
-
Limitations
-
-
-
The focus on word overlap could result in high scores for texts that use many common words, even when these texts
-lack coherence or meaningful context.
-
This metric does not consider the order of words, which could lead to overestimated scores for scrambled outputs.
-
Models that effectively use infrequent words might be undervalued, as these words might not overlap as often.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/FeaturesAUC.html b/docs/_build/validmind/tests/model_validation/FeaturesAUC.html
deleted file mode 100644
index a6490e8af..000000000
--- a/docs/_build/validmind/tests/model_validation/FeaturesAUC.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.FeaturesAUC API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the discriminatory power of each individual feature within a binary classification model by calculating
-the Area Under the Curve (AUC) for each feature separately.
-
-
Purpose
-
-
The central objective of this metric is to quantify how well each feature on its own can differentiate between the
-two classes in a binary classification problem. It serves as a univariate analysis tool that can help in
-pre-modeling feature selection or post-modeling interpretation.
-
-
Test Mechanism
-
-
For each feature, the metric treats the feature values as raw scores to compute the AUC against the actual binary
-outcomes. It provides an AUC value for each feature, offering a simple yet powerful indication of each feature's
-univariate classification strength.
-
-
Signs of High Risk
-
-
-
A feature with a low AUC score may not be contributing significantly to the differentiation between the two
-classes, which could be a concern if it is expected to be predictive.
-
Conversely, a surprisingly high AUC for a feature not believed to be informative may suggest data leakage or
-other issues with the data.
-
-
-
Strengths
-
-
-
By isolating each feature, it highlights the individual contribution of features to the classification task
-without the influence of other variables.
-
Useful for both initial feature evaluation and for providing insights into the model's reliance on individual
-features after model training.
-
-
-
Limitations
-
-
-
Does not reflect the combined effects of features or any interaction between them, which can be critical in
-certain models.
-
The AUC values are calculated without considering the model's use of the features, which could lead to different
-interpretations of feature importance when considering the model holistically.
-
This metric is applicable only to binary classification tasks and cannot be directly extended to multiclass
-classification or regression without modifications.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/MeteorScore.html b/docs/_build/validmind/tests/model_validation/MeteorScore.html
deleted file mode 100644
index 063f9bf7f..000000000
--- a/docs/_build/validmind/tests/model_validation/MeteorScore.html
+++ /dev/null
@@ -1,310 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.MeteorScore API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the quality of machine-generated translations by comparing them to human-produced references using the
-METEOR score, which evaluates precision, recall, and word order.
-
-
Purpose
-
-
The METEOR (Metric for Evaluation of Translation with Explicit ORdering) score is designed to evaluate the quality
-of machine translations by comparing them against reference translations. It emphasizes both the accuracy and
-fluency of translations, incorporating precision, recall, and word order into its assessment.
-
-
Test Mechanism
-
-
The function starts by extracting the true and predicted values from the provided dataset and model. The METEOR
-score is computed for each pair of machine-generated translation (prediction) and its corresponding human-produced
-reference. This is done by considering unigram matches between the translations, including matches based on surface
-forms, stemmed forms, and synonyms. The score is a combination of unigram precision and recall, adjusted for word
-order through a fragmentation penalty. Scores are compiled into a dataframe, and histograms and bar charts are
-generated to visualize the distribution of METEOR scores. Additionally, a table of descriptive statistics (mean,
-median, standard deviation, minimum, and maximum) is compiled for the METEOR scores, providing a comprehensive
-summary of the model's performance.
-
-
Signs of High Risk
-
-
-
Lower METEOR scores can indicate a lack of alignment between the machine-generated translations and their
-human-produced references, highlighting potential deficiencies in both the accuracy and fluency of translations.
-
Significant discrepancies in word order or an excessive fragmentation penalty could signal issues with how the
-translation model processes and reconstructs sentence structures, potentially compromising the natural flow of
-translated text.
-
Persistent underperformance across a variety of text types or linguistic contexts might suggest a broader
-inability of the model to adapt to the nuances of different languages or dialects, pointing towards gaps in its
-training or inherent limitations.
-
-
-
Strengths
-
-
-
Incorporates a balanced consideration of precision and recall, weighted towards recall to reflect the importance
-of content coverage in translations.
-
Directly accounts for word order, offering a nuanced evaluation of translation fluency beyond simple lexical
-matching.
-
Adapts to various forms of lexical similarity, including synonyms and stemmed forms, allowing for flexible
-matching.
-
-
-
Limitations
-
-
-
While comprehensive, the complexity of METEOR's calculation can make it computationally intensive, especially for
-large datasets.
-
The use of external resources for synonym and stemming matching may introduce variability based on the resources'
-quality and relevance to the specific translation task.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/ModelMetadata.html b/docs/_build/validmind/tests/model_validation/ModelMetadata.html
deleted file mode 100644
index b80d5f8ee..000000000
--- a/docs/_build/validmind/tests/model_validation/ModelMetadata.html
+++ /dev/null
@@ -1,286 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.ModelMetadata API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Compare metadata of different models and generate a summary table with the results.
-
-
Purpose: The purpose of this function is to compare the metadata of different models, including information about their architecture, framework, framework version, and programming language.
-
-
Test Mechanism: The function retrieves the metadata for each model using get_model_info, renames columns according to a predefined set of labels, and compiles this information into a summary table.
-
-
Signs of High Risk:
-
-
-
Inconsistent or missing metadata across models can indicate potential issues in model documentation or management.
-
Significant differences in framework versions or programming languages might pose challenges in model integration and deployment.
-
-
-
Strengths:
-
-
-
Provides a clear comparison of essential model metadata.
-
Standardizes metadata labels for easier interpretation and comparison.
-
Helps identify potential compatibility or consistency issues across models.
-
-
-
Limitations:
-
-
-
Assumes that the get_model_info function returns all necessary metadata fields.
-
Relies on the correctness and completeness of the metadata provided by each model.
-
Does not include detailed parameter information, focusing instead on high-level metadata.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/ModelPredictionResiduals.html b/docs/_build/validmind/tests/model_validation/ModelPredictionResiduals.html
deleted file mode 100644
index 3c6208b03..000000000
--- a/docs/_build/validmind/tests/model_validation/ModelPredictionResiduals.html
+++ /dev/null
@@ -1,294 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.ModelPredictionResiduals API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses normality and behavior of residuals in regression models through visualization and statistical tests.
-
-
Purpose
-
-
The Model Prediction Residuals test aims to visualize the residuals of model predictions and assess their normality
-using the Kolmogorov-Smirnov (KS) test. It helps to identify potential issues related to model assumptions and
-effectiveness.
-
-
Test Mechanism
-
-
The function calculates residuals and generates
-two figures: one for the time series of residuals and one for the histogram of residuals.
-It also calculates the KS test for normality and summarizes the results in a table.
-
-
Signs of High Risk
-
-
-
Residuals are not normally distributed, indicating potential issues with model assumptions.
-
High skewness or kurtosis in the residuals, which may suggest model misspecification.
-
-
-
Strengths
-
-
-
Provides clear visualizations of residuals over time and their distribution.
-
Includes statistical tests to assess the normality of residuals.
-
Helps in identifying potential model misspecifications and assumption violations.
-
-
-
Limitations
-
-
-
Assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access the pandas
-DataFrame.
-
Only generates plots for datasets with a datetime index, resulting in errors for other types of indices.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/RegardScore.html b/docs/_build/validmind/tests/model_validation/RegardScore.html
deleted file mode 100644
index 61a03b3a0..000000000
--- a/docs/_build/validmind/tests/model_validation/RegardScore.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.RegardScore API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the sentiment and potential biases in text generated by NLP models by computing and visualizing regard
-scores.
-
-
Purpose
-
-
The RegardScore test aims to evaluate the levels of regard (positive, negative, neutral, or other) in texts
-generated by NLP models. It helps in understanding the sentiment and bias present in the generated content.
-
-
Test Mechanism
-
-
This test extracts the true and predicted values from the provided dataset and model. It then computes the regard
-scores for each text instance using a preloaded regard evaluation tool. The scores are compiled into dataframes,
-and visualizations such as histograms and bar charts are generated to display the distribution of regard scores.
-Additionally, descriptive statistics (mean, median, standard deviation, minimum, and maximum) are calculated for
-the regard scores, providing a comprehensive overview of the model's performance.
-
-
Signs of High Risk
-
-
-
Noticeable skewness in the histogram, especially when comparing the predicted regard scores with the target
-regard scores, can indicate biases or inconsistencies in the model.
-
Lack of neutral scores in the model's predictions, despite a balanced distribution in the target data, might
-signal an issue.
-
-
-
Strengths
-
-
-
Provides a clear evaluation of regard levels in generated texts, aiding in ensuring content appropriateness.
-
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of
-regard scores.
-
Descriptive statistics offer a concise summary of the model's performance in generating texts with balanced
-sentiments.
-
-
-
Limitations
-
-
-
The accuracy of the regard scores is contingent upon the underlying regard tool.
-
The scores provide a broad overview but do not specify which portions or tokens of the text are responsible for
-high regard.
-
Supplementary, in-depth analysis might be needed for granular insights.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/RegressionResidualsPlot.html b/docs/_build/validmind/tests/model_validation/RegressionResidualsPlot.html
deleted file mode 100644
index e27dc5164..000000000
--- a/docs/_build/validmind/tests/model_validation/RegressionResidualsPlot.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.RegressionResidualsPlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates regression model performance using residual distribution and actual vs. predicted plots.
-
-
Purpose
-
-
The RegressionResidualsPlot metric aims to evaluate the performance of regression models. By generating and
-analyzing two plots – a distribution of residuals and a scatter plot of actual versus predicted values – this tool
-helps to visually appraise how well the model predicts and the nature of errors it makes.
-
-
Test Mechanism
-
-
The process begins by extracting the true output values (y_true) and the model's predicted values (y_pred).
-Residuals are computed by subtracting predicted from true values. These residuals are then visualized using a
-histogram to display their distribution. Additionally, a scatter plot is derived to compare true values against
-predicted values, together with a "Perfect Fit" line, which represents an ideal match (predicted values equal
-actual values), facilitating the assessment of the model's predictive accuracy.
-
-
Signs of High Risk
-
-
-
Residuals showing a non-normal distribution, especially those with frequent extreme values.
-
Significant deviations of predicted values from actual values in the scatter plot.
-
Sparse density of data points near the "Perfect Fit" line in the scatter plot, indicating poor prediction
-accuracy.
-
Visible patterns or trends in the residuals plot, suggesting the model's failure to capture the underlying data
-structure adequately.
-
-
-
Strengths
-
-
-
Provides a direct, visually intuitive assessment of a regression model’s accuracy and handling of data.
-
Visual plots can highlight issues of underfitting or overfitting.
-
Can reveal systematic deviations or trends that purely numerical metrics might miss.
-
Applicable across various regression model types.
-
-
-
Limitations
-
-
-
Relies on visual interpretation, which can be subjective and less precise than numerical evaluations.
-
May be difficult to interpret in cases with multi-dimensional outputs due to the plots’ two-dimensional nature.
-
Overlapping data points in the residuals plot can complicate interpretation efforts.
-
Does not summarize model performance into a single quantifiable metric, which might be needed for comparative or
-summary analyses.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/RougeScore.html b/docs/_build/validmind/tests/model_validation/RougeScore.html
deleted file mode 100644
index 06208aa67..000000000
--- a/docs/_build/validmind/tests/model_validation/RougeScore.html
+++ /dev/null
@@ -1,308 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.RougeScore API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the quality of machine-generated text using ROUGE metrics and visualizes the results to provide
-comprehensive performance insights.
-
-
Purpose
-
-
The ROUGE Score test is designed to evaluate the quality of text generated by machine learning models using various
-ROUGE metrics. ROUGE, which stands for Recall-Oriented Understudy for Gisting Evaluation, measures the overlap of
-n-grams, word sequences, and word pairs between machine-generated text and reference texts. This evaluation is
-crucial for tasks like text summarization, machine translation, and text generation, where the goal is to produce
-text that accurately reflects the content and meaning of human-crafted references.
-
-
Test Mechanism
-
-
The test extracts the true and predicted values from the provided dataset and model. It initializes the ROUGE
-evaluator with the specified metric (e.g., ROUGE-1). For each pair of true and predicted texts, it calculates the
-ROUGE scores and compiles them into a dataframe. Histograms and bar charts are generated for each ROUGE metric
-(Precision, Recall, and F1 Score) to visualize their distribution. Additionally, a table of descriptive statistics
-(mean, median, standard deviation, minimum, and maximum) is compiled for each metric, providing a comprehensive
-summary of the model's performance.
-
-
Signs of High Risk
-
-
-
Consistently low scores across ROUGE metrics could indicate poor quality in the generated text, suggesting that
-the model fails to capture the essential content of the reference texts.
-
Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
-
Low recall scores may indicate that important information from the reference text is being omitted.
-
An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the
-model's ability to balance informativeness and conciseness.
-
-
-
Strengths
-
-
-
Provides a multifaceted evaluation of text quality through different ROUGE metrics, offering a detailed view of
-model performance.
-
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the
-scores.
-
Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
-
-
-
Limitations
-
-
-
ROUGE metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or
-grammatical quality of the text.
-
The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
-
While useful for comparison, ROUGE scores alone do not provide a complete assessment of a model's performance and
-should be supplemented with other metrics and qualitative analysis.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/TimeSeriesPredictionWithCI.html b/docs/_build/validmind/tests/model_validation/TimeSeriesPredictionWithCI.html
deleted file mode 100644
index f6c1fab39..000000000
--- a/docs/_build/validmind/tests/model_validation/TimeSeriesPredictionWithCI.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.TimeSeriesPredictionWithCI API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses predictive accuracy and uncertainty in time series models, highlighting breaches beyond confidence
-intervals.
-
-
Purpose
-
-
The purpose of the Time Series Prediction with Confidence Intervals (CI) test is to visualize the actual versus
-predicted values for time series data, including confidence intervals, and to compute and report the number of
-breaches beyond these intervals. This helps in evaluating the reliability and accuracy of the model's predictions.
-
-
Test Mechanism
-
-
The function performs the following steps:
-
-
-
Calculates the standard deviation of prediction errors.
-
Determines the confidence intervals using a specified confidence level, typically 95%.
-
Counts the number of actual values that fall outside the confidence intervals, referred to as breaches.
-
Generates a plot visualizing the actual values, predicted values, and confidence intervals.
-
Returns a DataFrame summarizing the breach information, including the total breaches, upper breaches, and lower
-breaches.
-
-
-
Signs of High Risk
-
-
-
A high number of breaches indicates that the model's predictions are not reliable within the specified confidence
-level.
-
Significant deviations between actual and predicted values may highlight model inadequacies or issues with data
-quality.
-
-
-
Strengths
-
-
-
Provides a visual representation of prediction accuracy and the uncertainty around predictions.
-
Includes a statistical measure of prediction reliability through confidence intervals.
-
Computes and reports breaches, offering a quantitative assessment of prediction performance.
-
-
-
Limitations
-
-
-
Assumes that the dataset is provided as a DataFrameDataset object with a datetime index.
-
Requires that dataset.y_pred(model) returns the predicted values for the model.
-
The calculation of confidence intervals assumes normally distributed errors, which may not hold for all datasets.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/TimeSeriesPredictionsPlot.html b/docs/_build/validmind/tests/model_validation/TimeSeriesPredictionsPlot.html
deleted file mode 100644
index d0d2a3c7a..000000000
--- a/docs/_build/validmind/tests/model_validation/TimeSeriesPredictionsPlot.html
+++ /dev/null
@@ -1,288 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.TimeSeriesPredictionsPlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the R-Squared values of regression models over specified time segments in time series data to assess
-segment-wise model performance.
-
-
Purpose
-
-
The TimeSeriesR2SquareBySegments test aims to evaluate the R-Squared values for several regression models across
-different segments of time series data. This helps in determining how well the models explain the variability in
-the data within each specific time segment.
-
-
Test Mechanism
-
-
-
Provides a visual representation of model performance across different time segments.
-
Allows for identification of segments where the model performs poorly.
-
Calculating the R-Squared values for each segment.
-
Generating a bar chart to visually represent the R-Squared values across different models and segments.
-
-
-
Signs of High Risk
-
-
-
Significantly low R-Squared values for certain time segments, indicating poor model performance in those periods.
-
Large variability in R-Squared values across different segments for the same model, suggesting inconsistent
-performance.
-
-
-
Strengths
-
-
-
Provides a visual representation of how well models perform over different time periods.
-
Helps identify time segments where models may need improvement or retraining.
-
Facilitates comparison between multiple models in a straightforward manner.
-
-
-
Limitations
-
-
-
Assumes datasets are provided as DataFrameDataset objects with the attributes y, y_pred, and
-feature_columns.
-
Requires that dataset.y_pred(model) returns predicted values for the model.
-
Assumes that both y_true and y_pred are pandas Series with datetime indices, which may not always be the case.
-
May not account for more nuanced temporal dependencies within the segments.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/TokenDisparity.html b/docs/_build/validmind/tests/model_validation/TokenDisparity.html
deleted file mode 100644
index a73da37e5..000000000
--- a/docs/_build/validmind/tests/model_validation/TokenDisparity.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.TokenDisparity API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the token disparity between reference and generated texts, visualizing the results through histograms and
-bar charts, alongside compiling a comprehensive table of descriptive statistics for token counts.
-
-
Purpose
-
-
The Token Disparity test aims to assess the difference in the number of tokens between reference texts and texts
-generated by the model. Understanding token disparity is essential for evaluating how well the generated content
-matches the expected length and richness of the reference texts.
-
-
Test Mechanism
-
-
The test extracts true and predicted values from the dataset and model. It computes the number of tokens in each
-reference and generated text. The results are visualized using histograms and bar charts to display the
-distribution of token counts. Additionally, a table of descriptive statistics, including the mean, median, standard
-deviation, minimum, and maximum token counts, is compiled to provide a detailed summary of token usage.
-
-
Signs of High Risk
-
-
-
Significant disparity in token counts between reference and generated texts could indicate issues with text
-generation quality, such as verbosity or lack of detail.
-
Consistently low token counts in generated texts compared to references might suggest that the model is producing
-incomplete or overly concise outputs.
-
-
-
Strengths
-
-
-
Provides a simple yet effective evaluation of text length and token usage.
-
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of
-token counts.
-
Descriptive statistics offer a concise summary of the model's performance in generating texts of appropriate
-length.
-
-
-
Limitations
-
-
-
Token counts alone do not provide a complete assessment of text quality and should be supplemented with other
-metrics and qualitative analysis.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/ToxicityScore.html b/docs/_build/validmind/tests/model_validation/ToxicityScore.html
deleted file mode 100644
index b759244c3..000000000
--- a/docs/_build/validmind/tests/model_validation/ToxicityScore.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.ToxicityScore API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the toxicity levels of texts generated by NLP models to identify and mitigate harmful or offensive content.
-
-
Purpose
-
-
The ToxicityScore metric is designed to evaluate the toxicity levels of texts generated by models. This is crucial
-for identifying and mitigating harmful or offensive content in machine-generated texts.
-
-
Test Mechanism
-
-
The function starts by extracting the input, true, and predicted values from the provided dataset and model. The
-toxicity score is computed for each text using a preloaded toxicity evaluation tool. The scores are compiled into
-dataframes, and histograms and bar charts are generated to visualize the distribution of toxicity scores.
-Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is
-compiled for the toxicity scores, providing a comprehensive summary of the model's performance.
-
-
Signs of High Risk
-
-
-
Drastic spikes in toxicity scores indicate potentially toxic content within the associated text segment.
-
Persistent high toxicity scores across multiple texts may suggest systemic issues in the model's text generation
-process.
-
-
-
Strengths
-
-
-
Provides a clear evaluation of toxicity levels in generated texts, helping to ensure content safety and
-appropriateness.
-
Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of
-toxicity scores.
-
Descriptive statistics offer a concise summary of the model's performance in generating non-toxic texts.
-
-
-
Limitations
-
-
-
The accuracy of the toxicity scores is contingent upon the underlying toxicity tool.
-
The scores provide a broad overview but do not specify which portions or tokens of the text are responsible for
-high toxicity.
-
Supplementary, in-depth analysis might be needed for granular insights.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn.html b/docs/_build/validmind/tests/model_validation/sklearn.html
deleted file mode 100644
index 963b5befc..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn.html
+++ /dev/null
@@ -1,274 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates clustering model performance by measuring mutual information between true and predicted labels, adjusting
-for chance.
-
-
Purpose
-
-
The purpose of this metric (Adjusted Mutual Information) is to evaluate the performance of a machine learning
-model, more specifically, a clustering model. It measures the mutual information between the true labels and the
-ones predicted by the model, adjusting for chance.
-
-
Test Mechanism
-
-
The Adjusted Mutual Information (AMI) uses sklearn's adjusted_mutual_info_score function. This function
-calculates the mutual information between the true labels and the ones predicted while correcting for the chance
-correlation expected due to random label assignments. This test requires the model, the training dataset, and the
-test dataset as inputs.
-
-
Signs of High Risk
-
-
-
Low Adjusted Mutual Information Score: This score ranges between 0 and 1. A low score (closer to 0) can indicate
-poor model performance as the predicted labels do not align well with the true labels.
-
In case of high-dimensional data, if the algorithm shows high scores, this could also be a potential risk as AMI
-may not perform reliably.
-
-
-
Strengths
-
-
-
The AMI metric takes into account the randomness of the predicted labels, which makes it more robust than the
-simple Mutual Information.
-
The scale of AMI is not dependent on the sizes of the clustering, allowing for comparability between different
-datasets or models.
-
Good for comparing the output of clustering algorithms where the number of clusters is not known a priori.
-
-
-
Limitations
-
-
-
Adjusted Mutual Information does not take into account the continuous nature of some data. As a result, it may
-not be the best choice for regression or other continuous types of tasks.
-
AMI has the drawback of being biased towards clusterings with a higher number of clusters.
-
In comparison to other metrics, AMI can be slower to compute.
-
The interpretability of the score can be complex as it depends on the understanding of information theory
-concepts.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/AdjustedRandIndex.html b/docs/_build/validmind/tests/model_validation/sklearn/AdjustedRandIndex.html
deleted file mode 100644
index fa1a12e3e..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/AdjustedRandIndex.html
+++ /dev/null
@@ -1,300 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.AdjustedRandIndex API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Measures the similarity between two data clusters using the Adjusted Rand Index (ARI) metric in clustering machine
-learning models.
-
-
Purpose
-
-
The Adjusted Rand Index (ARI) metric is intended to measure the similarity between two data clusters. This metric
-is specifically used for clustering machine learning models to quantify how well the model is clustering and
-producing data groups. It involves comparing the model's produced clusters against the actual (true) clusters found
-in the dataset.
-
-
Test Mechanism
-
-
The Adjusted Rand Index (ARI) is calculated using the adjusted_rand_score method from the sklearn.metrics
-module in Python. The test requires inputs including the model itself and the model's training and test datasets.
-The model's computed clusters and the true clusters are compared, and the similarities are measured to compute the
-ARI.
-
-
Signs of High Risk
-
-
-
If the ARI is close to zero, it signifies that the model's cluster assignments are random and do not match the
-actual dataset clusters, indicating a high risk.
-
An ARI of less than zero indicates that the model's clustering performance is worse than random.
-
-
-
Strengths
-
-
-
ARI is normalized and provides a consistent metric between -1 and +1, irrespective of raw cluster sizes or
-dataset size variations.
-
It does not require a ground truth for computation, making it ideal for unsupervised learning model evaluations.
-
It penalizes for false positives and false negatives, providing a robust measure of clustering quality.
-
-
-
Limitations
-
-
-
In real-world situations, true clustering is often unknown, which can hinder the practical application of the ARI.
-
The ARI requires all individual data instances to be independent, which may not always hold true.
-
It may be difficult to interpret the implications of an ARI score without context or a benchmark, as it is
-heavily dependent on the characteristics of the dataset used.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/CalibrationCurve.html b/docs/_build/validmind/tests/model_validation/sklearn/CalibrationCurve.html
deleted file mode 100644
index 704c6b575..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/CalibrationCurve.html
+++ /dev/null
@@ -1,318 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.CalibrationCurve API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the calibration of probability estimates by comparing predicted probabilities against observed
-frequencies.
-
-
Purpose
-
-
The Calibration Curve test assesses how well a model's predicted probabilities align with actual
-observed frequencies. This is crucial for applications requiring accurate probability estimates,
-such as risk assessment, decision-making systems, and cost-sensitive applications where probability
-calibration directly impacts business decisions.
-
-
Test Mechanism
-
-
The test uses sklearn's calibration_curve function to:
-
-
-
Sort predictions into bins based on predicted probabilities
-
Calculate the mean predicted probability in each bin
-
Compare against the observed frequency of positive cases
-
Plot the results against the perfect calibration line (y=x)
-The resulting curve shows how well the predicted probabilities match empirical probabilities.
-
-
-
Signs of High Risk
-
-
-
Significant deviation from the perfect calibration line
-
Systematic overconfidence (predictions too close to 0 or 1)
-
Systematic underconfidence (predictions clustered around 0.5)
-
Empty or sparse bins indicating poor probability coverage
-
Sharp discontinuities in the calibration curve
-
Different calibration patterns across different probability ranges
-
Consistent over/under estimation in critical probability regions
-
Large confidence intervals in certain probability ranges
-
-
-
Strengths
-
-
-
Visual and intuitive interpretation of probability quality
-
Identifies systematic biases in probability estimates
-
Supports probability threshold selection
-
Helps understand model confidence patterns
-
Applicable across different classification models
-
Enables comparison between different models
-
Guides potential need for recalibration
-
Critical for risk-sensitive applications
-
-
-
Limitations
-
-
-
Sensitive to the number of bins chosen
-
Requires sufficient samples in each bin for reliable estimates
-
May mask local calibration issues within bins
-
Does not account for feature-dependent calibration issues
-
Limited to binary classification problems
-
Cannot detect all forms of miscalibration
-
Assumes bin boundaries are appropriate for the problem
-
May be affected by class imbalance
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/ClassifierPerformance.html b/docs/_build/validmind/tests/model_validation/sklearn/ClassifierPerformance.html
deleted file mode 100644
index 946777087..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/ClassifierPerformance.html
+++ /dev/null
@@ -1,315 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.ClassifierPerformance API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy,
-and ROC AUC scores.
-
-
Purpose
-
-
The Classifier Performance test is designed to evaluate the performance of Machine Learning classification models.
-It accomplishes this by computing precision, recall, F1-Score, and accuracy, as well as the ROC AUC (Receiver
-operating characteristic - Area under the curve) scores, thereby providing a comprehensive analytic view of the
-models' performance. The test is adaptable, handling binary and multiclass models equally effectively.
-
-
Test Mechanism
-
-
The test produces a report that includes precision, recall, F1-Score, and accuracy, by leveraging the
-classification_report from scikit-learn's metrics module. For multiclass models, macro and weighted averages for
-these scores are also calculated. Additionally, the ROC AUC scores are calculated and included in the report using
-the multiclass_roc_auc_score function. The outcome of the test (report format) differs based on whether the model
-is binary or multiclass.
-
-
Signs of High Risk
-
-
-
Low values for precision, recall, F1-Score, accuracy, and ROC AUC, indicating poor performance.
-
Imbalance in precision and recall scores.
-
A low ROC AUC score, especially scores close to 0.5 or lower, suggesting a failing model.
-
-
-
Strengths
-
-
-
Versatile, capable of assessing both binary and multiclass models.
-
Utilizes a variety of commonly employed performance metrics, offering a comprehensive view of model performance.
-
The use of ROC-AUC as a metric is beneficial for evaluating unbalanced datasets.
-
-
-
Limitations
-
-
-
Assumes correctly identified labels for binary classification models.
-
Specifically designed for classification models and not suitable for regression models.
-
May provide limited insights if the test dataset does not represent real-world scenarios adequately.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.html b/docs/_build/validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.html
deleted file mode 100644
index e25d67605..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.html
+++ /dev/null
@@ -1,370 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.ClassifierThresholdOptimization API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Analyzes and visualizes different threshold optimization methods for binary classification models.
-
-
Purpose
-
-
The Classifier Threshold Optimization test identifies optimal decision thresholds using various
-methods to balance different performance metrics. This helps adapt the model's decision boundary
-to specific business requirements, such as minimizing false positives in fraud detection or
-achieving target recall in medical diagnosis.
-
-
Test Mechanism
-
-
The test implements multiple threshold optimization methods:
F1-score optimization (balancing precision and recall)
-
Precision-Recall equality point
-
Target recall achievement
-
Naive (0.5) threshold
-For each method, it computes ROC and PR curves, identifies optimal points, and provides
-comprehensive performance metrics at each threshold.
-
-
-
Signs of High Risk
-
-
-
Large discrepancies between different optimization methods
-
Optimal thresholds far from the default 0.5
-
Poor performance metrics across all thresholds
-
Significant gap between achieved and target recall
-
Unstable thresholds across different methods
-
Extreme trade-offs between precision and recall
-
Threshold optimization showing minimal impact
-
Business metrics not improving with optimization
-
-
-
Strengths
-
-
-
Multiple optimization strategies for different needs
-
Visual and numerical results for comparison
-
Support for business-driven optimization (target recall)
-
Comprehensive performance metrics at each threshold
-
Integration with ROC and PR curves
-
Handles class imbalance through various metrics
-
Enables informed threshold selection
-
Supports cost-sensitive decision making
-
-
-
Limitations
-
-
-
Assumes cost of false positives/negatives are known
-
May need adjustment for highly imbalanced datasets
-
Threshold might not be stable across different samples
-
Cannot handle multi-class problems directly
-
Optimization methods may conflict with business needs
-
Requires sufficient validation data
-
May not capture temporal changes in optimal threshold
-
Single threshold may not be optimal for all subgroups
-
-
-
Arguments:
-
-
-
dataset: VMDataset containing features and target
-
model: VMModel containing predictions
-
methods: List of methods to compare (default: ['youden', 'f1', 'precision_recall'])
-
target_recall: Target recall value if using 'target_recall' method
-
-
-
Returns:
-
-
-
Dictionary containing:
- - table: DataFrame comparing different threshold optimization methods
- (using weighted averages for precision, recall, and f1)
- - figure: Plotly figure showing ROC and PR curves with optimal thresholds
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.html b/docs/_build/validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.html
deleted file mode 100644
index 57f2230f6..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.html
+++ /dev/null
@@ -1,305 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.ClusterCosineSimilarity API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Measures the intra-cluster similarity of a clustering model using cosine similarity.
-
-
Purpose
-
-
The purpose of this metric is to measure how similar the data points within each cluster of a clustering model are.
-This is done using cosine similarity, which compares the multi-dimensional direction (but not magnitude) of data
-vectors. From a Model Risk Management perspective, this metric is used to quantitatively validate that clusters
-formed by a model have high intra-cluster similarity.
-
-
Test Mechanism
-
-
This test works by first extracting the true and predicted clusters of the model's training data. Then, it computes
-the centroid (average data point) of each cluster. Next, it calculates the cosine similarity between each data
-point within a cluster and its respective centroid. Finally, it outputs the mean cosine similarity of each cluster,
-highlighting how similar, on average, data points in a cluster are to the cluster's centroid.
-
-
Signs of High Risk
-
-
-
Low mean cosine similarity for one or more clusters: If the mean cosine similarity is low, the data points within
-the respective cluster have high variance in their directions. This can be indicative of poor clustering,
-suggesting that the model might not be suitably separating the data into distinct patterns.
-
High disparity between mean cosine similarity values across clusters: If there's a significant difference in mean
-cosine similarity across different clusters, this could indicate imbalance in how the model forms clusters.
-
-
-
Strengths
-
-
-
Cosine similarity operates in a multi-dimensional space, making it effective for measuring similarity in high
-dimensional datasets, typical for many machine learning problems.
-
It provides an agnostic view of the cluster performance by only considering the direction (and not the magnitude)
-of each vector.
-
This metric is not dependent on the scale of the variables, making it equally effective on different scales.
-
-
-
Limitations
-
-
-
Cosine similarity does not consider magnitudes (i.e. lengths) of vectors, only their direction. This means it may
-overlook instances where clusters have been adequately separated in terms of magnitude.
-
This method summarily assumes that centroids represent the average behavior of data points in each cluster. This
-might not always be true, especially in clusters with high amounts of variance or non-spherical shapes.
-
It primarily works with continuous variables and is not suitable for binary or categorical variables.
-
Lastly, although rare, perfect perpendicular vectors (cosine similarity = 0) could be within the same cluster,
-which may give an inaccurate representation of a 'bad' cluster due to low cosine similarity score.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.html b/docs/_build/validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.html
deleted file mode 100644
index 97189f541..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.html
+++ /dev/null
@@ -1,307 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.ClusterPerformanceMetrics API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the performance of clustering machine learning models using multiple established metrics.
-
-
Purpose
-
-
The ClusterPerformanceMetrics test is used to assess the performance and validity of clustering machine learning
-models. It evaluates homogeneity, completeness, V measure score, the Adjusted Rand Index, the Adjusted Mutual
-Information, and the Fowlkes-Mallows score of the model. These metrics provide a holistic understanding of the
-model's ability to accurately form clusters of the given dataset.
-
-
Test Mechanism
-
-
The ClusterPerformanceMetrics test runs a clustering ML model over a given dataset and then calculates six
-metrics using the Scikit-learn metrics computation functions: Homogeneity Score, Completeness Score, V Measure,
-Adjusted Rand Index (ARI), Adjusted Mutual Information (AMI), and Fowlkes-Mallows Score. It then returns the result
-as a summary, presenting the metric values for both training and testing datasets.
-
-
Signs of High Risk
-
-
-
Low Homogeneity Score: Indicates that the clusters formed contain a variety of classes, resulting in less pure
-clusters.
-
Low Completeness Score: Suggests that class instances are scattered across multiple clusters rather than being
-gathered in a single cluster.
-
Low V Measure: Reports a low overall clustering performance.
-
ARI close to 0 or Negative: Implies that clustering results are random or disagree with the true labels.
-
AMI close to 0: Means that clustering labels are random compared with the true labels.
-
Low Fowlkes-Mallows score: Signifies less precise and poor clustering performance in terms of precision and
-recall.
-
-
-
Strengths
-
-
-
Provides a comprehensive view of clustering model performance by examining multiple clustering metrics.
-
Uses established and widely accepted metrics from scikit-learn, providing reliability in the results.
-
Able to provide performance metrics for both training and testing datasets.
-
Clearly defined and human-readable descriptions of each score make it easy to understand what each score
-represents.
-
-
-
Limitations
-
-
-
Only applies to clustering models; not suitable for other types of machine learning models.
-
Does not test for overfitting or underfitting in the clustering model.
-
All the scores rely on ground truth labels, the absence or inaccuracy of which can lead to misleading results.
-
Does not consider aspects like computational efficiency of the model or its capability to handle high dimensional
-data.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/CompletenessScore.html b/docs/_build/validmind/tests/model_validation/sklearn/CompletenessScore.html
deleted file mode 100644
index c10ac6af2..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/CompletenessScore.html
+++ /dev/null
@@ -1,298 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.CompletenessScore API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates a clustering model's capacity to categorize instances from a single class into the same cluster.
-
-
Purpose
-
-
The Completeness Score metric is used to assess the performance of clustering models. It measures the extent to
-which all the data points that are members of a given class are elements of the same cluster. The aim is to
-determine the capability of the model to categorize all instances from a single class into the same cluster.
-
-
Test Mechanism
-
-
This test takes three inputs, a model and its associated training and testing datasets. It invokes the
-completeness_score function from the sklearn library on the labels predicted by the model. High scores indicate
-that data points from the same class generally appear in the same cluster, while low scores suggest the opposite.
-
-
Signs of High Risk
-
-
-
Low completeness score: This suggests that the model struggles to group instances from the same class into one
-cluster, indicating poor clustering performance.
-
-
-
Strengths
-
-
-
The Completeness Score provides an effective method for assessing the performance of a clustering model,
-specifically its ability to group class instances together.
-
This test metric conveniently relies on the capabilities provided by the sklearn library, ensuring consistent and
-reliable test results.
-
-
-
Limitations
-
-
-
This metric only evaluates a specific aspect of clustering, meaning it may not provide a holistic or complete
-view of the model's performance.
-
It cannot assess the effectiveness of the model in differentiating between separate classes, as it is solely
-focused on how well data points from the same class are grouped.
-
The Completeness Score only applies to clustering models; it cannot be used for other types of machine learning
-models.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/ConfusionMatrix.html b/docs/_build/validmind/tests/model_validation/sklearn/ConfusionMatrix.html
deleted file mode 100644
index 5e6de4881..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/ConfusionMatrix.html
+++ /dev/null
@@ -1,308 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.ConfusionMatrix API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix
-heatmap.
-
-
Purpose
-
-
The Confusion Matrix tester is designed to assess the performance of a classification Machine Learning model. This
-performance is evaluated based on how well the model is able to correctly classify True Positives, True Negatives,
-False Positives, and False Negatives - fundamental aspects of model accuracy.
-
-
Test Mechanism
-
-
The mechanism used involves taking the predicted results (y_test_predict) from the classification model and
-comparing them against the actual values (y_test_true). A confusion matrix is built using the unique labels
-extracted from y_test_true, employing scikit-learn's metrics. The matrix is then visually rendered with the help
-of Plotly's create_annotated_heatmap function. A heatmap is created which provides a two-dimensional graphical
-representation of the model's performance, showcasing distributions of True Positives (TP), True Negatives (TN),
-False Positives (FP), and False Negatives (FN).
-
-
Signs of High Risk
-
-
-
High numbers of False Positives (FP) and False Negatives (FN), depicting that the model is not effectively
-classifying the values.
-
Low numbers of True Positives (TP) and True Negatives (TN), implying that the model is struggling with correctly
-identifying class labels.
-
-
-
Strengths
-
-
-
It provides a simplified yet comprehensive visual snapshot of the classification model's predictive performance.
-
It distinctly brings out True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives
-(FN), thus making it easier to focus on potential areas of improvement.
-
The matrix is beneficial in dealing with multi-class classification problems as it can provide a simple view of
-complex model performances.
-
It aids in understanding the different types of errors that the model could potentially make, as it provides
-in-depth insights into Type-I and Type-II errors.
-
-
-
Limitations
-
-
-
In cases of unbalanced classes, the effectiveness of the confusion matrix might be lessened. It may wrongly
-interpret the accuracy of a model that is essentially just predicting the majority class.
-
It does not provide a single unified statistic that could evaluate the overall performance of the model.
-Different aspects of the model's performance are evaluated separately instead.
-
It mainly serves as a descriptive tool and does not offer the capability for statistical hypothesis testing.
-
Risks of misinterpretation exist because the matrix doesn't directly provide precision, recall, or F1-score data.
-These metrics have to be computed separately.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/FeatureImportance.html b/docs/_build/validmind/tests/model_validation/sklearn/FeatureImportance.html
deleted file mode 100644
index 74cb550fe..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/FeatureImportance.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.FeatureImportance API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Compute feature importance scores for a given model and generate a summary table
-with the top important features.
-
-
Purpose
-
-
The Feature Importance Comparison test is designed to compare the feature importance scores for different models
-when applied to various datasets. By doing so, it aims to identify the most impactful features and assess the
-consistency of feature importance across models.
-
-
Test Mechanism
-
-
This test works by iterating through each dataset-model pair and calculating permutation feature importance (PFI)
-scores. It then generates a summary table containing the top num_features important features for each model. The
-process involves:
-
-
-
Extracting features and target data from each dataset.
-
Computing PFI scores using sklearn.inspection.permutation_importance.
-
Sorting and selecting the top features based on their importance scores.
-
Compiling these features into a summary table for comparison.
-
-
-
Signs of High Risk
-
-
-
Key features expected to be important are ranked low, indicating potential issues with model training or data
-quality.
-
High variance in feature importance scores across different models, suggesting instability in feature selection.
-
-
-
Strengths
-
-
-
Provides a clear comparison of the most important features for each model.
-
Uses permutation importance, which is a model-agnostic method and can be applied to any estimator.
-
-
-
Limitations
-
-
-
Assumes that the dataset is provided as a DataFrameDataset object with x_df and y_df methods to access
-feature and target data.
-
Requires that model.model is compatible with sklearn.inspection.permutation_importance.
-
The function's output is dependent on the number of features specified by num_features, which defaults to 3 but
-can be adjusted.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/FowlkesMallowsScore.html b/docs/_build/validmind/tests/model_validation/sklearn/FowlkesMallowsScore.html
deleted file mode 100644
index 73422605e..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/FowlkesMallowsScore.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.FowlkesMallowsScore API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the similarity between predicted and actual cluster assignments in a model using the Fowlkes-Mallows
-score.
-
-
Purpose
-
-
The FowlkesMallowsScore is a performance metric used to validate clustering algorithms within machine learning
-models. The score intends to evaluate the matching grade between two clusters. It measures the similarity between
-the predicted and actual cluster assignments, thus gauging the accuracy of the model's clustering capability.
-
-
Test Mechanism
-
-
The FowlkesMallowsScore method applies the fowlkes_mallows_score function from the sklearn library to evaluate
-the model's accuracy in clustering different types of data. The test fetches the datasets from the model's training
-and testing datasets as inputs then compares the resulting clusters against the previously known clusters to obtain
-a score. A high score indicates a better clustering performance by the model.
-
-
Signs of High Risk
-
-
-
A low Fowlkes-Mallows score (near zero): This indicates that the model's clustering capability is poor and the
-algorithm isn't properly grouping data.
-
Inconsistently low scores across different datasets: This may indicate that the model's clustering performance is
-not robust and the model may fail when applied to unseen data.
-
-
-
Strengths
-
-
-
The Fowlkes-Mallows score is a simple and effective method for evaluating the performance of clustering
-algorithms.
-
This metric takes into account both precision and recall in its calculation, therefore providing a balanced and
-comprehensive measure of model performance.
-
The Fowlkes-Mallows score is non-biased meaning it treats False Positives and False Negatives equally.
-
-
-
Limitations
-
-
-
As a pairwise-based method, this score can be computationally intensive for large datasets and can become
-unfeasible as the size of the dataset increases.
-
The Fowlkes-Mallows score works best with balanced distribution of samples across clusters. If this condition is
-not met, the score can be skewed.
-
It does not handle mismatching numbers of clusters between the true and predicted labels. As such, it may return
-misleading results if the predicted labels suggest a different number of clusters than what is in the true labels.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/HomogeneityScore.html b/docs/_build/validmind/tests/model_validation/sklearn/HomogeneityScore.html
deleted file mode 100644
index 8e524e3ba..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/HomogeneityScore.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.HomogeneityScore API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses clustering homogeneity by comparing true and predicted labels, scoring from 0 (heterogeneous) to 1
-(homogeneous).
-
-
Purpose
-
-
The Homogeneity Score encapsulated in this performance test is used to measure the homogeneity of the clusters
-formed by a machine learning model. In simple terms, a clustering result satisfies homogeneity if all of its
-clusters contain only points which are members of a single class.
-
-
Test Mechanism
-
-
This test uses the homogeneity_score function from the sklearn.metrics library to compare the ground truth
-class labels of the training and testing sets with the labels predicted by the given model. The returned score is a
-metric of the clustering accuracy, and ranges from 0.0 to 1.0, with 1.0 denoting the highest possible degree of
-homogeneity.
-
-
Signs of High Risk
-
-
-
A score close to 0: This denotes that clusters are highly heterogenous and points within the same cluster might
-not belong to the same class.
-
A significantly lower score for testing data compared to the score for training data: This can indicate
-overfitting, where the model has learned to perfectly match the training data but fails to perform well on unseen
-data.
-
-
-
Strengths
-
-
-
It provides a simple quantitative measure of the degree to which clusters contain points from only one class.
-
Useful for validating clustering solutions where the ground truth — class membership of points — is known.
-
It's agnostic to the absolute labels, and cares only that the points within the same cluster have the same class
-label.
-
-
-
Limitations
-
-
-
The Homogeneity Score is not useful for clustering solutions where the ground truth labels are not known.
-
It doesn’t work well with differently sized clusters since it gives predominance to larger clusters.
-
The score does not address the actual number of clusters formed, or the evenness of cluster sizes. It only checks
-the homogeneity within the given clusters created by the model.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/HyperParametersTuning.html b/docs/_build/validmind/tests/model_validation/sklearn/HyperParametersTuning.html
deleted file mode 100644
index 8f3328be1..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/HyperParametersTuning.html
+++ /dev/null
@@ -1,331 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.HyperParametersTuning API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Performs exhaustive grid search over specified parameter ranges to find optimal model configurations
-across different metrics and decision thresholds.
-
-
Purpose
-
-
The Hyperparameter Tuning test systematically explores the model's parameter space to identify optimal
-configurations. It supports multiple optimization metrics and decision thresholds, providing a comprehensive
-view of how different parameter combinations affect various aspects of model performance.
-
-
Test Mechanism
-
-
The test uses scikit-learn's GridSearchCV to perform cross-validation for each parameter combination.
-For each specified threshold and optimization metric, it creates a scoring dictionary with
-threshold-adjusted metrics, performs grid search with cross-validation, records best parameters and
-corresponding scores, and combines results into a comparative table. This process is repeated for each
-optimization metric to provide a comprehensive view of model performance under different configurations.
-
-
Signs of High Risk
-
-
-
Large performance variations across different parameter combinations
-
Significant discrepancies between different optimization metrics
-
Best parameters at the edges of the parameter grid
-
Unstable performance across different thresholds
-
Overly complex model configurations (risk of overfitting)
-
Very different optimal parameters for different metrics
-
Cross-validation scores showing high variance
-
Extreme parameter values in best configurations
-
-
-
Strengths
-
-
-
Comprehensive exploration of parameter space
-
Supports multiple optimization metrics
-
Allows threshold optimization
-
Provides comparative view across different configurations
-
Uses cross-validation for robust evaluation
-
Helps understand trade-offs between different metrics
-
Enables systematic parameter selection
-
Supports both classification and clustering tasks
-
-
-
Limitations
-
-
-
Computationally expensive for large parameter grids
-
May not find global optimum (limited to grid points)
-
Cannot handle dependencies between parameters
-
Memory intensive for large datasets
-
Limited to scikit-learn compatible models
-
Cross-validation splits may not preserve time series structure
-
Grid search may miss optimal values between grid points
-
Resource intensive for high-dimensional parameter spaces
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/KMeansClustersOptimization.html b/docs/_build/validmind/tests/model_validation/sklearn/KMeansClustersOptimization.html
deleted file mode 100644
index 0229edf43..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/KMeansClustersOptimization.html
+++ /dev/null
@@ -1,308 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.KMeansClustersOptimization API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Optimizes the number of clusters in K-means models using Elbow and Silhouette methods.
-
-
Purpose
-
-
This metric is used to optimize the number of clusters used in K-means clustering models. It intends to measure and
-evaluate the optimal number of clusters by leveraging two methodologies, namely the Elbow method and the Silhouette
-method. This is crucial as an inappropriate number of clusters can either overly simplify or overcomplicate the
-structure of the data, thereby undermining the effectiveness of the model.
-
-
Test Mechanism
-
-
The test mechanism involves iterating over a predefined range of cluster numbers and applying both the Elbow method
-and the Silhouette method. The Elbow method computes the sum of the minimum euclidean distances between data points
-and their respective cluster centers (distortion). This value decreases as the number of clusters increases; the
-optimal number is typically at the 'elbow' point where the decrease in distortion becomes less pronounced.
-Meanwhile, the Silhouette method calculates the average silhouette score for each data point in the dataset,
-providing a measure of how similar each item is to its own cluster compared to other clusters. The optimal number
-of clusters under this method is the one that maximizes the average silhouette score. The results of both methods
-are plotted for visual inspection.
-
-
Signs of High Risk
-
-
-
A high distortion value or a low silhouette average score for the optimal number of clusters.
-
No clear 'elbow' point or plateau observed in the distortion plot, or a uniformly low silhouette average score
-across different numbers of clusters, suggesting the data is not amenable to clustering.
-
An optimal cluster number that is unreasonably high or low, suggestive of overfitting or underfitting,
-respectively.
-
-
-
Strengths
-
-
-
Provides both a visual and quantitative method to determine the optimal number of clusters.
-
Leverages two different methods (Elbow and Silhouette), thereby affording robustness and versatility in assessing
-the data's clusterability.
-
Facilitates improved model performance by allowing for an informed selection of the number of clusters.
-
-
-
Limitations
-
-
-
Assumes that a suitable number of clusters exists in the data, which may not always be true, especially for
-complex or noisy data.
-
Both methods may fail to provide definitive answers when the data lacks clear cluster structures.
-
Might not be straightforward to determine the 'elbow' point or maximize the silhouette average score, especially
-in larger and complicated datasets.
-
Assumes spherical clusters (due to using the Euclidean distance in the Elbow method), which might not align with
-the actual structure of the data.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/MinimumAccuracy.html b/docs/_build/validmind/tests/model_validation/sklearn/MinimumAccuracy.html
deleted file mode 100644
index da5a3d099..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/MinimumAccuracy.html
+++ /dev/null
@@ -1,298 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.MinimumAccuracy API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Checks if the model's prediction accuracy meets or surpasses a specified threshold.
-
-
Purpose
-
-
The Minimum Accuracy test’s objective is to verify whether the model's prediction accuracy on a specific dataset
-meets or surpasses a predetermined minimum threshold. Accuracy, which is simply the ratio of correct predictions to
-total predictions, is a key metric for evaluating the model's performance. Considering binary as well as multiclass
-classifications, accurate labeling becomes indispensable.
-
-
Test Mechanism
-
-
The test mechanism involves contrasting the model's accuracy score with a preset minimum threshold value, with the
-default being 0.7. The accuracy score is computed utilizing sklearn’s accuracy_score method, where the true
-labels y_true and predicted labels class_pred are compared. If the accuracy score is above the threshold, the
-test receives a passing mark. The test returns the result along with the accuracy score and threshold used for the
-test.
-
-
Signs of High Risk
-
-
-
Model fails to achieve or surpass the predefined score threshold.
-
Persistent scores below the threshold, indicating a high risk of inaccurate predictions.
-
-
-
Strengths
-
-
-
Simplicity, presenting a straightforward measure of holistic model performance across all classes.
-
Particularly advantageous when classes are balanced.
-
Versatile, as it can be implemented on both binary and multiclass classification tasks.
-
-
-
Limitations
-
-
-
Misleading accuracy scores when classes in the dataset are highly imbalanced.
-
Favoritism towards the majority class, giving an inaccurate perception of model performance.
-
Inability to measure the model's precision, recall, or capacity to manage false positives or false negatives.
-
Focused on overall correctness and may not be sufficient for all types of model analytics.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/MinimumF1Score.html b/docs/_build/validmind/tests/model_validation/sklearn/MinimumF1Score.html
deleted file mode 100644
index cd75c6eee..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/MinimumF1Score.html
+++ /dev/null
@@ -1,300 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.MinimumF1Score API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses if the model's F1 score on the validation set meets a predefined minimum threshold, ensuring balanced
-performance between precision and recall.
-
-
Purpose
-
-
The main objective of this test is to ensure that the F1 score, a balanced measure of precision and recall, of the
-model meets or surpasses a predefined threshold on the validation dataset. The F1 score is highly useful for
-gauging model performance in classification tasks, especially in cases where the distribution of positive and
-negative classes is skewed.
-
-
Test Mechanism
-
-
The F1 score for the validation dataset is computed through scikit-learn's metrics in Python. The scoring mechanism
-differs based on the classification problem: for multi-class problems, macro averaging is used, and for binary
-classification, the built-in f1_score calculation is used. The obtained F1 score is then assessed against the
-predefined minimum F1 score that is expected from the model.
-
-
Signs of High Risk
-
-
-
If a model returns an F1 score that is less than the established threshold, it is regarded as high risk.
-
A low F1 score might suggest that the model is not finding an optimal balance between precision and recall,
-failing to effectively identify positive classes while minimizing false positives.
-
-
-
Strengths
-
-
-
Provides a balanced measure of a model's performance by accounting for both false positives and false negatives.
-
Particularly advantageous in scenarios with imbalanced class distribution, where accuracy can be misleading.
-
Flexibility in setting the threshold value allows tailored minimum acceptable performance standards.
-
-
-
Limitations
-
-
-
May not be suitable for all types of models and machine learning tasks.
-
The F1 score assumes an equal cost for false positives and false negatives, which may not be true in some
-real-world scenarios.
-
Practitioners might need to rely on other metrics such as precision, recall, or the ROC-AUC score that align more
-closely with specific requirements.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/MinimumROCAUCScore.html b/docs/_build/validmind/tests/model_validation/sklearn/MinimumROCAUCScore.html
deleted file mode 100644
index 88266de33..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/MinimumROCAUCScore.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.MinimumROCAUCScore API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Validates model by checking if the ROC AUC score meets or surpasses a specified threshold.
-
-
Purpose
-
-
The Minimum ROC AUC Score test is used to determine the model's performance by ensuring that the Receiver Operating
-Characteristic Area Under the Curve (ROC AUC) score on the validation dataset meets or exceeds a predefined
-threshold. The ROC AUC score indicates how well the model can distinguish between different classes, making it a
-crucial measure in binary and multiclass classification tasks.
-
-
Test Mechanism
-
-
This test implementation calculates the multiclass ROC AUC score on the true target values and the model's
-predictions. The test converts the multi-class target variables into binary format using LabelBinarizer before
-computing the score. If this ROC AUC score is higher than the predefined threshold (defaulted to 0.5), the test
-passes; otherwise, it fails. The results, including the ROC AUC score, the threshold, and whether the test passed
-or failed, are then stored in a ThresholdTestResult object.
-
-
Signs of High Risk
-
-
-
A high risk or failure in the model's performance as related to this metric would be represented by a low ROC AUC
-score, specifically any score lower than the predefined minimum threshold. This suggests that the model is
-struggling to distinguish between different classes effectively.
-
-
-
Strengths
-
-
-
The test considers both the true positive rate and false positive rate, providing a comprehensive performance
-measure.
-
ROC AUC score is threshold-independent meaning it measures the model's quality across various classification
-thresholds.
-
Works robustly with binary as well as multi-class classification problems.
-
-
-
Limitations
-
-
-
ROC AUC may not be useful if the class distribution is highly imbalanced; it could perform well in terms of AUC
-but still fail to predict the minority class.
-
The test does not provide insight into what specific aspects of the model are causing poor performance if the ROC
-AUC score is unsatisfactory.
-
The use of macro average for multiclass ROC AUC score implies equal weightage to each class, which might not be
-appropriate if the classes are imbalanced.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/ModelParameters.html b/docs/_build/validmind/tests/model_validation/sklearn/ModelParameters.html
deleted file mode 100644
index a5150f9f6..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/ModelParameters.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.ModelParameters API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Extracts and displays model parameters in a structured format for transparency and reproducibility.
-
-
Purpose
-
-
The Model Parameters test is designed to provide transparency into model configuration and ensure
-reproducibility of machine learning models. It accomplishes this by extracting and presenting all
-relevant parameters that define the model's behavior, making it easier to audit, validate, and
-reproduce model training.
-
-
Test Mechanism
-
-
The test leverages scikit-learn's API convention of get_params() to extract model parameters. It
-produces a structured DataFrame containing parameter names and their corresponding values. For models
-that follow scikit-learn's API (including XGBoost, RandomForest, and other estimators), all
-parameters are automatically extracted and displayed.
-
-
Signs of High Risk
-
-
-
Missing crucial parameters that should be explicitly set
-
Extreme parameter values that could indicate overfitting (e.g., unlimited tree depth)
-
Inconsistent parameters across different versions of the same model type
-
Parameter combinations known to cause instability or poor performance
-
Default values used for critical parameters that should be tuned
-
-
-
Strengths
-
-
-
Universal compatibility with scikit-learn API-compliant models
-
Ensures transparency in model configuration
-
Facilitates model reproducibility and version control
-
Enables systematic parameter auditing
-
Supports both classification and regression models
-
Helps identify potential configuration issues
-
-
-
Limitations
-
-
-
Only works with models implementing scikit-learn's get_params() method
-
Cannot capture dynamic parameters set during model training
-
Does not validate parameter values for model-specific appropriateness
-
Parameter meanings and impacts may vary across different model types
-
Cannot detect indirect parameter interactions or their effects on model performance
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.html b/docs/_build/validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.html
deleted file mode 100644
index bd2de7ee0..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.ModelsPerformanceComparison API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates and compares the performance of multiple Machine Learning models using various metrics like accuracy,
-precision, recall, and F1 score.
-
-
Purpose
-
-
The Models Performance Comparison test aims to evaluate and compare the performance of various Machine Learning
-models using test data. It employs multiple metrics such as accuracy, precision, recall, and the F1 score, among
-others, to assess model performance and assist in selecting the most effective model for the designated task.
-
-
Test Mechanism
-
-
The test employs Scikit-learn’s performance metrics to evaluate each model's performance for both binary and
-multiclass classification tasks. To compare performances, the test runs each model against the test dataset, then
-produces a comprehensive classification report. This report includes metrics such as accuracy, precision, recall,
-and the F1 score. Based on whether the task at hand is binary or multiclass classification, it calculates metrics
-for all the classes and their weighted averages, macro averages, and per-class metrics. The test will be skipped if
-no models are supplied.
-
-
Signs of High Risk
-
-
-
Low scores in accuracy, precision, recall, and F1 metrics indicate a potentially high risk.
-
A low area under the Receiver Operating Characteristic (ROC) curve (roc_auc score) is another possible indicator
-of high risk.
-
If the metrics scores are significantly lower than alternative models, this might suggest a high risk of failure.
-
-
-
Strengths
-
-
-
Provides a simple way to compare the performance of multiple models, accommodating both binary and multiclass
-classification tasks.
-
Offers a holistic view of model performance through a comprehensive report of key performance metrics.
-
The inclusion of the ROC AUC score is advantageous, as this robust performance metric can effectively handle
-class imbalance issues.
-
-
-
Limitations
-
-
-
May not be suitable for more complex performance evaluations that consider factors such as prediction speed,
-computational cost, or business-specific constraints.
-
The test's reliability depends on the provided test dataset; hence, the selected models' performance could vary
-with unseen data or changes in the data distribution.
-
The ROC AUC score might not be as meaningful or easily interpretable for multilabel/multiclass tasks.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/OverfitDiagnosis.html b/docs/_build/validmind/tests/model_validation/sklearn/OverfitDiagnosis.html
deleted file mode 100644
index 932f017b6..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/OverfitDiagnosis.html
+++ /dev/null
@@ -1,305 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.OverfitDiagnosis API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses potential overfitting in a model's predictions, identifying regions where performance between training and
-testing sets deviates significantly.
-
-
Purpose
-
-
The Overfit Diagnosis test aims to identify areas in a model's predictions where there is a significant difference
-in performance between the training and testing sets. This test helps to pinpoint specific regions or feature
-segments where the model may be overfitting.
-
-
Test Mechanism
-
-
This test compares the model's performance on training versus test data, grouped by feature columns. It calculates
-the difference between the training and test performance for each group and identifies regions where this
-difference exceeds a specified threshold:
-
-
-
The test works for both classification and regression models.
-
It defaults to using the AUC metric for classification models and the MSE metric for regression models.
-
The threshold for identifying overfitting regions is set to 0.04 by default.
-
The test calculates the performance metrics for each feature segment and plots regions where the performance gap
-exceeds the threshold.
-
-
-
Signs of High Risk
-
-
-
Significant gaps between training and test performance metrics for specific feature segments.
-
Multiple regions with performance gaps exceeding the defined threshold.
-
Higher than expected differences in predicted versus actual values in the test set compared to the training set.
-
-
-
Strengths
-
-
-
Identifies specific areas where overfitting occurs.
Assesses the significance of each feature in a model by evaluating the impact on model performance when feature
-values are randomly rearranged.
-
-
Purpose
-
-
The Permutation Feature Importance (PFI) metric aims to assess the importance of each feature used by the Machine
-Learning model. The significance is measured by evaluating the decrease in the model's performance when the
-feature's values are randomly arranged.
-
-
Test Mechanism
-
-
PFI is calculated via the permutation_importance method from the sklearn.inspection module. This method
-shuffles the columns of the feature dataset and measures the impact on the model's performance. A significant
-decrease in performance after permutating a feature's values deems the feature as important. On the other hand, if
-performance remains the same, the feature is likely not important. The output of the PFI metric is a figure
-illustrating the importance of each feature.
-
-
Signs of High Risk
-
-
-
The model heavily relies on a feature with highly variable or easily permutable values, indicating instability.
-
A feature deemed unimportant by the model but expected to have a significant effect on the outcome based on
-domain knowledge is not influencing the model's predictions.
-
-
-
Strengths
-
-
-
Provides insights into the importance of different features and may reveal underlying data structure.
-
Can indicate overfitting if a particular feature or set of features overly impacts the model's predictions.
-
Model-agnostic and can be used with any classifier that provides a measure of prediction accuracy before and
-after feature permutation.
-
-
-
Limitations
-
-
-
Does not imply causality; it only presents the amount of information that a feature provides for the prediction
-task.
-
Does not account for interactions between features. If features are correlated, the permutation importance may
-allocate importance to one and not the other.
-
Cannot interact with certain libraries like statsmodels, pytorch, catboost, etc., thus limiting its applicability.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.html b/docs/_build/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.html
deleted file mode 100644
index 8434a78bc..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.html
+++ /dev/null
@@ -1,331 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.PopulationStabilityIndex API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across
-different datasets.
-
-
Purpose
-
-
The Population Stability Index (PSI) serves as a quantitative assessment for evaluating the stability of a machine
-learning model's output distributions when comparing two different datasets. Typically, these would be a
-development and a validation dataset or two datasets collected at different periods. The PSI provides a measurable
-indication of any significant shift in the model's performance over time or noticeable changes in the
-characteristics of the population the model is making predictions for.
-
-
Test Mechanism
-
-
The implementation of the PSI in this script involves calculating the PSI for each feature between the training and
-test datasets. Data from both datasets is sorted and placed into either a predetermined number of bins or
-quantiles. The boundaries for these bins are initially determined based on the distribution of the training data.
-The contents of each bin are calculated and their respective proportions determined. Subsequently, the PSI is
-derived for each bin through a logarithmic transformation of the ratio of the proportions of data for each feature
-in the training and test datasets. The PSI, along with the proportions of data in each bin for both datasets, are
-displayed in a summary table, a grouped bar chart, and a scatter plot.
-
-
Signs of High Risk
-
-
-
A high PSI value is a clear indicator of high risk. Such a value suggests a significant shift in the model
-predictions or severe changes in the characteristics of the underlying population.
-
This ultimately suggests that the model may not be performing as well as expected and that it may be less
-reliable for making future predictions.
-
-
-
Strengths
-
-
-
The PSI provides a quantitative measure of the stability of a model over time or across different samples, making
-it an invaluable tool for evaluating changes in a model's performance.
-
It allows for direct comparisons across different features based on the PSI value.
-
The calculation and interpretation of the PSI are straightforward, facilitating its use in model risk management.
-
The use of visual aids such as tables and charts further simplifies the comprehension and interpretation of the
-PSI.
-
-
-
Limitations
-
-
-
The PSI test does not account for the interdependence between features: features that are dependent on one
-another may show similar shifts in their distributions, which in turn may result in similar PSI values.
-
The PSI test does not inherently provide insights into why there are differences in distributions or why the PSI
-values may have changed.
-
The test may not handle features with significant outliers adequately.
-
Additionally, the PSI test is performed on model predictions, not on the underlying data distributions which can
-lead to misinterpretations. Any changes in PSI could be due to shifts in the model (model drift), changes in the
-relationships between features and the target variable (concept drift), or both. However, distinguishing between
-these causes is non-trivial.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.html b/docs/_build/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.html
deleted file mode 100644
index 98ddeb29a..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.html
+++ /dev/null
@@ -1,302 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.PrecisionRecallCurve API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve.
-
-
Purpose
-
-
The Precision Recall Curve metric is intended to evaluate the trade-off between precision and recall in
-classification models, particularly binary classification models. It assesses the model's capacity to produce
-accurate results (high precision), as well as its ability to capture a majority of all positive instances (high
-recall).
-
-
Test Mechanism
-
-
The test extracts ground truth labels and prediction probabilities from the model's test dataset. It applies the
-precision_recall_curve method from the sklearn metrics module to these extracted labels and predictions, which
-computes a precision-recall pair for each possible threshold. This calculation results in an array of precision and
-recall scores that can be plotted against each other to form the Precision-Recall Curve. This curve is then
-visually represented by using Plotly's scatter plot.
-
-
Signs of High Risk
-
-
-
A lower area under the Precision-Recall Curve signifies high risk.
-
This corresponds to a model yielding a high amount of false positives (low precision) and/or false negatives (low
-recall).
-
If the curve is closer to the bottom left of the plot, rather than being closer to the top right corner, it can
-be a sign of high risk.
-
-
-
Strengths
-
-
-
This metric aptly represents the balance between precision (minimizing false positives) and recall (minimizing
-false negatives), which is especially critical in scenarios where both values are significant.
-
Through the graphic representation, it enables an intuitive understanding of the model's performance across
-different threshold levels.
-
-
-
Limitations
-
-
-
This metric is only applicable to binary classification models - it raises errors for multiclass classification
-models or Foundation models.
-
It may not fully represent the overall accuracy of the model if the cost of false positives and false negatives
-are extremely different, or if the dataset is heavily imbalanced.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/ROCCurve.html b/docs/_build/validmind/tests/model_validation/sklearn/ROCCurve.html
deleted file mode 100644
index 09f781174..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/ROCCurve.html
+++ /dev/null
@@ -1,308 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.ROCCurve API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic
-(ROC) curve and calculating the Area Under Curve (AUC) score.
-
-
Purpose
-
-
The Receiver Operating Characteristic (ROC) curve is designed to evaluate the performance of binary classification
-models. This curve illustrates the balance between the True Positive Rate (TPR) and False Positive Rate (FPR)
-across various threshold levels. In combination with the Area Under the Curve (AUC), the ROC curve aims to measure
-the model's discrimination ability between the two defined classes in a binary classification problem (e.g.,
-default vs non-default). Ideally, a higher AUC score signifies superior model performance in accurately
-distinguishing between the positive and negative classes.
-
-
Test Mechanism
-
-
First, this script selects the target model and datasets that require binary classification. It then calculates the
-predicted probabilities for the test set, and uses this data, along with the true outcomes, to generate and plot
-the ROC curve. Additionally, it includes a line signifying randomness (AUC of 0.5). The AUC score for the model's
-ROC curve is also computed, presenting a numerical estimation of the model's performance. If any Infinite values
-are detected in the ROC threshold, these are effectively eliminated. The resulting ROC curve, AUC score, and
-thresholds are consequently saved for future reference.
-
-
Signs of High Risk
-
-
-
A high risk is potentially linked to the model's performance if the AUC score drops below or nears 0.5.
-
Another warning sign would be the ROC curve lying closer to the line of randomness, indicating no discriminative
-ability.
-
For the model to be deemed competent at its classification tasks, it is crucial that the AUC score is
-significantly above 0.5.
-
-
-
Strengths
-
-
-
The ROC Curve offers an inclusive visual depiction of a model's discriminative power throughout all conceivable
-classification thresholds, unlike other metrics that solely disclose model performance at one fixed threshold.
-
Despite the proportions of the dataset, the AUC Score, which represents the entire ROC curve as a single data
-point, continues to be consistent, proving to be the ideal choice for such situations.
-
-
-
Limitations
-
-
-
The primary limitation is that this test is exclusively structured for binary classification tasks, thus limiting
-its application towards other model types.
-
Furthermore, its performance might be subpar with models that output probabilities highly skewed towards 0 or 1.
-
At the extreme, the ROC curve could reflect high performance even when the majority of classifications are
-incorrect, provided that the model's ranking format is retained. This phenomenon is commonly termed the "Class
-Imbalance Problem".
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/RegressionErrors.html b/docs/_build/validmind/tests/model_validation/sklearn/RegressionErrors.html
deleted file mode 100644
index 986245ce6..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/RegressionErrors.html
+++ /dev/null
@@ -1,310 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.RegressionErrors API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the performance and error distribution of a regression model using various error metrics.
-
-
Purpose
-
-
The purpose of the Regression Errors test is to measure the performance of a regression model by calculating
-several error metrics. This evaluation helps determine the model's accuracy and potential issues like overfitting
-or bias by analyzing differences in error metrics between the training and testing datasets.
-
-
Test Mechanism
-
-
The test computes the following error metrics:
-
-
-
Mean Absolute Error (MAE): Average of the absolute differences between true values and predicted values.
-
Mean Squared Error (MSE): Average of the squared differences between true values and predicted values.
-
Root Mean Squared Error (RMSE): Square root of the mean squared error.
-
Mean Absolute Percentage Error (MAPE): Average of the absolute differences between true values and predicted
-values, divided by the true values, and expressed as a percentage.
-
Mean Bias Deviation (MBD): Average bias between true values and predicted values.
-
-
-
These metrics are calculated separately for the training and testing datasets and compared to identify
-discrepancies.
-
-
Signs of High Risk
-
-
-
High values for MAE, MSE, RMSE, or MAPE indicating poor model performance.
-
Large differences in error metrics between the training and testing datasets, suggesting overfitting.
-
Significant deviation of MBD from zero, indicating systematic bias in model predictions.
-
-
-
Strengths
-
-
-
Provides a comprehensive overview of model performance through multiple error metrics.
-
Individual metrics offer specific insights, e.g., MAE for interpretability, MSE for emphasizing larger errors.
-
RMSE is useful for being in the same unit as the target variable.
-
MAPE allows the error to be expressed as a percentage.
-
MBD detects systematic bias in model predictions.
-
-
-
Limitations
-
-
-
MAE and MSE are sensitive to outliers.
-
RMSE heavily penalizes larger errors, which might not always be desirable.
-
MAPE can be misleading when actual values are near zero.
-
MBD may not be suitable if bias varies with the magnitude of actual values.
-
These metrics may not capture all nuances of model performance and should be interpreted with domain-specific
-context.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/RegressionErrorsComparison.html b/docs/_build/validmind/tests/model_validation/sklearn/RegressionErrorsComparison.html
deleted file mode 100644
index 1939e4612..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/RegressionErrorsComparison.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.RegressionErrorsComparison API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses multiple regression error metrics to compare model performance across different datasets, emphasizing
-systematic overestimation or underestimation and large percentage errors.
-
-
Purpose
-
-
The purpose of this test is to compare regression errors for different models applied to various datasets. It aims
-to examine model performance using multiple error metrics, thereby identifying areas where models may be
-underperforming or exhibiting bias.
-
-
Test Mechanism
-
-
The function iterates through each dataset-model pair and calculates various error metrics, including Mean Absolute
-Error (MAE), Mean Squared Error (MSE), Mean Absolute Percentage Error (MAPE), and Mean Bias Deviation (MBD). The
-results are summarized in a table, which provides a comprehensive view of each model's performance on the datasets.
-
-
Signs of High Risk
-
-
-
High Mean Absolute Error (MAE) or Mean Squared Error (MSE), indicating poor model performance.
-
High Mean Absolute Percentage Error (MAPE), suggesting large percentage errors, especially problematic if the
-true values are small.
-
Mean Bias Deviation (MBD) significantly different from zero, indicating systematic overestimation or
-underestimation by the model.
-
-
-
Strengths
-
-
-
Provides multiple error metrics to assess model performance from different perspectives.
-
Includes a check to avoid division by zero when calculating MAPE.
-
-
-
Limitations
-
-
-
Assumes that the dataset is provided as a DataFrameDataset object with y, y_pred, and feature_columns
-attributes.
-
Relies on the logger from validmind.logging to warn about zero values in y_true, which should be correctly
-implemented and imported.
-
Requires that dataset.y_pred(model) returns the predicted values for the model.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/RegressionPerformance.html b/docs/_build/validmind/tests/model_validation/sklearn/RegressionPerformance.html
deleted file mode 100644
index 5c785364f..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/RegressionPerformance.html
+++ /dev/null
@@ -1,294 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.RegressionPerformance API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the performance of a regression model using five different metrics: MAE, MSE, RMSE, MAPE, and MBD.
-
-
Purpose
-
-
The Regression Models Performance Comparison metric is used to measure the performance of regression models. It
-calculates multiple evaluation metrics, including Mean Absolute Error (MAE), Mean Squared Error (MSE),
-Root Mean Squared Error (RMSE), Mean Absolute Percentage Error (MAPE), and Mean Bias Deviation (MBD), thereby
-enabling a comprehensive view of model performance.
-
-
Test Mechanism
-
-
The test uses the sklearn library to calculate the MAE, MSE, RMSE, MAPE, and MBD. These calculations encapsulate both
-the direction and the magnitude of error in predictions, thereby providing a multi-faceted view of model accuracy.
-
-
Signs of High Risk
-
-
-
High values of MAE, MSE, RMSE, and MAPE, which indicate a high error rate and imply a larger departure of the
-model's predictions from the true values.
-
A large value of MBD, which shows a consistent bias in the model’s predictions.
-
-
-
Strengths
-
-
-
The metric evaluates models on five different metrics offering a comprehensive analysis of model performance.
-
It is designed to handle regression tasks and can be seamlessly integrated with libraries like sklearn.
-
-
-
Limitations
-
-
-
The metric only evaluates regression models and does not evaluate classification models.
-
The test assumes that the models have been trained and tested appropriately prior to evaluation. It does not
-handle pre-processing, feature selection, or other stages in the model lifecycle.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/RegressionR2Square.html b/docs/_build/validmind/tests/model_validation/sklearn/RegressionR2Square.html
deleted file mode 100644
index 6ba718138..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/RegressionR2Square.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.RegressionR2Square API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the overall goodness-of-fit of a regression model by evaluating R-squared (R2) and Adjusted R-squared (Adj
-R2) scores to determine the model's explanatory power over the dependent variable.
-
-
Purpose
-
-
The purpose of the RegressionR2Square Metric test is to measure the overall goodness-of-fit of a regression model.
-Specifically, this Python-based test evaluates the R-squared (R2) and Adjusted R-squared (Adj R2) scores, which are
-statistical measures used to assess the strength of the relationship between the model's predictors and the
-response variable.
-
-
Test Mechanism
-
-
The test deploys the r2_score method from the Scikit-learn metrics module to measure the R2 score on both
-training and test sets. This score reflects the proportion of the variance in the dependent variable that is
-predictable from the independent variables. The test also calculates the Adjusted R2 score, which accounts for the
-number of predictors in the model to penalize model complexity and reduce overfitting. The Adjusted R2 score will
-be smaller if unnecessary predictors are included in the model.
-
-
Signs of High Risk
-
-
-
Low R2 or Adjusted R2 scores, suggesting that the model does not explain much variation in the dependent variable.
-
Significant discrepancy between R2 scores on the training set and test set, indicating overfitting and poor
-generalization to unseen data.
-
-
-
Strengths
-
-
-
Widely-used measure in regression analysis, providing a sound general indication of model performance.
-
Easy to interpret and understand, as it represents the proportion of the dependent variable's variance explained
-by the independent variables.
-
Adjusted R2 score helps control overfitting by penalizing unnecessary predictors.
-
-
-
Limitations
-
-
-
Sensitive to the inclusion of unnecessary predictors even though Adjusted R2 penalizes complexity.
-
Less reliable in cases of non-linear relationships or when the underlying assumptions of linear regression are
-violated.
-
Does not provide insight on whether the correct regression model was used or if key assumptions have been met.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.html b/docs/_build/validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.html
deleted file mode 100644
index 3e9025f39..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.html
+++ /dev/null
@@ -1,305 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.RegressionR2SquareComparison API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Compares R-Squared and Adjusted R-Squared values for different regression models across multiple datasets to assess
-model performance and relevance of features.
-
-
Purpose
-
-
The Regression R2 Square Comparison test aims to compare the R-Squared and Adjusted R-Squared values for different
-regression models across various datasets. It helps in assessing how well each model explains the variability in
-the dataset, and whether the models include irrelevant features.
-
-
Test Mechanism
-
-
This test operates by:
-
-
-
Iterating through each dataset-model pair.
-
Calculating the R-Squared values to measure how much of the variability in the dataset is explained by the model.
-
Calculating the Adjusted R-Squared values, which adjust the R-Squared based on the number of predictors in the
-model, making it more reliable when comparing models with different numbers of features.
-
Generating a summary table containing these values for each combination of dataset and model.
-
-
-
Signs of High Risk
-
-
-
If the R-Squared values are significantly low, it indicates the model isn't explaining much of the variability in
-the dataset.
-
A significant difference between R-Squared and Adjusted R-Squared values might indicate that the model includes
-irrelevant features.
-
-
-
Strengths
-
-
-
Provides a quantitative measure of model performance in terms of variance explained.
-
Adjusted R-Squared accounts for the number of predictors, making it a more reliable measure when comparing models
-with different numbers of features.
-
Useful for time-series forecasting and regression tasks.
-
-
-
Limitations
-
-
-
Assumes the dataset is provided as a DataFrameDataset object with y, y_pred, and feature_columns attributes.
-
Relies on adj_r2_score from the statsmodels.statsutils module, which needs to be correctly implemented and
-imported.
-
Requires that dataset.y_pred(model) returns the predicted values for the model.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.html b/docs/_build/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.html
deleted file mode 100644
index 7f01a1d46..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.RobustnessDiagnosis API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the robustness of a machine learning model by evaluating performance decay under noisy conditions.
-
-
Purpose
-
-
The Robustness Diagnosis test aims to evaluate the resilience of a machine learning model when subjected to
-perturbations or noise in its input data. This is essential for understanding the model's ability to handle
-real-world scenarios where data may be imperfect or corrupted.
-
-
Test Mechanism
-
-
This test introduces Gaussian noise to the numeric input features of the datasets at varying scales of standard
-deviation. The performance of the model is then measured using a specified metric. The process includes:
-
-
-
Adding Gaussian noise to numerical input features based on scaling factors.
-
Evaluating the model's performance on the perturbed data using metrics like AUC for classification tasks and MSE
-for regression tasks.
-
Aggregating and plotting the results to visualize performance decay relative to perturbation size.
-
-
-
Signs of High Risk
-
-
-
A significant drop in performance metrics with minimal noise.
-
Performance decay values exceeding the specified threshold.
-
Consistent failure to meet performance standards across multiple perturbation scales.
-
-
-
Strengths
-
-
-
Provides insights into the model's robustness against noisy or corrupted data.
-
Utilizes a variety of performance metrics suitable for both classification and regression tasks.
-
Visualization helps in understanding the extent of performance degradation.
-
-
-
Limitations
-
-
-
Gaussian noise might not adequately represent all types of real-world data perturbations.
-
Performance thresholds are somewhat arbitrary and might need tuning.
-
The test may not account for more complex or unstructured noise patterns that could affect model robustness.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/SHAPGlobalImportance.html b/docs/_build/validmind/tests/model_validation/sklearn/SHAPGlobalImportance.html
deleted file mode 100644
index 4fdd8bc2b..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/SHAPGlobalImportance.html
+++ /dev/null
@@ -1,384 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.SHAPGlobalImportance API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Selects SHAP values for binary or multiclass classification.
-
-
For regression models, returns the SHAP values directly as there are no classes.
-
-
Arguments:
-
-
-
shap_values: The SHAP values returned by the SHAP explainer. For multiclass
-classification, this will be a list where each element corresponds to a class.
-For regression, this will be a single array of SHAP values.
-
class_of_interest: The class index for which to retrieve SHAP values. If None
-(default), the function will assume binary classification and use class 1
-by default.
-
-
-
Returns:
-
-
-
The SHAP values for the specified class (classification) or for the regression
- output.
-
-
-
Raises:
-
-
-
ValueError: If class_of_interest is specified and is out of bounds for the
-number of classes.
Evaluates and visualizes global feature importance using SHAP values for model explanation and risk identification.
-
-
Purpose
-
-
The SHAP (SHapley Additive exPlanations) Global Importance metric aims to elucidate model outcomes by attributing
-them to the contributing features. It assigns a quantifiable global importance to each feature via their respective
-absolute Shapley values, thereby making it suitable for tasks like classification (both binary and multiclass).
-This metric forms an essential part of model risk management.
-
-
Test Mechanism
-
-
The exam begins with the selection of a suitable explainer which aligns with the model's type. For tree-based
-models like XGBClassifier, RandomForestClassifier, CatBoostClassifier, TreeExplainer is used whereas for linear
-models like LogisticRegression, XGBRegressor, LinearRegression, it is the LinearExplainer. Once the explainer
-calculates the Shapley values, these values are visualized using two specific graphical representations:
-
-
-
Mean Importance Plot: This graph portrays the significance of individual features based on their absolute
-Shapley values. It calculates the average of these absolute Shapley values across all instances to highlight the
-global importance of features.
-
Summary Plot: This visual tool combines the feature importance with their effects. Every dot on this chart
-represents a Shapley value for a certain feature in a specific case. The vertical axis is denoted by the feature
-whereas the horizontal one corresponds to the Shapley value. A color gradient indicates the value of the feature,
-gradually changing from low to high. Features are systematically organized in accordance with their importance.
-
-
-
Signs of High Risk
-
-
-
Overemphasis on certain features in SHAP importance plots, thus hinting at the possibility of model overfitting
-
Anomalies such as unexpected or illogical features showing high importance, which might suggest that the model's
-decisions are rooted in incorrect or undesirable reasoning
-
A SHAP summary plot filled with high variability or scattered data points, indicating a cause for concern
-
-
-
Strengths
-
-
-
SHAP does more than just illustrating global feature significance, it offers a detailed perspective on how
-different features shape the model's decision-making logic for each instance.
-
It provides clear insights into model behavior.
-
-
-
Limitations
-
-
-
High-dimensional data can convolute interpretations.
-
Associating importance with tangible real-world impact still involves a certain degree of subjectivity.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.html b/docs/_build/validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.html
deleted file mode 100644
index 30aab721c..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.html
+++ /dev/null
@@ -1,315 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.ScoreProbabilityAlignment API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Analyzes the alignment between credit scores and predicted probabilities.
-
-
Purpose
-
-
The Score-Probability Alignment test evaluates how well credit scores align with
-predicted default probabilities. This helps validate score scaling, identify potential
-calibration issues, and ensure scores reflect risk appropriately.
-
-
Test Mechanism
-
-
The test:
-
-
-
Groups scores into bins
-
Calculates average predicted probability per bin
-
Tests monotonicity of relationship
-
Analyzes probability distribution within score bands
-
-
-
Signs of High Risk
-
-
-
Non-monotonic relationship between scores and probabilities
-
Large probability variations within score bands
-
Unexpected probability jumps between adjacent bands
-
Poor alignment with expected odds-to-score relationship
-
Inconsistent probability patterns across score ranges
-
Clustering of probabilities at extreme values
-
Score bands with similar probability profiles
-
Unstable probability estimates in key decision bands
-
-
-
Strengths
-
-
-
Direct validation of score-to-probability relationship
-
Identifies potential calibration issues
-
Supports score band validation
-
Helps understand model behavior
-
Useful for policy setting
-
Visual and numerical results
-
Easy to interpret
-
Supports regulatory documentation
-
-
-
Limitations
-
-
-
Sensitive to bin selection
-
Requires sufficient data per bin
-
May mask within-bin variations
-
Point-in-time analysis only
-
Cannot detect all forms of miscalibration
-
Assumes scores should align with probabilities
-
May oversimplify complex relationships
-
Limited to binary outcomes
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/SilhouettePlot.html b/docs/_build/validmind/tests/model_validation/sklearn/SilhouettePlot.html
deleted file mode 100644
index 03225aa85..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/SilhouettePlot.html
+++ /dev/null
@@ -1,308 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.SilhouettePlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Calculates and visualizes Silhouette Score, assessing the degree of data point suitability to its cluster in ML
-models.
-
-
Purpose
-
-
This test calculates the Silhouette Score, which is a model performance metric used in clustering applications.
-Primarily, the Silhouette Score evaluates how similar a data point is to its own cluster compared to other
-clusters. The metric ranges between -1 and 1, where a high value indicates that the object is well matched to its
-own cluster and poorly matched to neighboring clusters. Thus, the goal is to achieve a high Silhouette Score,
-implying well-separated clusters.
-
-
Test Mechanism
-
-
The test first extracts the true and predicted labels from the model's training data. The test runs the Silhouette
-Score function, which takes as input the training dataset features and the predicted labels, subsequently
-calculating the average score. This average Silhouette Score is printed for reference. The script then calculates
-the silhouette coefficients for each data point, helping to form the Silhouette Plot. Each cluster is represented
-in this plot, with color distinguishing between different clusters. A red dashed line indicates the average
-Silhouette Score. The Silhouette Scores are also collected into a structured table, facilitating model performance
-analysis and comparison.
-
-
Signs of High Risk
-
-
-
A low Silhouette Score, potentially indicating that the clusters are not well separated and that data points may
-not be fitting well to their respective clusters.
-
A Silhouette Plot displaying overlapping clusters or the absence of clear distinctions between clusters visually
-also suggests poor clustering performance.
-
-
-
Strengths
-
-
-
The Silhouette Score provides a clear and quantitative measure of how well data points have been grouped into
-clusters, offering insights into model performance.
-
The Silhouette Plot provides an intuitive, graphical representation of the clustering mechanism, aiding visual
-assessments of model performance.
-
It does not require ground truth labels, so it's useful when true cluster assignments are not known.
-
-
-
Limitations
-
-
-
The Silhouette Score may be susceptible to the influence of outliers, which could impact its accuracy and
-reliability.
-
It assumes the clusters are convex and isotropic, which might not be the case with complex datasets.
-
Due to the average nature of the Silhouette Score, the metric does not account for individual data point
-assignment nuances, so potentially relevant details may be omitted.
-
Computationally expensive for large datasets, as it requires pairwise distance computations.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/TrainingTestDegradation.html b/docs/_build/validmind/tests/model_validation/sklearn/TrainingTestDegradation.html
deleted file mode 100644
index db5cbc8ea..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/TrainingTestDegradation.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.TrainingTestDegradation API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Tests if model performance degradation between training and test datasets exceeds a predefined threshold.
-
-
Purpose
-
-
The TrainingTestDegradation class serves as a test to verify that the degradation in performance between the
-training and test datasets does not exceed a predefined threshold. This test measures the model's ability to
-generalize from its training data to unseen test data, assessing key classification metrics such as accuracy,
-precision, recall, and f1 score to verify the model's robustness and reliability.
-
-
Test Mechanism
-
-
The code applies several predefined metrics, including accuracy, precision, recall, and f1 scores, to the model's
-predictions for both the training and test datasets. It calculates the degradation as the difference between the
-training score and test score divided by the training score. The test is considered successful if the degradation
-for each metric is less than the preset maximum threshold of 10%. The results are summarized in a table showing
-each metric's train score, test score, degradation percentage, and pass/fail status.
-
-
Signs of High Risk
-
-
-
A degradation percentage that exceeds the maximum allowed threshold of 10% for any of the evaluated metrics.
-
A high difference or gap between the metric scores on the training and the test datasets.
-
The 'Pass/Fail' column displaying 'Fail' for any of the evaluated metrics.
-
-
-
Strengths
-
-
-
Provides a quantitative measure of the model's ability to generalize to unseen data, which is key for predicting
-its practical real-world performance.
-
By evaluating multiple metrics, it takes into account different facets of model performance and enables a more
-holistic evaluation.
-
The use of a variable predefined threshold allows the flexibility to adjust the acceptability criteria for
-different scenarios.
-
-
-
Limitations
-
-
-
The test compares raw performance on training and test data but does not factor in the nature of the data. Areas
-with less representation in the training set might still perform poorly on unseen data.
-
It requires good coverage and balance in the test and training datasets to produce reliable results, which may
-not always be available.
-
The test is currently only designed for classification tasks.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/VMeasure.html b/docs/_build/validmind/tests/model_validation/sklearn/VMeasure.html
deleted file mode 100644
index 821432058..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/VMeasure.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.VMeasure API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates homogeneity and completeness of a clustering model using the V Measure Score.
-
-
Purpose
-
-
The purpose of this metric, V Measure Score (V Score), is to evaluate the performance of a clustering model. It
-measures the homogeneity and completeness of a set of cluster labels, where homogeneity refers to each cluster
-containing only members of a single class and completeness meaning all members of a given class are assigned to the
-same cluster.
-
-
Test Mechanism
-
-
ClusterVMeasure is a class that inherits from another class, ClusterPerformance. It uses the v_measure_score
-function from the sklearn module's metrics package. The required inputs to perform this metric are the model, train
-dataset, and test dataset. The test is appropriate for models tasked with clustering.
-
-
Signs of High Risk
-
-
-
Low V Measure Score: A low V Measure Score indicates that the clustering model has poor homogeneity or
-completeness, or both. This might signal that the model is failing to correctly cluster the data.
-
-
-
Strengths
-
-
-
The V Measure Score is a harmonic mean between homogeneity and completeness. This ensures that both attributes
-are taken into account when evaluating the model, providing an overall measure of its cluster validity.
-
The metric does not require knowledge of the ground truth classes when measuring homogeneity and completeness,
-making it applicable in instances where such information is unavailable.
-
-
-
Limitations
-
-
-
The V Measure Score can be influenced by the number of clusters, which means that it might not always reflect the
-quality of the clustering. Partitioning the data into many small clusters could lead to high homogeneity but low
-completeness, leading to a low V Measure Score even if the clustering might be useful.
-
It assumes equal importance of homogeneity and completeness. In some applications, one may be more important than
-the other. The V Measure Score does not provide flexibility in assigning different weights to homogeneity and
-completeness.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.html b/docs/_build/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.html
deleted file mode 100644
index 020cdc0d4..000000000
--- a/docs/_build/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.html
+++ /dev/null
@@ -1,311 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.sklearn.WeakspotsDiagnosis API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Identifies and visualizes weak spots in a machine learning model's performance across various sections of the
-feature space.
-
-
Purpose
-
-
The weak spots test is applied to evaluate the performance of a machine learning model within specific regions of
-its feature space. This test slices the feature space into various sections, evaluating the model's outputs within
-each section against specific performance metrics (e.g., accuracy, precision, recall, and F1 scores). The ultimate
-aim is to identify areas where the model's performance falls below the set thresholds, thereby exposing its
-possible weaknesses and limitations.
-
-
Test Mechanism
-
-
The test mechanism adopts an approach of dividing the feature space of the training dataset into numerous bins. The
-model's performance metrics (accuracy, precision, recall, F1 scores) are then computed for each bin on both the
-training and test datasets. A "weak spot" is identified if any of the performance metrics fall below a
-predetermined threshold for a particular bin on the test dataset. The test results are visually plotted as bar
-charts for each performance metric, indicating the bins which fail to meet the established threshold.
-
-
Signs of High Risk
-
-
-
Any performance metric of the model dropping below the set thresholds.
-
Significant disparity in performance between the training and test datasets within a bin could be an indication
-of overfitting.
-
Regions or slices with consistently low performance metrics. Such instances could mean that the model struggles
-to handle specific types of input data adequately, resulting in potentially inaccurate predictions.
-
-
-
Strengths
-
-
-
The test helps pinpoint precise regions of the feature space where the model's performance is below par, allowing
-for more targeted improvements to the model.
-
The graphical presentation of the performance metrics offers an intuitive way to understand the model's
-performance across different feature areas.
-
The test exhibits flexibility, letting users set different thresholds for various performance metrics according
-to the specific requirements of the application.
-
-
-
Limitations
-
-
-
The binning system utilized for the feature space in the test could over-simplify the model's behavior within
-each bin. The granularity of this slicing depends on the chosen 'bins' parameter and can sometimes be arbitrary.
-
The effectiveness of this test largely hinges on the selection of thresholds for the performance metrics, which
-may not hold universally applicable and could be subjected to the specifications of a particular model and
-application.
-
The test is unable to handle datasets with a text column, limiting its application to numerical or categorical
-data types only.
-
Despite its usefulness in highlighting problematic regions, the test does not offer direct suggestions for model
-improvement.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels.html b/docs/_build/validmind/tests/model_validation/statsmodels.html
deleted file mode 100644
index c647c7f35..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels.html
+++ /dev/null
@@ -1,254 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates ARIMA models for time-series forecasting, ranking them using Bayesian and Akaike Information Criteria.
-
-
Purpose
-
-
The AutoARIMA validation test is designed to evaluate and rank AutoRegressive Integrated Moving Average (ARIMA)
-models. These models are primarily used for forecasting time-series data. The validation test automatically fits
-multiple ARIMA models, with varying parameters, to every variable within the given dataset. The models are then
-ranked based on their Bayesian Information Criterion (BIC) and Akaike Information Criterion (AIC) values, which
-provide a basis for the efficient model selection process.
-
-
Test Mechanism
-
-
This metric proceeds by generating an array of feasible combinations of ARIMA model parameters which are within a
-prescribed limit. These limits include max_p, max_d, max_q; they represent the autoregressive, differencing,
-and moving average components respectively. Upon applying these sets of parameters, the validation test fits each
-ARIMA model to the time-series data provided. For each model, it subsequently proceeds to calculate and record both
-the BIC and AIC values, which serve as performance indicators for the model fit. Prior to this parameter fitting
-process, the Augmented Dickey-Fuller test for data stationarity is conducted on the data series. If a series is
-found to be non-stationary, a warning message is sent out, given that ARIMA models necessitate input series to be
-stationary.
-
-
Signs of High Risk
-
-
-
If the p-value of the Augmented Dickey-Fuller test for a variable exceeds 0.05, a warning is logged. This warning
-indicates that the series might not be stationary, leading to potentially inaccurate results.
-
Consistent failure in fitting ARIMA models (as made evident through logged errors) might disclose issues with
-either the data or model stability.
-
-
-
Strengths
-
-
-
The AutoARIMA validation test simplifies the often complex task of selecting the most suitable ARIMA model based
-on BIC and AIC criteria.
-
The mechanism incorporates a check for non-stationarity within the data, which is a critical prerequisite for
-ARIMA models.
-
The exhaustive search through all possible combinations of model parameters enhances the likelihood of
-identifying the best-fit model.
-
-
-
Limitations
-
-
-
This validation test can be computationally costly as it involves creating and fitting multiple ARIMA models for
-every variable.
-
Although the test checks for non-stationarity and logs warnings where present, it does not apply any
-transformations to the data to establish stationarity.
-
The selection of models leans solely on BIC and AIC criteria, which may not yield the best predictive model in
-all scenarios.
-
The test is only applicable to regression tasks involving time-series data, and may not work effectively for
-other types of machine learning tasks.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.html b/docs/_build/validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.html
deleted file mode 100644
index 6823d9a1c..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.html
+++ /dev/null
@@ -1,308 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.CumulativePredictionProbabilities API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Visualizes cumulative probabilities of positive and negative classes for both training and testing in classification models.
-
-
Purpose
-
-
This metric is utilized to evaluate the distribution of predicted probabilities for positive and negative classes
-in a classification model. It provides a visual assessment of the model's behavior by plotting the cumulative
-probabilities for positive and negative classes across both the training and test datasets.
-
-
Test Mechanism
-
-
The classification model is evaluated by first computing the predicted probabilities for each instance in both
-the training and test datasets, which are then added as a new column in these sets. The cumulative probabilities
-for positive and negative classes are subsequently calculated and sorted in ascending order. Cumulative
-distributions of these probabilities are created for both positive and negative classes across both training and
-test datasets. These cumulative probabilities are represented visually in a plot, containing two subplots - one for
-the training data and the other for the test data, with lines representing cumulative distributions of positive and
-negative classes.
-
-
Signs of High Risk
-
-
-
Imbalanced distribution of probabilities for either positive or negative classes.
-
Notable discrepancies or significant differences between the cumulative probability distributions for the
-training data versus the test data.
-
Marked discrepancies or large differences between the cumulative probability distributions for positive and
-negative classes.
-
-
-
Strengths
-
-
-
Provides a visual illustration of data, which enhances the ease of understanding and interpreting the model's
-behavior.
-
Allows for the comparison of model's behavior across training and testing datasets, providing insights about how
-well the model is generalized.
-
Differentiates between positive and negative classes and their respective distribution patterns, aiding in
-problem diagnosis.
-
-
-
Limitations
-
-
-
Exclusive to classification tasks and specifically to classification models.
-
Graphical results necessitate human interpretation and may not be directly applicable for automated risk
-detection.
-
The method does not give a solitary quantifiable measure of model risk, instead, it offers a visual
-representation and broad distributional information.
-
If the training and test datasets are not representative of the overall data distribution, the metric could
-provide misleading results.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/DurbinWatsonTest.html b/docs/_build/validmind/tests/model_validation/statsmodels/DurbinWatsonTest.html
deleted file mode 100644
index 3e9e21cc8..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/DurbinWatsonTest.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.DurbinWatsonTest API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses autocorrelation in time series data features using the Durbin-Watson statistic.
-
-
Purpose
-
-
The Durbin-Watson Test metric detects autocorrelation in time series data (where a set of data values influences
-their predecessors). Autocorrelation is a crucial factor for regression tasks as these often assume the
-independence of residuals. A model with significant autocorrelation may give unreliable predictions.
-
-
Test Mechanism
-
-
Utilizing the durbin_watson function in the statsmodels Python library, the Durbin-Watson (DW) Test metric
-generates a statistical value for each feature of the training dataset. The function is looped over all columns of
-the dataset, calculating and caching the DW value for each column for further analysis. A DW metric value nearing 2
-indicates no autocorrelation. Conversely, values approaching 0 suggest positive autocorrelation, and those leaning
-towards 4 imply negative autocorrelation.
-
-
Signs of High Risk
-
-
-
If a feature's DW value significantly deviates from 2, it could signal a high risk due to potential
-autocorrelation issues in the dataset.
-
A value closer to 0 could imply positive autocorrelation, while a value nearer to 4 could point to negative
-autocorrelation, both leading to potentially unreliable prediction models.
-
-
-
Strengths
-
-
-
The metric specializes in identifying autocorrelation in prediction model residuals.
-
Autocorrelation detection assists in diagnosing violation of various modeling technique assumptions, particularly
-in regression analysis and time-series data modeling.
-
-
-
Limitations
-
-
-
The Durbin-Watson Test mainly detects linear autocorrelation and could overlook other types of relationships.
-
The metric is highly sensitive to data points order. Shuffling the order could lead to notably different results.
-
The test only checks for first-order autocorrelation (between a variable and its immediate predecessor) and fails
-to detect higher-order autocorrelation.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/GINITable.html b/docs/_build/validmind/tests/model_validation/statsmodels/GINITable.html
deleted file mode 100644
index c07a6e7fe..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/GINITable.html
+++ /dev/null
@@ -1,311 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.GINITable API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates classification model performance using AUC, GINI, and KS metrics for training and test datasets.
-
-
Purpose
-
-
The 'GINITable' metric is designed to evaluate the performance of a classification model by emphasizing its
-discriminatory power. Specifically, it calculates and presents three important metrics - the Area under the ROC
-Curve (AUC), the GINI coefficient, and the Kolmogorov-Smirnov (KS) statistic - for both training and test datasets.
-
-
Test Mechanism
-
-
Using a dictionary for storing performance metrics for both the training and test datasets, the 'GINITable' metric
-calculates each of these metrics sequentially. The Area under the ROC Curve (AUC) is calculated via the
-roc_auc_score function from the Scikit-Learn library. The GINI coefficient, a measure of statistical dispersion,
-is then computed by doubling the AUC and subtracting 1. Finally, the Kolmogorov-Smirnov (KS) statistic is
-calculated via the roc_curve function from Scikit-Learn, with the False Positive Rate (FPR) subtracted from the
-True Positive Rate (TPR) and the maximum value taken from the resulting data. These metrics are then stored in a
-pandas DataFrame for convenient visualization.
-
-
Signs of High Risk
-
-
-
Low values for performance metrics may suggest a reduction in model performance, particularly a low AUC which
-indicates poor classification performance, or a low GINI coefficient, which could suggest a decreased ability to
-discriminate different classes.
-
A high KS value may be an indicator of potential overfitting, as this generally signifies a substantial
-divergence between positive and negative distributions.
-
Significant discrepancies between the performance on the training dataset and the test dataset may present
-another signal of high risk.
-
-
-
Strengths
-
-
-
Offers three key performance metrics (AUC, GINI, and KS) in one test, providing a more comprehensive evaluation
-of the model.
-
Provides a direct comparison between the model's performance on training and testing datasets, which aids in
-identifying potential underfitting or overfitting.
-
The applied metrics are class-distribution invariant, thereby remaining effective for evaluating model
-performance even when dealing with imbalanced datasets.
-
Presents the metrics in a user-friendly table format for easy comprehension and analysis.
-
-
-
Limitations
-
-
-
The GINI coefficient and KS statistic are both dependent on the AUC value. Therefore, any errors in the
-calculation of the latter will adversely impact the former metrics too.
-
Mainly suited for binary classification models and may require modifications for effective application in
-multi-class scenarios.
-
The metrics used are threshold-dependent and may exhibit high variability based on the chosen cut-off points.
-
The test does not incorporate a method to efficiently handle missing or inefficiently processed data, which could
-lead to inaccuracies in the metrics if the data is not appropriately preprocessed.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.html b/docs/_build/validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.html
deleted file mode 100644
index 045f86b57..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.html
+++ /dev/null
@@ -1,297 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.KolmogorovSmirnov API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses whether each feature in the dataset aligns with a normal distribution using the Kolmogorov-Smirnov test.
-
-
Purpose
-
-
The Kolmogorov-Smirnov (KS) test evaluates the distribution of features in a dataset to determine their alignment
-with a normal distribution. This is important because many statistical methods and machine learning models assume
-normality in the data distribution.
-
-
Test Mechanism
-
-
This test calculates the KS statistic and corresponding p-value for each feature in the dataset. It does so by
-comparing the cumulative distribution function of the feature with an ideal normal distribution. The KS statistic
-and p-value for each feature are then stored in a dictionary. The p-value threshold to reject the normal
-distribution hypothesis is not preset, providing flexibility for different applications.
-
-
Signs of High Risk
-
-
-
Elevated KS statistic for a feature combined with a low p-value, indicating a significant divergence from a
-normal distribution.
-
Features with notable deviations that could create problems if the model assumes normality in data distribution.
-
-
-
Strengths
-
-
-
The KS test is sensitive to differences in the location and shape of empirical cumulative distribution functions.
-
It is non-parametric and adaptable to various datasets, as it does not assume any specific data distribution.
-
Provides detailed insights into the distribution of individual features.
-
-
-
Limitations
-
-
-
The test's sensitivity to disparities in the tails of data distribution might cause false alarms about
-non-normality.
-
Less effective for multivariate distributions, as it is designed for univariate distributions.
-
Does not identify specific types of non-normality, such as skewness or kurtosis, which could impact model fitting.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/Lilliefors.html b/docs/_build/validmind/tests/model_validation/statsmodels/Lilliefors.html
deleted file mode 100644
index e243734d3..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/Lilliefors.html
+++ /dev/null
@@ -1,307 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.Lilliefors API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the normality of feature distributions in an ML model's training dataset using the Lilliefors test.
-
-
Purpose
-
-
The purpose of this metric is to utilize the Lilliefors test, named in honor of the Swedish statistician Hubert
-Lilliefors, in order to assess whether the features of the machine learning model's training dataset conform to a
-normal distribution. This is done because the assumption of normal distribution plays a vital role in numerous
-statistical procedures as well as numerous machine learning models. Should the features fail to follow a normal
-distribution, some model types may not operate at optimal efficiency. This can potentially lead to inaccurate
-predictions.
-
-
Test Mechanism
-
-
The application of this test happens across all feature columns within the training dataset. For each feature, the
-Lilliefors test returns a test statistic and p-value. The test statistic quantifies how far the feature's
-distribution is from an ideal normal distribution, whereas the p-value aids in determining the statistical
-relevance of this deviation. The final results are stored within a dictionary, the keys of which correspond to the
-name of the feature column, and the values being another dictionary which houses the test statistic and p-value.
-
-
Signs of High Risk
-
-
-
If the p-value corresponding to a specific feature sinks below a pre-established significance level, generally
-set at 0.05, then it can be deduced that the distribution of that feature significantly deviates from a normal
-distribution. This can present a high risk for models that assume normality, as these models may perform
-inaccurately or inefficiently in the presence of such a feature.
-
-
-
Strengths
-
-
-
One advantage of the Lilliefors test is its utility irrespective of whether the mean and variance of the normal
-distribution are known in advance. This makes it a more robust option in real-world situations where these values
-might not be known.
-
The test has the ability to screen every feature column, offering a holistic view of the dataset.
-
-
-
Limitations
-
-
-
Despite the practical applications of the Lilliefors test in validating normality, it does come with some
-limitations.
-
It is only capable of testing unidimensional data, thus rendering it ineffective for datasets with interactions
-between features or multi-dimensional phenomena.
-
The test might not be as sensitive as some other tests (like the Anderson-Darling test) in detecting deviations
-from a normal distribution.
-
Like any other statistical test, Lilliefors test may also produce false positives or negatives. Hence, banking
-solely on this test, without considering other characteristics of the data, may give rise to risks.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.html b/docs/_build/validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.html
deleted file mode 100644
index e4fa80008..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.html
+++ /dev/null
@@ -1,305 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.PredictionProbabilitiesHistogram API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the predictive probability distribution for binary classification to evaluate model performance and
-potential overfitting or bias.
-
-
Purpose
-
-
The Prediction Probabilities Histogram test is designed to generate histograms displaying the Probability of
-Default (PD) predictions for both positive and negative classes in training and testing datasets. This helps in
-evaluating the performance of a classification model.
-
-
Test Mechanism
-
-
The metric follows these steps to execute the test:
-
-
-
Extracts the target column from both the train and test datasets.
-
Uses the model's predict function to calculate probabilities.
-
Adds these probabilities as a new column to the training and testing dataframes.
-
Generates histograms for each class (0 or 1) within the training and testing datasets.
-
Sets different opacities for the histograms to enhance visualization.
-
Overlays the four histograms (two for training and two for testing) on two different subplot frames.
-
Returns a plotly graph object displaying the visualization.
-
-
-
Signs of High Risk
-
-
-
Significant discrepancies between the histograms of training and testing data.
-
Large disparities between the histograms for the positive and negative classes.
-
Potential overfitting or bias indicated by significant issues.
-
Unevenly distributed probabilities suggesting inaccurate model predictions.
-
-
-
Strengths
-
-
-
Offers a visual representation of the PD predictions made by the model, aiding in understanding its behavior.
-
Assesses both the training and testing datasets, adding depth to model validation.
-
Highlights disparities between classes, providing insights into class imbalance or data skewness.
-
Effectively visualizes risk spread, which is particularly beneficial for credit risk prediction.
-
-
-
Limitations
-
-
-
Specifically tailored for binary classification scenarios and not suited for multi-class classification tasks.
-
Provides a robust visual representation but lacks a quantifiable measure to assess model performance.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionCoeffs.html b/docs/_build/validmind/tests/model_validation/statsmodels/RegressionCoeffs.html
deleted file mode 100644
index d0599ba67..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionCoeffs.html
+++ /dev/null
@@ -1,306 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.RegressionCoeffs API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the significance and uncertainty of predictor variables in a regression model through visualization of
-coefficients and their 95% confidence intervals.
-
-
Purpose
-
-
The RegressionCoeffs metric visualizes the estimated regression coefficients alongside their 95% confidence intervals,
-providing insights into the impact and significance of predictor variables on the response variable. This visualization
-helps to understand the variability and uncertainty in the model's estimates, aiding in the evaluation of the
-significance of each predictor.
-
-
Test Mechanism
-
-
The function operates by extracting the estimated coefficients and their standard errors from the regression model.
-Using these, it calculates the confidence intervals at a 95% confidence level, which indicates the range within which
-the true coefficient value is expected to fall 95% of the time. The confidence intervals are computed using the
-Z-value associated with the 95% confidence level. The coefficients and their confidence intervals are then visualized
-in a bar plot. The x-axis represents the predictor variables, the y-axis represents the estimated coefficients, and
-the error bars depict the confidence intervals.
-
-
Signs of High Risk
-
-
-
The confidence interval for a coefficient contains the zero value, suggesting that the predictor may not significantly
-contribute to the model.
-
Multiple coefficients with confidence intervals that include zero, potentially indicating issues with model reliability.
-
Very wide confidence intervals, which may suggest high uncertainty in the coefficient estimates and potential model
-instability.
-
-
-
Strengths
-
-
-
Provides a clear visualization that allows for easy interpretation of the significance and impact of predictor
-variables.
-
Includes confidence intervals, which provide additional information about the uncertainty surrounding each coefficient
-estimate.
-
-
-
Limitations
-
-
-
The method assumes normality of residuals and independence of observations, assumptions that may not always hold true
-in practice.
-
It does not address issues related to multi-collinearity among predictor variables, which can affect the interpretation
-of coefficients.
-
This metric is limited to regression tasks using tabular data and is not applicable to other types of machine learning
-tasks or data structures.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.html b/docs/_build/validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.html
deleted file mode 100644
index bd33d8d40..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.html
+++ /dev/null
@@ -1,301 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.RegressionFeatureSignificance API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses and visualizes the statistical significance of features in a regression model.
-
-
Purpose
-
-
The Regression Feature Significance metric assesses the significance of each feature in a given set of regression
-model. It creates a visualization displaying p-values for every feature of the model, assisting model developers
-in understanding which features are most influential in their model.
-
-
Test Mechanism
-
-
The test mechanism involves extracting the model's coefficients and p-values for each feature, and then plotting these
-values. The x-axis on the plot contains the p-values while the y-axis denotes the coefficients of each feature. A
-vertical red line is drawn at the threshold for p-value significance, which is 0.05 by default. Any features with
-p-values to the left of this line are considered statistically significant at the chosen level.
-
-
Signs of High Risk
-
-
-
Any feature with a high p-value (greater than the threshold) is considered a potential high risk, as it suggests
-the feature is not statistically significant and may not be reliably contributing to the model's predictions.
-
A high number of such features may indicate problems with the model validation, variable selection, and overall
-reliability of the model predictions.
-
-
-
Strengths
-
-
-
Helps identify the features that significantly contribute to a model's prediction, providing insights into the
-feature importance.
-
Provides tangible, easy-to-understand visualizations to interpret the feature significance.
-
-
-
Limitations
-
-
-
This metric assumes model features are independent, which may not always be the case. Multicollinearity (high
-correlation amongst predictors) can cause high variance and unreliable statistical tests of significance.
-
The p-value strategy for feature selection doesn't take into account the magnitude of the effect, focusing solely
-on whether the feature is likely non-zero.
-
This test is specific to regression models and wouldn't be suitable for other types of ML models.
-
P-value thresholds are somewhat arbitrary and do not always indicate practical significance, only statistical
-significance.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.html b/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.html
deleted file mode 100644
index 45e6ff4e5..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.html
+++ /dev/null
@@ -1,303 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.RegressionModelForecastPlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Generates plots to visually compare the forecasted outcomes of a regression model against actual observed values over
-a specified date range.
-
-
Purpose
-
-
This metric is useful for time-series models or any model where the outcome changes over time, allowing direct
-comparison of predicted vs actual values. It can help identify overfitting or underfitting situations as well as
-general model performance.
-
-
Test Mechanism
-
-
This test generates a plot with the x-axis representing the date ranging from the specified "start_date" to the
-"end_date", while the y-axis shows the value of the outcome variable. Two lines are plotted: one representing the
-forecasted values and the other representing the observed values. The "start_date" and "end_date" can be parameters
-of this test; if these parameters are not provided, they are set to the minimum and maximum date available in the
-dataset.
-
-
Signs of High Risk
-
-
-
High risk or failure signs could be deduced visually from the plots if the forecasted line significantly deviates
-from the observed line, indicating the model's predicted values are not matching actual outcomes.
-
A model that struggles to handle the edge conditions like maximum and minimum data points could also be
-considered a sign of risk.
-
-
-
Strengths
-
-
-
Visualization: The plot provides an intuitive and clear illustration of how well the forecast matches the actual
-values, making it straightforward even for non-technical stakeholders to interpret.
-
Flexibility: It allows comparison for multiple models and for specified time periods.
-
Model Evaluation: It can be useful in identifying overfitting or underfitting situations, as these will manifest
-as discrepancies between the forecasted and observed values.
-
-
-
Limitations
-
-
-
Interpretation Bias: Interpretation of the plot is subjective and can lead to different conclusions by different
-evaluators.
-
Lack of Precision: Visual representation might not provide precise values of the deviation.
-
Inapplicability: Limited to cases where the order of data points (time-series) matters, it might not be of much
-use in problems that are not related to time series prediction.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.html b/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.html
deleted file mode 100644
index 776473a65..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.html
+++ /dev/null
@@ -1,316 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.RegressionModelForecastPlotLevels API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the alignment between forecasted and observed values in regression models through visual plots
-
-
Purpose
-
-
This test aims to visually assess the performance of a regression model by comparing its forecasted values against
-the actual observed values for both the raw and transformed (integrated) data. This helps determine the accuracy
-of the model and can help identify overfitting or underfitting. The integration is applied to highlight the trend
-rather than the absolute level.
-
-
Test Mechanism
-
-
This test generates two plots:
-
-
-
Raw data vs forecast
-
Transformed data vs forecast
-
-
-
The transformed data is created by performing a cumulative sum on the raw data.
-
-
Signs of High Risk
-
-
-
Significant deviation between forecasted and observed values.
-
Patterns suggesting overfitting or underfitting.
-
Large discrepancies in the plotted forecasts, indicating potential issues with model generalizability and
-precision.
-
-
-
Strengths
-
-
-
Provides an intuitive, visual way to assess multiple regression models, aiding in easier interpretation and
-evaluation of forecast accuracy.
-
-
-
Limitations
-
-
-
Relies heavily on visual interpretation, which may vary between individuals.
-
Does not provide a numerical metric to quantify forecast accuracy, relying solely on visual assessment.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.html b/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.html
deleted file mode 100644
index bb192be94..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.html
+++ /dev/null
@@ -1,320 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.RegressionModelSensitivityPlot API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the sensitivity of a regression model to changes in independent variables by applying shocks and
-visualizing the impact.
-
-
Purpose
-
-
The Regression Sensitivity Plot test is designed to perform sensitivity analysis on regression models. This test
-aims to measure the impact of slight changes (shocks) applied to individual variables on the system's outcome while
-keeping all other variables constant. By doing so, it analyzes the effects of each independent variable on the
-dependent variable within the regression model, helping identify significant risk factors that could substantially
-influence the model's output.
-
-
Test Mechanism
-
-
This test operates by initially applying shocks of varying magnitudes, defined by specific parameters, to each of
-the model's features, one at a time. With all other variables held constant, a new prediction is made for each
-dataset subjected to shocks. Any changes in the model's predictions are directly attributed to the shocks applied.
-If the transformation parameter is set to "integrate," initial predictions and target values undergo transformation
-via an integration function before being plotted. Finally, a plot demonstrating observed values against predicted
-values for each model is generated, showcasing a distinct line graph illustrating predictions for each shock.
-
-
Signs of High Risk
-
-
-
Drastic alterations in model predictions due to minor shocks to an individual variable, indicating high
-sensitivity and potential over-dependence on that variable.
-
Unusually high or unpredictable shifts in response to shocks, suggesting potential model instability or
-overfitting.
-
-
-
Strengths
-
-
-
Helps identify variables that strongly influence model outcomes, aiding in understanding feature importance.
-
Generates visual plots, making results easily interpretable even to non-technical stakeholders.
-
Useful in identifying overfitting and detecting unstable models that react excessively to minor variable changes.
-
-
-
Limitations
-
-
-
Operates on the assumption that all other variables remain unchanged during the application of a shock, which may
-not reflect real-world interdependencies.
-
Best compatible with linear models and may not effectively evaluate the sensitivity of non-linear models.
-
Provides a visual representation without a numerical risk measure, potentially introducing subjectivity in
-interpretation.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelSummary.html b/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelSummary.html
deleted file mode 100644
index 73808c21d..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionModelSummary.html
+++ /dev/null
@@ -1,294 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.RegressionModelSummary API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates regression model performance using metrics including R-Squared, Adjusted R-Squared, MSE, and RMSE.
-
-
Purpose
-
-
The Regression Model Summary test evaluates the performance of regression models by measuring their predictive
-ability regarding dependent variables given changes in the independent variables. It uses conventional regression
-metrics such as R-Squared, Adjusted R-Squared, Mean Squared Error (MSE), and Root Mean Squared Error (RMSE) to
-assess the model's accuracy and fit.
-
-
Test Mechanism
-
-
This test uses the sklearn library to calculate the R-Squared, Adjusted R-Squared, MSE, and RMSE. It outputs a
-table with the results of these metrics along with the feature columns used by the model.
-
-
Signs of High Risk
-
-
-
Low R-Squared and Adjusted R-Squared values.
-
High MSE and RMSE values.
-
-
-
Strengths
-
-
-
Offers an extensive evaluation of regression models by combining four key measures of model accuracy and fit.
-
Provides a comprehensive view of the model's performance.
-
Both the R-Squared and Adjusted R-Squared measures are readily interpretable.
-
-
-
Limitations
-
-
-
RMSE and MSE might be sensitive to outliers.
-
A high R-Squared or Adjusted R-Squared may not necessarily indicate a good model, especially in cases of
-overfitting.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.html b/docs/_build/validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.html
deleted file mode 100644
index ce0ad4987..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.html
+++ /dev/null
@@ -1,298 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.RegressionPermutationFeatureImportance API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the significance of each feature in a model by evaluating the impact on model performance when feature
-values are randomly rearranged.
-
-
Purpose
-
-
The primary purpose of this metric is to determine which features significantly impact the performance of a
-regression model developed using statsmodels. The metric measures how much the prediction accuracy deteriorates
-when each feature's values are permuted.
-
-
Test Mechanism
-
-
This metric shuffles the values of each feature one at a time in the dataset, computes the model's performance
-after each permutation, and compares it to the baseline performance. A significant decrease in performance
-indicates the importance of the feature.
-
-
Signs of High Risk
-
-
-
Significant reliance on a feature that, when permuted, leads to a substantial decrease in performance, suggesting
-overfitting or high model dependency on that feature.
-
Features identified as unimportant despite known impacts from domain knowledge, suggesting potential issues in
-model training or data preprocessing.
-
-
-
Strengths
-
-
-
Directly assesses the impact of each feature on model performance, providing clear insights into model
-dependencies.
-
Model-agnostic within the scope of statsmodels, applicable to any regression model that outputs predictions.
-
-
-
Limitations
-
-
-
The metric is specific to statsmodels and cannot be used with other types of models without adaptation.
-
It does not capture interactions between features, which can lead to underestimating the importance of correlated
-features.
-
Assumes independence of features when calculating importance, which might not always hold true.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/ScorecardHistogram.html b/docs/_build/validmind/tests/model_validation/statsmodels/ScorecardHistogram.html
deleted file mode 100644
index b0e036779..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/ScorecardHistogram.html
+++ /dev/null
@@ -1,307 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.ScorecardHistogram API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
The Scorecard Histogram test evaluates the distribution of credit scores between default and non-default instances,
-providing critical insights into the performance and generalizability of credit-risk models.
-
-
Purpose
-
-
The Scorecard Histogram test metric provides a visual interpretation of the credit scores generated by a machine
-learning model for credit-risk classification tasks. It aims to compare the alignment of the model's scoring
-decisions with the actual outcomes of credit loan applications. It helps in identifying potential discrepancies
-between the model's predictions and real-world risk levels.
-
-
Test Mechanism
-
-
This metric uses logistic regression to generate a histogram of credit scores for both default (negative class) and
-non-default (positive class) instances. Using both training and test datasets, the metric calculates the credit
-score of each instance with a scorecard method, considering the impact of different features on the likelihood of
-default. It includes the default point to odds (PDO) scaling factor and predefined target score and odds settings.
-Histograms for training and test sets are computed and plotted separately to offer insights into the model's
-generalizability to unseen data.
-
-
Signs of High Risk
-
-
-
Discrepancies between the distributions of training and testing data, indicating a model's poor generalization
-ability
-
Skewed distributions favoring specific scores or classes, representing potential bias
-
-
-
Strengths
-
-
-
Provides a visual interpretation of the model's credit scoring system, enhancing comprehension of model behavior
-
Enables a direct comparison between actual and predicted scores for both training and testing data
-
Its intuitive visualization helps understand the model's ability to differentiate between positive and negative
-classes
-
Can unveil patterns or anomalies not easily discerned through numerical metrics alone
-
-
-
Limitations
-
-
-
Despite its value for visual interpretation, it doesn't quantify the performance of the model and therefore may
-lack precision for thorough model evaluation
-
The quality of input data can strongly influence the metric, as bias or noise in the data will affect both the
-score calculation and resultant histogram
-
Its specificity to credit scoring models limits its applicability across a wider variety of machine learning
-tasks and models
-
The metric's effectiveness is somewhat tied to the subjective interpretation of the analyst, relying on their
-judgment of the characteristics and implications of the plot.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/model_validation/statsmodels/statsutils.html b/docs/_build/validmind/tests/model_validation/statsmodels/statsutils.html
deleted file mode 100644
index 352eecab0..000000000
--- a/docs/_build/validmind/tests/model_validation/statsmodels/statsutils.html
+++ /dev/null
@@ -1,257 +0,0 @@
-
-
-
-
-
-
- validmind.tests.model_validation.statsmodels.statsutils API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses potential bias in a Large Language Model by analyzing the distribution and order of exemplars in the
-prompt.
-
-
Purpose
-
-
The Bias Evaluation test calculates if and how the order and distribution of exemplars (examples) in a few-shot
-learning prompt affect the output of a Large Language Model (LLM). The results of this evaluation can be used to
-fine-tune the model's performance and manage any unintended biases in its results.
-
-
Test Mechanism
-
-
This test uses two checks:
-
-
-
Distribution of Exemplars: The number of positive vs. negative examples in a prompt is varied. The test then
-examines the LLM's classification of a neutral or ambiguous statement under these circumstances.
-
Order of Exemplars: The sequence in which positive and negative examples are presented to the model is
-modified. Their resultant effect on the LLM's response is studied.
-
-
-
For each test case, the LLM grades the input prompt on a scale of 1 to 10. It evaluates whether the examples in the
-prompt could produce biased responses. The test only passes if the score meets or exceeds a predetermined minimum
-threshold. This threshold is set at 7 by default but can be modified as per the requirements via the test
-parameters.
-
-
Signs of High Risk
-
-
-
A skewed result favoring either positive or negative responses may suggest potential bias in the model. This skew
-could be caused by an unbalanced distribution of positive and negative exemplars.
-
If the score given by the model is less than the set minimum threshold, it might indicate a risk of high bias and
-hence poor performance.
-
-
-
Strengths
-
-
-
This test provides a quantitative measure of potential bias, offering clear guidelines for developers about
-whether their Large Language Model (LLM) contains significant bias.
-
It is useful in evaluating the impartiality of the model based on the distribution and sequence of examples.
-
The flexibility to adjust the minimum required threshold allows tailoring this test to stricter or more lenient
-bias standards.
-
-
-
Limitations
-
-
-
The test may not pick up on more subtle forms of bias or biases that are not directly related to the distribution
-or order of exemplars.
-
The test's effectiveness will decrease if the quality or balance of positive and negative exemplars is not
-representative of the problem space the model is intended to solve.
-
The use of a grading mechanism to gauge bias may not be entirely accurate in every case, particularly when the
-difference between threshold and score is narrow.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/prompt_validation/Clarity.html b/docs/_build/validmind/tests/prompt_validation/Clarity.html
deleted file mode 100644
index a5c788ae8..000000000
--- a/docs/_build/validmind/tests/prompt_validation/Clarity.html
+++ /dev/null
@@ -1,298 +0,0 @@
-
-
-
-
-
-
- validmind.tests.prompt_validation.Clarity API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates and scores the clarity of prompts in a Large Language Model based on specified guidelines.
-
-
Purpose
-
-
The Clarity evaluation metric is used to assess how clear the prompts of a Large Language Model (LLM) are. This
-assessment is particularly important because clear prompts assist the LLM in more accurately interpreting and
-responding to instructions.
-
-
Test Mechanism
-
-
The evaluation uses an LLM to scrutinize the clarity of prompts, factoring in considerations such as the inclusion
-of relevant details, persona adoption, step-by-step instructions, usage of examples, and specification of desired
-output length. Each prompt is rated on a clarity scale of 1 to 10, and any prompt scoring at or above the preset
-threshold (default of 7) will be marked as clear. It is important to note that this threshold can be adjusted via
-test parameters, providing flexibility in the evaluation process.
-
-
Signs of High Risk
-
-
-
Prompts that consistently score below the clarity threshold
-
Repeated failure of prompts to adhere to guidelines for clarity, including detail inclusion, persona adoption,
-explicit step-by-step instructions, use of examples, and specification of output length
-
-
-
Strengths
-
-
-
Encourages the development of more effective prompts that aid the LLM in interpreting instructions accurately
-
Applies a quantifiable measure (a score from 1 to 10) to evaluate the clarity of prompts
-
Threshold for clarity is adjustable, allowing for flexible evaluation depending on the context
-
-
-
Limitations
-
-
-
Scoring system is subjective and relies on the AI’s interpretation of 'clarity'
-
The test assumes that all required factors (detail inclusion, persona adoption, step-by-step instructions, use of
-examples, and specification of output length) contribute equally to clarity, which might not always be the case
-
The evaluation may not be as effective if used on non-textual models
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/prompt_validation/Conciseness.html b/docs/_build/validmind/tests/prompt_validation/Conciseness.html
deleted file mode 100644
index a868b8bb1..000000000
--- a/docs/_build/validmind/tests/prompt_validation/Conciseness.html
+++ /dev/null
@@ -1,298 +0,0 @@
-
-
-
-
-
-
- validmind.tests.prompt_validation.Conciseness API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Analyzes and grades the conciseness of prompts provided to a Large Language Model.
-
-
Purpose
-
-
The Conciseness Assessment is designed to evaluate the brevity and succinctness of prompts provided to a Language
-Learning Model (LLM). A concise prompt strikes a balance between offering clear instructions and eliminating
-redundant or unnecessary information, ensuring that the LLM receives relevant input without being overwhelmed.
-
-
Test Mechanism
-
-
Using an LLM, this test conducts a conciseness analysis on input prompts. The analysis grades the prompt on a scale
-from 1 to 10, where the grade reflects how well the prompt delivers clear instructions without being verbose.
-Prompts that score equal to or above a predefined threshold (default set to 7) are deemed successfully concise.
-This threshold can be adjusted to meet specific requirements.
-
-
Signs of High Risk
-
-
-
Prompts that consistently score below the predefined threshold.
-
Prompts that are overly wordy or contain unnecessary information.
-
Prompts that create confusion or ambiguity due to excess or unnecessary information.
-
-
-
Strengths
-
-
-
Ensures clarity and effectiveness of the prompts.
-
Promotes brevity and preciseness in prompts without sacrificing essential information.
-
Useful for models like LLMs, where input prompt length and clarity greatly influence model performance.
-
Provides a quantifiable measure of prompt conciseness.
-
-
-
Limitations
-
-
-
The conciseness score is based on an AI's assessment, which might not fully capture human interpretation of
-conciseness.
-
The predefined threshold for conciseness could be subjective and might need adjustment based on application.
-
The test is dependent on the LLM’s understanding of conciseness, which might vary from model to model.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/prompt_validation/Delimitation.html b/docs/_build/validmind/tests/prompt_validation/Delimitation.html
deleted file mode 100644
index f7f16b1cf..000000000
--- a/docs/_build/validmind/tests/prompt_validation/Delimitation.html
+++ /dev/null
@@ -1,299 +0,0 @@
-
-
-
-
-
-
- validmind.tests.prompt_validation.Delimitation API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates the proper use of delimiters in prompts provided to Large Language Models.
-
-
Purpose
-
-
The Delimitation Test aims to assess whether prompts provided to the Language Learning Model (LLM) correctly use
-delimiters to mark different sections of the input. Well-delimited prompts help simplify the interpretation process
-for the LLM, ensuring that the responses are precise and accurate.
-
-
Test Mechanism
-
-
The test employs an LLM to examine prompts for appropriate use of delimiters such as triple quotation marks, XML
-tags, and section titles. Each prompt is assigned a score from 1 to 10 based on its delimitation integrity. Prompts
-with scores equal to or above the preset threshold (which is 7 by default, although it can be adjusted as
-necessary) pass the test.
-
-
Signs of High Risk
-
-
-
Prompts missing, improperly placed, or incorrectly used delimiters, leading to misinterpretation by the LLM.
-
High-risk scenarios with complex prompts involving multiple tasks or diverse data where correct delimitation is
-crucial.
-
Scores below the threshold, indicating a high risk.
-
-
-
Strengths
-
-
-
Ensures clarity in demarcating different components of given prompts.
-
Reduces ambiguity in understanding prompts, especially for complex tasks.
-
Provides a quantified insight into the appropriateness of delimiter usage, aiding continuous improvement.
-
-
-
Limitations
-
-
-
Only checks for the presence and placement of delimiters, not whether the correct delimiter type is used for the
-specific data or task.
-
May not fully reveal the impacts of poor delimitation on the LLM's final performance.
-
The preset score threshold may not be refined enough for complex tasks and prompts, requiring regular manual
-adjustment.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/prompt_validation/NegativeInstruction.html b/docs/_build/validmind/tests/prompt_validation/NegativeInstruction.html
deleted file mode 100644
index 0bb0e6985..000000000
--- a/docs/_build/validmind/tests/prompt_validation/NegativeInstruction.html
+++ /dev/null
@@ -1,304 +0,0 @@
-
-
-
-
-
-
- validmind.tests.prompt_validation.NegativeInstruction API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates and grades the use of affirmative, proactive language over negative instructions in LLM prompts.
-
-
Purpose
-
-
The Negative Instruction test is utilized to scrutinize the prompts given to a Large Language Model (LLM). The
-objective is to ensure these prompts are expressed using proactive, affirmative language. The focus is on
-instructions indicating what needs to be done rather than what needs to be avoided, thereby guiding the LLM more
-efficiently towards the desired output.
-
-
Test Mechanism
-
-
An LLM is employed to evaluate each prompt. The prompt is graded based on its use of positive instructions with
-scores ranging between 1-10. This grade reflects how effectively the prompt leverages affirmative language while
-shying away from negative or restrictive instructions. A prompt that attains a grade equal to or above a
-predetermined threshold (7 by default) is regarded as adhering effectively to the best practices of positive
-instruction. This threshold can be custom-tailored through the test parameters.
-
-
Signs of High Risk
-
-
-
Low score obtained from the LLM analysis, indicating heavy reliance on negative instructions in the prompts.
-
Failure to surpass the preset minimum threshold.
-
The LLM generates ambiguous or undesirable outputs as a consequence of the negative instructions used in the
-prompt.
-
-
-
Strengths
-
-
-
Encourages the usage of affirmative, proactive language in prompts, aiding in more accurate and advantageous
-model responses.
-
The test result provides a comprehensible score, helping to understand how well a prompt follows the positive
-instruction best practices.
-
-
-
Limitations
-
-
-
Despite an adequate score, a prompt could still be misleading or could lead to undesired responses due to factors
-not covered by this test.
-
The test necessitates an LLM for evaluation, which might not be available or feasible in certain scenarios.
-
A numeric scoring system, while straightforward, may oversimplify complex issues related to prompt designing and
-instruction clarity.
-
The effectiveness of the test hinges significantly on the predetermined threshold level, which can be subjective
-and may need to be adjusted according to specific use-cases.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/prompt_validation/Robustness.html b/docs/_build/validmind/tests/prompt_validation/Robustness.html
deleted file mode 100644
index 416b1d112..000000000
--- a/docs/_build/validmind/tests/prompt_validation/Robustness.html
+++ /dev/null
@@ -1,302 +0,0 @@
-
-
-
-
-
-
- validmind.tests.prompt_validation.Robustness API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Assesses the robustness of prompts provided to a Large Language Model under varying conditions and contexts. This test
-specifically measures the model's ability to generate correct classifications with the given prompt even when the
-inputs are edge cases or otherwise difficult to classify.
-
-
Purpose
-
-
The Robustness test is meant to evaluate the resilience and reliability of prompts provided to a Language Learning
-Model (LLM). The aim of this test is to guarantee that the prompts consistently generate accurate and expected
-outputs, even in diverse or challenging scenarios. This test is only applicable to LLM-powered text classification
-tasks where the prompt has a single input variable.
-
-
Test Mechanism
-
-
The Robustness test appraises prompts under various conditions, alterations, and contexts to ascertain their
-stability in producing consistent responses from the LLM. Factors evaluated include different phrasings, inclusion
-of potential distracting elements, and various input complexities. By default, the test generates 10 inputs for a
-prompt but can be adjusted according to test parameters.
-
-
Signs of High Risk
-
-
-
If the output from the tests diverges extensively from the expected results, this indicates high risk.
-
When the prompt doesn't give a consistent performance across various tests.
-
A high risk is indicated when the prompt is susceptible to breaking, especially when the output is expected to be
-of a specific type.
-
-
-
Strengths
-
-
-
The robustness test helps to ensure stable performance of the LLM prompts and lowers the chances of generating
-unexpected or off-target outputs.
-
This test is vital for applications where predictability and reliability of the LLM’s output are crucial.
-
-
-
Limitations
-
-
-
Currently, the test only supports single-variable prompts, which restricts its application to more complex models.
-
When there are too many target classes (over 10), the test is skipped, which can leave potential vulnerabilities
-unchecked in complex multi-class models.
-
The test may not account for all potential conditions or alterations that could show up in practical use
-scenarios.
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/prompt_validation/Specificity.html b/docs/_build/validmind/tests/prompt_validation/Specificity.html
deleted file mode 100644
index 447394fc1..000000000
--- a/docs/_build/validmind/tests/prompt_validation/Specificity.html
+++ /dev/null
@@ -1,300 +0,0 @@
-
-
-
-
-
-
- validmind.tests.prompt_validation.Specificity API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Evaluates and scores the specificity of prompts provided to a Large Language Model (LLM), based on clarity, detail,
-and relevance.
-
-
Purpose
-
-
The Specificity Test evaluates the clarity, precision, and effectiveness of the prompts provided to a Language
-Model (LLM). It aims to ensure that the instructions embedded in a prompt are indisputably clear and relevant,
-thereby helping to remove ambiguity and steer the LLM towards desired outputs. This level of specificity
-significantly affects the accuracy and relevance of LLM outputs.
-
-
Test Mechanism
-
-
The Specificity Test employs an LLM to grade each prompt based on clarity, detail, and relevance parameters within
-a specificity scale that extends from 1 to 10. On this scale, prompts scoring equal to or more than a predefined
-threshold (set to 7 by default) pass the evaluation, while those scoring below this threshold fail it. Users can
-adjust this threshold as per their requirements.
-
-
Signs of High Risk
-
-
-
Prompts scoring consistently below the established threshold
-
Vague or ambiguous prompts that do not provide clear direction to the LLM
-
Overly verbose prompts that may confuse the LLM instead of providing clear guidance
-
-
-
Strengths
-
-
-
Enables precise and clear communication with the LLM to achieve desired outputs
-
Serves as a crucial means to measure the effectiveness of prompts
-
Highly customizable, allowing users to set their threshold based on specific use cases
-
-
-
Limitations
-
-
-
This test doesn't consider the content comprehension capability of the LLM
-
High specificity score doesn't guarantee a high-quality response from the LLM, as the model's performance is also
-dependent on various other factors
-
Striking a balance between specificity and verbosity can be challenging, as overly detailed prompts might confuse
-or mislead the model
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docs/_build/validmind/tests/prompt_validation/ai_powered_test.html b/docs/_build/validmind/tests/prompt_validation/ai_powered_test.html
deleted file mode 100644
index 6f22f7379..000000000
--- a/docs/_build/validmind/tests/prompt_validation/ai_powered_test.html
+++ /dev/null
@@ -1,305 +0,0 @@
-
-
-
-
-
-
- validmind.tests.prompt_validation.ai_powered_test API documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
Allows for setting options on the input object that are passed by the user
-when using the input to run a test or set of tests
-
-
To allow options, just override this method in the subclass (see VMDataset)
-and ensure that it returns a new instance of the input with the specified options
-set.
-
-
Arguments:
-
-
-
**kwargs: Arbitrary keyword arguments that will be passed to the input object
-
-
-
Returns:
-
-
-
VMInput: A new instance of the input with the specified options set
Child classes should be used to support new dataset types (tensor, polars etc)
-by converting the user's dataset into a numpy array collecting metadata like
-column names and then call this (parent) class __init__ method.
-
-
This way we can support multiple dataset types but under the hood we only
-need to work with numpy arrays and pandas dataframes in this class.
-
-
Attributes:
-
-
-
raw_dataset (np.ndarray): The raw dataset as a NumPy array.
-
input_id (str): Identifier for the dataset.
-
index (np.ndarray): The raw dataset index as a NumPy array.
-
columns (Set[str]): The column names of the dataset.
-
target_column (str): The target column name of the dataset.
-
feature_columns (List[str]): The feature column names of the dataset.
-
feature_columns_numeric (List[str]): The numeric feature column names of the dataset.
-
feature_columns_categorical (List[str]): The categorical feature column names of the dataset.
-
text_column (str): The text column name of the dataset for NLP tasks.
-
target_class_labels (Dict): The class labels for the target columns.
-
df (pd.DataFrame): The dataset as a pandas DataFrame.
-
extra_columns (Dict): Extra columns to include in the dataset.
Support options provided when passing an input to run_test or run_test_suite
-
-
Example:
-
-
-
# to only use a certain subset of columns in the dataset:
-run_test(
- "validmind.SomeTestID",
- inputs={
- "dataset":{
- "input_id":"my_dataset_id",
- "columns":["col1","col2"],
- }
- }
-)
-
-# behind the scenes, this retrieves the dataset object (VMDataset) from the registry
-# and then calls the `with_options()` method and passes `{"columns": ...}`
-
-
-
-
Arguments:
-
-
-
**kwargs: Options:
-
-
columns: Filter columns in the dataset
-
-
-
-
Returns:
-
-
-
VMDataset: A new instance of the dataset with only the specified columns
Returns the ipywidget compatible representation of the figure. Ideally
-we would render images as-is, but Plotly FigureWidgets don't work well
-on Google Colab when they are combined with ipywidgets.
-
-
-
-
-
-
-
- def
- serialize(self):
-
-
-
-
-
-
Serializes the Figure to a dictionary so it can be sent to the API
-
-
-
-
-
-
-
- def
- serialize_files(self):
-
-
-
-
-
-
Creates a requests-compatible files object to be sent to the API
figure (Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget,
-bytes, Figure]): The figure to add (can be either a VM Figure object,
-a raw figure object from the supported libraries, or a png image as
-raw bytes)
-
-
-
-
-
-
-
-
- def
- remove_figure(self, index:int=0):
-
-
-
-
-
-
Remove a figure from the result by index
-
-
Arguments:
-
-
-
index (int): The index of the figure to remove (default is 0)
-
-
-
-
-
-
-
-
- def
- to_widget(self):
-
-
-
-
-
-
Create an ipywdiget representation of the result... Must be overridden by subclasses
-
-
-
-
-
-
-
- def
- check_result_id_exist(self):
-
-
-
-
-
-
Check if the result_id exists in any test block across all sections
section_id (str): The section ID within the model document to insert the
-test result
-
position (int): The position (index) within the section to insert the test
-result
-
unsafe (bool): If True, log the result even if it contains sensitive data
-i.e. raw data from input datasets
-
-
-
-
-
-
-
Inherited Members
-
-
validmind.vm_models.result.result.Result
-
show
-
-
-
-
-
-
-
-
@dataclass
-
- class
- TestSuite:
-
-
-
-
-
-
Base class for test suites. Test suites are used to define a grouping of tests that
-can be run as a suite against datasets and models. Test Suites can be defined by
-inheriting from this base class and defining the list of tests as a class variable.
-
-
Tests can be a flat list of strings or may be nested into sections by using a dict
Returns the total number of tests in the test suite
-
-
-
-
-
-
-
- def
- get_default_config(self) -> dict:
-
-
-
-
-
-
Returns the default configuration for the test suite
-
-
Each test in a test suite can accept parameters and those parameters can have
-default values. Both the parameters and their defaults are set in the test
-class and a config object can be passed to the test suite's run method to
-override the defaults. This function returns a dictionary containing the
-parameters and their default values for every test to allow users to view
-and set values
-
-
Returns:
-
-
-
dict: A dictionary of test names and their default parameters
-{% enddefaultmacro %}
-{% defaultmacro is_public(doc) %}
- {#
- This macro is a bit unconventional in that its output is not rendered, but treated as a boolean:
- Returning no text is interpreted as false, returning any other text is iterpreted as true.
- Implementing this as a macro makes it very easy to override with a custom template, see
- https://github.com/mitmproxy/pdoc/tree/main/examples/custom-template.
- #}
- {% if doc.name == "__init__" and (doc.docstring or (doc.kind == "function" and doc.signature_without_self.parameters)) %}
- {# show constructors that have a docstring or at least one extra argument #}
- true
- {% elif doc.name == "__doc__" %}
- {# We don't want to document __doc__ itself, https://github.com/mitmproxy/pdoc/issues/235 #}
- {% elif doc.kind == "module" and doc.fullname not in all_modules %}
- {# Skip modules that were manually excluded, https://github.com/mitmproxy/pdoc/issues/334 #}
- {% elif (doc.qualname or doc.name) is in(module.obj.__all__ or []) %}
- {# members starting with an underscore are still public if mentioned in __all__ #}
- true
- {% elif not doc.name.startswith("_") and (doc.kind != "variable" or doc.is_enum_member or doc.docstring) %}
- {# members not starting with an underscore are considered public by default #}
- true
- {% endif %}
-{% enddefaultmacro %}
-{# fmt: off #}
-{% defaultmacro inherited(cls) %}
- {% for base, members in cls.inherited_members.items() %}
- {% set m = None %}{# workaround for https://github.com/pallets/jinja/issues/1427 #}
- {% set member_html %}
- {% for m in members if is_public(m) | trim %}
-
- {% if module.name == "validmind" %}
- ValidMind Library
- {% else %}
- {% set parts = module.modulename.split(".") %}
- {% for part in parts %}
- {%- set fullname = ".".join(parts[:loop.index]) -%}
- {%- if fullname in all_modules and fullname != module.modulename -%}
- {{ part }}
- {%- else -%}
- {{ part }}
- {%- endif -%}
- {%- if loop.nextitem -%}
- .
- {%- endif -%}
- {% endfor %}
- {% endif %}
-
-{% enddefaultmacro %}
diff --git a/docs/templates/module.qmd.jinja2 b/docs/templates/module.qmd.jinja2
new file mode 100644
index 000000000..2c42b8d1a
--- /dev/null
+++ b/docs/templates/module.qmd.jinja2
@@ -0,0 +1,312 @@
+{% import "macros/docstring.jinja2" as doc %}
+{% import "macros/types.jinja2" as types %}
+{% import "macros/navigation.jinja2" as nav %}
+{% import "macros/signatures.jinja2" as signatures %}
+---
+title: "{% if module.name == "validmind" %}ValidMind Library Python API{% else %}[validmind](/validmind/validmind.qmd).{{ module.name }}{% endif +%}"
+{% if module.name == "validmind" %}
+aliases:
+ - index.html
+{% endif %}
+sidebar: validmind-reference
+{% if module.name == "validmind" %}
+toc: false
+{% else %}
+toc-depth: 4
+toc-expand: 4
+{% endif %}
+# module.qmd.jinja2
+---
+
+{% if module.docstring %}
+{{ doc.format_docstring(module.docstring) }}
+{% endif %}
+
+{% if module.members and module.name == "validmind" %}
+
+
+{% if module.members.__version__ %}
+## __version__
+
+{{ signatures.render_version_signature(module.members.__version__) }}
+{% else %}
+::: {.signature}
+
+{{ module.members.__version__.value | replace("'", "") if module.members.__version__.value else module.members.__version__.members.__version__.value | replace("'", "") }}
+
+:::
+{% endif %}
+
+{# Process root-level aliases #}
+{% if module.all_list %}
+{# Use __all__ list ordering when available #}
+{% for member_name in module.all_list %}
+{% if member_name in module.members %}
+{% set member = module.members[member_name] %}
+{% if is_public(member, module, full_data, is_root) and member.kind == "alias" %}
+{% set target = resolve_alias(member, full_data) %}
+{% if target and target.docstring %}
+## {{ member.name }}
+
+{% if target.kind == "function" %}
+{{ signatures.render_signature(target) }}
+{% endif %}
+
+{{ doc.format_docstring(target.docstring) }}
+{% endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% else %}
+{# Fallback to original sorting method #}
+{% for member in module.members | sort_members %}
+{% if is_public(member, module, full_data, is_root) and member.kind == "alias" %}
+{% set target = resolve_alias(member, full_data) %}
+{% if target and target.docstring %}
+## {{ member.name }}
+
+{% if target.kind == "function" %}
+{{ signatures.render_signature(target) }}
+{% endif %}
+
+{{ doc.format_docstring(target.docstring) }}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endif %}
+{% endif %}
+
+{% if module.members %}
+{# List modules #}
+{% set has_modules = namespace(value=false) %}
+{% for member in module.members | sort_members %}
+{% if is_public(member, module, full_data, is_root) and member.kind == "module" %}
+{% set has_modules.value = true %}
+{% endif %}
+{% endfor %}
+
+{% if not is_root %}
+{% for member in module.members | sort_members %}
+{% if is_public(member, module, full_data, is_root) and member.kind == "module" %}
+- [{{ member.name }}]({{ module.name }}/{{ member.name }}.qmd)
+{% endif %}
+{% endfor %}
+{% endif %}
+
+{# Process module-level aliases #}
+{% if not is_root %}
+
+{# Process module-level alias attributes (like describe_test_suite) #}
+{% for member_name, member in module.members.items() %}
+{% if member.kind == "attribute" and member.labels is defined and "module-attribute" in member.labels and member.value is defined and member.value.cls == "ExprName" and member.value.name in module.members %}
+{# This is a module-level alias pointing to another function in the same module #}
+{% set target_name = member.value.name %}
+{% set target = module.members[target_name] %}
+
+## {{ member_name }}{% if target.kind == "function" %}{% endif %}
+
+*This function is an alias for [{{ target_name }}](#{{ target_name }}).*
+{% endif %}
+{% endfor %}
+
+{% if module.all_list %}
+{# Use __all__ list ordering when available #}
+{% for member_name in module.all_list %}
+{% if member_name in module.members %}
+{% set member = module.members[member_name] %}
+{% if is_public(member, module, full_data, is_root) and member.kind == "alias" %}
+{% set resolved = resolve_alias(member, full_data) %}
+{% if resolved.kind == "function" or (resolved.kind == "attribute" and not module.path.startswith('validmind.tests')) %}
+## {{ member.name }}{% if resolved.kind == "function" %}{% endif %}
+
+{{ signatures.render_signature(resolved, full_data=full_data, module=module) }}
+
+{% if resolved.docstring %}
+{{ doc.format_docstring(resolved.docstring) }}
+{% endif %}
+{% endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% else %}
+{# Fallback to original sorting method #}
+{% for member in module.members | sort_members %}
+{% if is_public(member, module, full_data, is_root) and member.kind == "alias" %}
+{% set resolved = resolve_alias(member, full_data) %}
+{% if resolved.kind == "function" or (resolved.kind == "attribute" and not module.path.startswith('validmind.tests')) %}
+## {{ member.name }}{% if resolved.kind == "function" %}{% endif %}
+
+{{ signatures.render_signature(resolved, full_data=full_data, module=module) }}
+
+{% if resolved.docstring %}
+{{ doc.format_docstring(resolved.docstring) }}
+{% endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endif %}
+{% endif %}
+
+{# List classes and functions #}
+{% if module.all_list %}
+{# Use __all__ list ordering when available #}
+{% for member_name in module.all_list %}
+{% if member_name in module.members %}
+{% set member = module.members[member_name] %}
+{% if is_public(member, module, full_data, is_root) %}
+{% set resolved = resolve_alias(member, full_data) %}
+{% if resolved.kind == "attribute" and member.kind != "alias" and module.name == "validmind" and member.name in get_all_members(full_data['validmind'].get('members', {})) and resolved.value %}
+## {{ member.name }}
+
+{{ signatures.render_signature(resolved, full_data=full_data, module=module) }}
+
+{% if resolved.docstring %}
+{{ doc.format_docstring(resolved.docstring) }}
+{% endif %}
+
+{% elif resolved.kind == "class" %}
+
+{% set __module_path = module.path|default('') %}
+{% set __is_test_module = __module_path.startswith('validmind.tests.') %}
+{% set __is_error_class = resolved.name.endswith('Error') %}
+{% set __is_test_suite = module.name == "test_suites" or __module_path == "validmind.test_suites" %}
+
+{# Skip rendering test suite classes in the main test_suites.qmd file #}
+{% if __is_test_suite and module.path == "validmind.test_suites" %}
+ {# Skip the class in the main test_suites module, individual test suite modules will show them #}
+{% elif not (__is_test_module and __is_error_class) %}
+ {% include "class.qmd.jinja2" with context %}
+{% endif %}
+{% elif resolved.kind == "function" and member.kind != "alias" %}
+{% include "function.qmd.jinja2" %}
+{% endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% else %}
+{# Fallback to original sorting method #}
+{% for member in module.members | sort_members %}
+{% if is_public(member, module, full_data, is_root) %}
+{% set resolved = resolve_alias(member, full_data) %}
+{% if resolved.kind == "attribute" and member.kind != "alias" and module.name == "validmind" and member.name in get_all_members(full_data['validmind'].get('members', {})) and resolved.value %}
+## {{ member.name }}
+
+{{ signatures.render_signature(resolved, full_data=full_data, module=module) }}
+
+{% if resolved.docstring %}
+{{ doc.format_docstring(resolved.docstring) }}
+{% endif %}
+
+{% elif resolved.kind == "class" %}
+
+{% set __module_path = module.path|default('') %}
+{% set __is_test_module = __module_path.startswith('validmind.tests.') %}
+{% set __is_error_class = resolved.name.endswith('Error') %}
+{% set __is_test_suite = module.name == "test_suites" or __module_path == "validmind.test_suites" %}
+
+{# Skip rendering test suite classes in the main test_suites.qmd file #}
+{% if __is_test_suite and module.path == "validmind.test_suites" %}
+ {# Skip the class in the main test_suites module, individual test suite modules will show them #}
+{% elif not (__is_test_module and __is_error_class) %}
+ {% include "class.qmd.jinja2" with context %}
+{% endif %}
+{% elif resolved.kind == "function" and member.kind != "alias" %}
+{% include "function.qmd.jinja2" %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endif %}
+{% endif %}
+
+{% if module.name == "validmind" %}
+{% if module.all_list %}
+{# Use __all__ list ordering when available #}
+{% for member_name in module.all_list %}
+{% if member_name in module.members %}
+{% set member = module.members[member_name] %}
+{% if is_public(member, module, full_data, is_root) %}
+{% set resolved = resolve_alias(member, full_data) %}
+{% if member.kind == "class" or (member.kind == "alias" and member.target_path and member.target_path.split(".")[-1][0].isupper()) %}
+{% set target = resolve_alias(resolved, full_data) %}
+
+{# Skip rendering TestSuite classes to avoid duplication #}
+{% set is_test_suite_class = member.target_path and 'test_suites' in member.target_path %}
+{% if not is_test_suite_class %}
+
+## {{ member.name }}
+
+{{ signatures.render_signature(target) }}
+
+{% if target.docstring %}
+{{ doc.format_docstring(target.docstring) }}
+{% endif %}
+
+{% if target.members %}
+{% for method_name, method in target.members.items() %}
+{% if method.kind == "function" and (not method_name.startswith('_') or method_name in ['__init__']) %}
+
+
+### {{ member.name if method_name == '__init__' else method_name }}
+
+{% set method_with_parent = method %}
+{% set _ = method_with_parent.update({"parent": {"name": member.name}}) %}
+{{ signatures.render_signature(method_with_parent) }}
+
+{% if method.docstring %}
+{{ doc.format_docstring(method.docstring) }}
+{% endif %}
+
+{% endif %}
+{% endfor %}
+{% endif %}
+
+{% endif %}
+{% endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% else %}
+{# Fallback to original sorting method #}
+{% for member in module.members | sort_members %}
+{% if is_public(member, module, full_data, is_root) %}
+{% set resolved = resolve_alias(member, full_data) %}
+{% if member.kind == "class" or (member.kind == "alias" and member.target_path and member.target_path.split(".")[-1][0].isupper()) %}
+{% set target = resolve_alias(resolved, full_data) %}
+
+{# Skip rendering TestSuite classes to avoid duplication #}
+{% set is_test_suite_class = member.target_path and 'test_suites' in member.target_path %}
+{% if not is_test_suite_class %}
+
+## {{ member.name }}
+
+{{ signatures.render_signature(target) }}
+
+{% if target.docstring %}
+{{ doc.format_docstring(target.docstring) }}
+{% endif %}
+
+{% if target.members %}
+{% for method_name, method in target.members.items() %}
+{% if method.kind == "function" and (not method_name.startswith('_') or method_name in ['__init__']) %}
+
+
+### {{ member.name if method_name == '__init__' else method_name }}
+
+{% set method_with_parent = method %}
+{% set _ = method_with_parent.update({"parent": {"name": member.name}}) %}
+{{ signatures.render_signature(method_with_parent) }}
+
+{% if method.docstring %}
+{{ doc.format_docstring(method.docstring) }}
+{% endif %}
+
+{% endif %}
+{% endfor %}
+{% endif %}
+
+{% endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endif %}
+{% endif %}
diff --git a/docs/templates/sidebar.qmd.jinja2 b/docs/templates/sidebar.qmd.jinja2
new file mode 100644
index 000000000..3936b8d41
--- /dev/null
+++ b/docs/templates/sidebar.qmd.jinja2
@@ -0,0 +1,70 @@
+# sidebar.qmd.jinja2
+website:
+ sidebar:
+ - id: validmind-reference
+ title: "ValidMind Library Python API"
+ collapsed: false
+ collapse-level: 2
+ contents:
+ - validmind/validmind.qmd
+ - text: "---"
+ - text: "Python API"
+ # Root level items from validmind.qmd
+ {% if documented_items.get('root') %}
+ {% for item in documented_items['root'] %}
+ {% if "__version__" in item.text %}
+ - text: "`{{ module.members.__version__.members.__version__.value | replace("'", "") if module.members.__version__.members.__version__.value else module.members.__version__.value | replace("'", "") }}`"
+ file: {{ item.file | replace("__version__", "version__") }}
+ {% else %}
+ {% set cleaned_path = item.file | replace(' ', '') | replace('', '') %}
+ - text: "{{ item.text | replace('', '\'>') }}"
+ file: {{ cleaned_path }}
+ {% if item.contents is defined and item.contents %}
+ contents:
+ {% for method in item.contents %}
+ {% set cleaned_method_path = method.file | replace(' ', '') | replace('', '') %}
+ - text: "{{ method.text | replace('', '\'>') }}"
+ file: {{ cleaned_method_path }}
+ {% endfor %}
+ {% endif %}
+ {% endif %}
+ {% endfor %}
+ {% endif %}
+ # All module documentation pages
+ - text: "---"
+ - text: "Submodules"
+ {% if module.members.__version__ %}
+ - text: "__version__"
+ file: validmind/validmind/version.qmd
+ {% endif %}
+ {% for member in module.members | sort_members %}
+ {% if is_public(member, module, full_data, is_root) and member.kind == "module" %}
+ {% set module_name = member.name %}
+ {% set has_children = qmd_files | has_subfiles(module_name) %}
+ {% if has_children %}
+ - text: "{{ module_name }}"
+ file: validmind/validmind/{{ module_name }}.qmd
+ contents:
+ {% for item in qmd_files | get_child_files(module_name) %}
+ {% if item.contents is defined %}
+ {% set cleaned_item_path = item.file | replace(' ', '') | replace('', '') %}
+ - text: "{{ item.text | replace('', '\'>') }}"
+ file: {{ cleaned_item_path }}
+ contents:
+ {% for child in item.contents %}
+ {% set cleaned_child_path = child.file | replace(' ', '') | replace('', '') %}
+ - text: "{{ child.text | replace('', '\'>') }}"
+ file: {{ cleaned_child_path }}
+ {% endfor %}
+ {% else %}
+ {% set cleaned_item_path = item.file | replace(' ', '') | replace('', '') %}
+ - text: "{{ item.text | replace('', '\'>') }}"
+ file: {{ cleaned_item_path }}
+ {% endif %}
+ {% endfor %}
+ {% else %}
+ - text: "{{ module_name }}"
+ file: validmind/validmind/{{ module_name }}.qmd
+ {% endif %}
+ {% endif %}
+ {% endfor %}
\ No newline at end of file
diff --git a/docs/templates/version.qmd.jinja2 b/docs/templates/version.qmd.jinja2
new file mode 100644
index 000000000..d67619fed
--- /dev/null
+++ b/docs/templates/version.qmd.jinja2
@@ -0,0 +1,8 @@
+---
+title: "[validmind](/validmind/validmind.qmd).__version__"
+sidebar: validmind-reference
+---
+
+
+{% from "macros/signatures.jinja2" import render_version_signature %}
+{{ render_version_signature(module.members.__version__) }}
diff --git a/docs/validmind.css b/docs/validmind.css
new file mode 100644
index 000000000..afffae6e4
--- /dev/null
+++ b/docs/validmind.css
@@ -0,0 +1,160 @@
+#quarto-sidebar.sidebar {
+ background-color: #FFFFFF !important;
+}
+
+.sidebar-header .sidebar-title a {
+text-decoration: none;
+}
+
+.sidebar.sidebar-navigation:not(.rollup) {
+border-right: none !important;
+}
+
+.sidebar-item {
+ color: #747678;
+ line-height: 1.1;
+}
+
+nav#TOC {
+ border: none;
+ background-color: #fff;
+}
+
+p code:not(.sourceCode), li code:not(.sourceCode), td code:not(.sourceCode) {
+ color: #003B4F;
+ background-color: #F0F1F1;
+ font-size: 0.9em;
+ border: none;
+}
+
+div.sourceCode, div.sourceCode pre.sourceCode {
+ color: #003B4F;
+ background-color: #F0F1F1;
+ border: none;
+}
+
+.prefix {
+ position: relative;
+ margin-right: 0px;
+}
+
+.prefix::before {
+ content: "Class";
+ opacity: 0.6;
+ font-size: 0.9em;
+}
+
+.suffix {
+ position: relative;
+ margin-left: 1px;
+}
+
+.suffix::after {
+ content: "()";
+ opacity: 0.6;
+ font-size: 0.9em;
+}
+
+.muted {
+ opacity: 0.6;
+}
+
+.version {
+ font-weight: bold;
+ border: 1px solid #196972;
+ border-radius: 3px;
+ padding: 2px 6px;
+ display: inline-block;
+}
+
+.signature {
+ font-family: 'JetBrains Mono', 'Fira Code', Menlo, Monaco, 'Courier New', monospace;
+ color: #003B4F;
+ background-color: #F0F1F1;
+ padding: 0 25px;
+ border-radius: 5px;
+ margin: 1em 0;
+ white-space: pre-wrap;
+ overflow-x: auto;
+ font-size: 0.9em;
+ line-height: 1.5;
+}
+
+.signature .param {
+ margin-bottom: 0px;
+}
+
+.signature .params {
+ display: block;
+ margin-left: 20px;
+ margin-bottom: 0px;
+}
+
+.signature .muted {
+ display: inline;
+ white-space: nowrap;
+}
+
+.signature p {
+ margin-bottom: 0;
+}
+
+.signature .kw {
+ color: #008080;
+ font-weight: bold;
+ padding-right: 4px;
+}
+
+.signature .name {
+ color: #de257e;
+ font-weight: bold;
+ padding-right: 2px;
+}
+
+.signature .n {
+ color: #003B4F;
+}
+
+.signature .o {
+ color: #5E5E5E;
+ padding-left: 2px;
+ padding-right: 4px;
+}
+
+.signature .p {
+ padding-right: 2px;
+}
+
+.signature .kc {
+ color: #008080;
+ font-weight: bold;
+}
+
+.signature .bp {
+ color: #008080;
+ font-weight: bold;
+}
+
+.signature .nb {
+ color: #008080;
+ font-weight: bold;
+}
+
+.signature .s1 {
+ color: #8225de;
+}
+
+.signature .ann {
+ color: #20794D;
+}
+
+.signature .decorators {
+ display: block;
+ margin-bottom: -20px;
+}
+
+.signature .decorator {
+ display: inline-block;
+ color: #5E5E5E;
+ font-size: 0.9em;
+}
diff --git a/docs/validmind.qmd b/docs/validmind.qmd
new file mode 100644
index 000000000..9a4c5f469
--- /dev/null
+++ b/docs/validmind.qmd
@@ -0,0 +1,532 @@
+---
+title: "ValidMind Library Python API"
+aliases:
+ - index.html
+sidebar: validmind-reference
+toc: false
+# module.qmd.jinja2
+---
+
+
+
+The ValidMind Library is a suite of developer tools and methods designed to automate the documentation and validation of your models.
+
+Designed to be model agnostic, the ValidMind Library provides all the standard functionality without requiring you to rewrite any functions as long as your model is built in Python.
+
+With a rich array of documentation tools and test suites, from documenting descriptions of your datasets to testing your models for weak spots and overfit areas, the ValidMind Library helps you automate model documentation by feeding the ValidMind Platform with documentation artifacts and test results.
+
+To install the ValidMind Library:
+
+```bash
+pip install validmind
+```
+
+To initialize the ValidMind Library, paste the code snippet with the model identifier credentials directly into your development source code, replacing this example with your own:
+
+```python
+import validmind as vm
+
+vm.init(
+ api_host = "https://api.dev.vm.validmind.ai/api/v1/tracking/tracking",
+ api_key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+ api_secret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+ project = ""
+)
+```
+
+After you have pasted the code snippet into your development source code and executed the code, the Python Library API will register with ValidMind. You can now use the ValidMind Library to document and test your models, and to upload to the ValidMind Platform.
+
+
+
+## \_\_version\_\_
+
+
+
+::: {.signature}
+
+2.8.20
+
+:::
+
+## init
+
+
+
+::: {.signature}
+
+definit(project:Optional\[str\]=None,api_key:Optional\[str\]=None,api_secret:Optional\[str\]=None,api_host:Optional\[str\]=None,model:Optional\[str\]=None,monitoring:bool=False,generate_descriptions:Optional\[bool\]=None):
+
+:::
+
+
+
+Initializes the API client instances and calls the /ping endpoint to ensure the provided credentials are valid and we can connect to the ValidMind API.
+
+If the API key and secret are not provided, the client will attempt to retrieve them from the environment variables `VM_API_KEY` and `VM_API_SECRET`.
+
+**Arguments**
+
+- `project (str, optional)`: The project CUID. Alias for model. Defaults to None. \[DEPRECATED\]
+- `model (str, optional)`: The model CUID. Defaults to None.
+- `api_key (str, optional)`: The API key. Defaults to None.
+- `api_secret (str, optional)`: The API secret. Defaults to None.
+- `api_host (str, optional)`: The API host. Defaults to None.
+- `monitoring (bool, optional)`: The ongoing monitoring flag. Defaults to False.
+- `generate_descriptions (bool, optional)`: Whether to use GenAI to generate test result descriptions. Defaults to True.
+
+**Raises**
+
+- `ValueError`: If the API key and secret are not provided
+
+## init_dataset
+
+
+
+::: {.signature}
+
+definit_dataset(dataset:Union\[pd.DataFrame, pl.DataFrame, np.ndarray, torch.utils.data.tensordataset\],model:Optional\[validmind.vm_models.VMModel\]=None,index:Optional\[Any\]=None,index_name:Optional\[str\]=None,date_time_index:bool=False,columns:Optional\[List\[str\]\]=None,text_column:Optional\[str\]=None,target_column:Optional\[str\]=None,feature_columns:Optional\[List\[str\]\]=None,extra_columns:Optional\[Dict\[str, Any\]\]=None,class_labels:Optional\[Dict\[str, Any\]\]=None,type:Optional\[str\]=None,input_id:Optional\[str\]=None,\_\_log:bool=True) → validmind.vm_models.VMDataset:
+
+:::
+
+
+
+Initializes a VM Dataset, which can then be passed to other functions that can perform additional analysis and tests on the data. This function also ensures we are reading a valid dataset type.
+
+The following dataset types are supported:
+
+- Pandas DataFrame
+- Polars DataFrame
+- Numpy ndarray
+- Torch TensorDataset
+
+**Arguments**
+
+- `dataset`: Dataset from various Python libraries.
+- `model (VMModel)`: ValidMind model object.
+- `index (Any)`: Index for the dataset.
+- `index_name (str)`: Name of the index column.
+- `date_time_index (bool)`: Whether the index is a datetime index.
+- `columns (List[str])`: List of column names.
+- `text_column (str)`: Name of the text column.
+- `target_column (str)`: The name of the target column in the dataset.
+- `feature_columns (List[str])`: A list of names of feature columns in the dataset.
+- `extra_columns (Dict[str, Any])`: A dictionary containing the names of the prediction_column and group_by_columns in the dataset.
+- `class_labels (Dict[str, Any])`: A list of class labels for classification problems.
+- `type (str)`: The type of dataset (one of DATASET_TYPES) - DEPRECATED.
+- `input_id (str)`: The input ID for the dataset (e.g. "my_dataset"). By default, this will be set to `dataset` but if you are passing this dataset as a test input using some other key than `dataset`, then you should set this to the same key.
+- `__log (bool, optional)`: Whether to log the input. Defaults to True.
+
+**Returns**
+
+- A VM Dataset instance.
+
+**Raises**
+
+- `ValueError`: If the dataset type is not supported.
+
+## init_model
+
+
+
+::: {.signature}
+
+definit_model(model:Optional\[object\]=None,input_id:str='model',attributes:Optional\[Dict\[str, Any\]\]=None,predict_fn:Optional\[Callable\]=None,\_\_log:bool=True,\*\*kwargs:Any) → validmind.vm_models.VMModel:
+
+:::
+
+
+
+Initializes a VM Model, which can then be passed to other functions that can perform additional analysis and tests on the data. This function also ensures we are creating a model supported libraries.
+
+**Arguments**
+
+- `model`: A trained model or VMModel instance.
+- `input_id (str)`: The input ID for the model (e.g. "my_model"). By default, this will be set to `model` but if you are passing this model as a test input using some other key than `model`, then you should set this to the same key.
+- `attributes (dict)`: A dictionary of model attributes.
+- `predict_fn (callable)`: A function that takes an input and returns a prediction.
+- `**kwargs`: Additional arguments to pass to the model.
+
+**Returns**
+
+- A VM Model instance.
+
+**Raises**
+
+- `ValueError`: If the model type is not supported.
+
+## init_r_model
+
+
+
+::: {.signature}
+
+definit_r_model(model_path:str,input_id:str='model') → validmind.vm_models.VMModel:
+
+:::
+
+
+
+Initialize a VM Model from an R model.
+
+LogisticRegression and LinearRegression models are converted to sklearn models by extracting the coefficients and intercept from the R model. XGB models are loaded using the xgboost since xgb models saved in .json or .bin format can be loaded directly with either Python or R.
+
+**Arguments**
+
+- `model_path (str)`: The path to the R model saved as an RDS or XGB file.
+- `input_id (str, optional)`: The input ID for the model. Defaults to "model".
+
+**Returns**
+
+- A VM Model instance.
+
+## get_test_suite
+
+
+
+::: {.signature}
+
+defget_test_suite(test_suite_id:Optional\[str\]=None,section:Optional\[str\]=None,\*args:Any,\*\*kwargs:Any) → validmind.vm_models.TestSuite:
+
+:::
+
+
+
+Gets a TestSuite object for the current project or a specific test suite.
+
+This function provides an interface to retrieve the TestSuite instance for the current project or a specific TestSuite instance identified by test_suite_id. The project Test Suite will contain sections for every section in the project's documentation template and these Test Suite Sections will contain all the tests associated with that template section.
+
+**Arguments**
+
+- `test_suite_id (str, optional)`: The test suite name. If not passed, then the project's test suite will be returned. Defaults to None.
+- `section (str, optional)`: The section of the documentation template from which to retrieve the test suite. This only applies if test_suite_id is None. Defaults to None.
+- `args`: Additional arguments to pass to the TestSuite.
+- `kwargs`: Additional keyword arguments to pass to the TestSuite.
+
+**Returns**
+
+- The TestSuite instance.
+
+## log_metric
+
+
+
+::: {.signature}
+
+deflog_metric(key:str,value:float,inputs:Optional\[List\[str\]\]=None,params:Optional\[Dict\[str, Any\]\]=None,recorded_at:Optional\[str\]=None,thresholds:Optional\[Dict\[str, Any\]\]=None):
+
+:::
+
+
+
+Logs a unit metric.
+
+Unit metrics are key-value pairs where the key is the metric name and the value is a scalar (int or float). These key-value pairs are associated with the currently selected model (inventory model in the ValidMind Platform) and keys can be logged to over time to create a history of the metric. On the ValidMind Platform, these metrics will be used to create plots/visualizations for documentation and dashboards etc.
+
+**Arguments**
+
+- `key (str)`: The metric key
+- `value (Union[int, float])`: The metric value
+- `inputs (List[str])`: List of input IDs
+- `params (Dict[str, Any])`: Parameters used to generate the metric
+- `recorded_at (str)`: Timestamp when the metric was recorded
+- `thresholds (Dict[str, Any])`: Thresholds for the metric
+
+## preview_template
+
+
+
+::: {.signature}
+
+defpreview_template():
+
+:::
+
+
+
+Preview the documentation template for the current project.
+
+This function will display the documentation template for the current project. If the project has not been initialized, then an error will be raised.
+
+**Raises**
+
+- `ValueError`: If the project has not been initialized.
+
+## print_env
+
+
+
+::: {.signature}
+
+defprint_env():
+
+:::
+
+
+
+Prints a log of the running environment for debugging.
+
+Output includes: ValidMind Library version, operating system details, installed dependencies, and the ISO 8601 timestamp at log creation.
+
+## reload
+
+
+
+::: {.signature}
+
+defreload():
+
+:::
+
+
+
+Reconnect to the ValidMind API and reload the project configuration.
+
+## run_documentation_tests
+
+
+
+::: {.signature}
+
+defrun_documentation_tests(section:Optional\[str\]=None,send:bool=True,fail_fast:bool=False,inputs:Optional\[Dict\[str, Any\]\]=None,config:Optional\[Dict\[str, Any\]\]=None,\*\*kwargs:Any) → Union\[validmind.vm_models.TestSuite, Dict\[str, validmind.vm_models.TestSuite\]\]:
+
+:::
+
+
+
+Collect and run all the tests associated with a template.
+
+This function will analyze the current project's documentation template and collect all the tests associated with it into a test suite. It will then run the test suite, log the results to the ValidMind API, and display them to the user.
+
+**Arguments**
+
+- `section (str or list, optional)`: The section(s) to preview. Defaults to None.
+- `send (bool, optional)`: Whether to send the results to the ValidMind API. Defaults to True.
+- `fail_fast (bool, optional)`: Whether to stop running tests after the first failure. Defaults to False.
+- `inputs (dict)`: A dictionary of test inputs to pass to the TestSuite.
+- `config`: A dictionary of test parameters to override the defaults.
+- `**kwargs`: backwards compatibility for passing in test inputs using keyword arguments.
+
+**Returns**
+
+- TestSuite or dict: The completed TestSuite instance or a dictionary of TestSuites if section is a list.
+
+**Raises**
+
+- `ValueError`: If the project has not been initialized.
+
+## run_test_suite
+
+
+
+::: {.signature}
+
+defrun_test_suite(test_suite_id:str,send:bool=True,fail_fast:bool=False,config:Optional\[Dict\[str, Any\]\]=None,inputs:Optional\[Dict\[str, Any\]\]=None,\*\*kwargs:Any) → validmind.vm_models.TestSuite:
+
+:::
+
+
+
+High Level function for running a test suite.
+
+This function provides a high level interface for running a test suite. A test suite is a collection of tests. This function will automatically find the correct test suite class based on the test_suite_id, initialize each of the tests, and run them.
+
+**Arguments**
+
+- `test_suite_id (str)`: The test suite name. For example, 'classifier_full_suite'.
+- `config (dict, optional)`: A dictionary of parameters to pass to the tests in the test suite. Defaults to None.
+- `send (bool, optional)`: Whether to post the test results to the API. send=False is useful for testing. Defaults to True.
+- `fail_fast (bool, optional)`: Whether to stop running tests after the first failure. Defaults to False.
+- `inputs (dict, optional)`: A dictionary of test inputs to pass to the TestSuite, such as `model`, `dataset` `models`, etc. These inputs will be accessible by any test in the test suite. See the test documentation or `vm.describe_test()` for more details on the inputs required for each. Defaults to None.
+- `**kwargs`: backwards compatibility for passing in test inputs using keyword arguments.
+
+**Returns**
+
+- The TestSuite instance.
+
+**Raises**
+
+- `ValueError`: If the test suite name is not found or if there is an error initializing the test suite.
+
+## tags
+
+
+
+::: {.signature}
+
+deftags(\*tags:str):
+
+:::
+
+
+
+Decorator for specifying tags for a test.
+
+**Arguments**
+
+- `*tags`: The tags to apply to the test.
+
+## tasks
+
+
+
+::: {.signature}
+
+deftasks(\*tasks:str):
+
+:::
+
+
+
+Decorator for specifying the task types that a test is designed for.
+
+**Arguments**
+
+- `*tasks`: The task types that the test is designed for.
+
+## test
+
+
+
+::: {.signature}
+
+deftest(func_or_id:Union\[Callable\[..., Any\], str, None\]):
+
+:::
+
+
+
+Decorator for creating and registering custom tests
+
+This decorator registers the function it wraps as a test function within ValidMind under the provided ID. Once decorated, the function can be run using the `validmind.tests.run_test` function.
+
+The function can take two different types of arguments:
+
+- Inputs: ValidMind model or dataset (or list of models/datasets). These arguments must use the following names: `model`, `models`, `dataset`, `datasets`.
+- Parameters: Any additional keyword arguments of any type (must have a default value) that can have any name.
+
+The function should return one of the following types:
+
+- Table: Either a list of dictionaries or a pandas DataFrame
+- Plot: Either a matplotlib figure or a plotly figure
+- Scalar: A single number (int or float)
+- Boolean: A single boolean value indicating whether the test passed or failed
+
+The function may also include a docstring. This docstring will be used and logged as the metric's description.
+
+**Arguments**
+
+- `func_or_id (Union[Callable[..., Any], str, None])`: Either the function to decorate or the test ID. If None, the function name is used.
+
+**Returns**
+
+- The decorated function.
+
+## log_text
+
+
+
+::: {.signature}
+
+deflog_text(content_id:str,text:str,\_json:Optional\[Dict\[str, Any\]\]=None) → Dict\[str, Any\]:
+
+:::
+
+
+
+Logs free-form text to ValidMind API.
+
+**Arguments**
+
+- `content_id (str)`: Unique content identifier for the text.
+- `text (str)`: The text to log. Will be converted to HTML with MathML support.
+- `_json (dict, optional)`: Additional metadata to associate with the text. Defaults to None.
+
+**Returns**
+
+- An accordion widget containing the logged text as HTML.
+
+**Raises**
+
+- `ValueError`: If content_id or text are empty or not strings.
+- `Exception`: If the API call fails.
+
+
+
+## RawData
+
+
+
+::: {.signature}
+
+classRawData:
+
+:::
+
+
+
+Holds raw data for a test result.
+
+
+
+### RawData
+
+
+
+::: {.signature}
+
+RawData(log:bool=False,\*\*kwargs:Any)
+
+:::
+
+
+
+Create a new RawData object.
+
+**Arguments**
+
+- `log (bool)`: If True, log the raw data to ValidMind.
+- `**kwargs`: Keyword arguments to set as attributes, such as `RawData(log=True, dataset_duplicates=df_duplicates)`.
+
+
+
+### inspect
+
+
+
+::: {.signature}
+
+definspect(self,show:bool=True) → Optional\[Dict\[str, Any\]\]:
+
+:::
+
+
+
+Inspect the raw data.
+
+**Arguments**
+
+- `show (bool)`: If True, print the raw data. If False, return it.
+
+**Returns**
+
+- If True, print the raw data and return None. If False, return the raw data dictionary.
+
+
+
+### serialize
+
+
+
+::: {.signature}
+
+defserialize(self) → Dict\[str, Any\]:
+
+:::
+
+
+
+Serialize the raw data to a dictionary
+
+**Returns**
+
+- The serialized raw data
diff --git a/docs/validmind/datasets.qmd b/docs/validmind/datasets.qmd
new file mode 100644
index 000000000..f02b4a9c9
--- /dev/null
+++ b/docs/validmind/datasets.qmd
@@ -0,0 +1,16 @@
+---
+title: "[validmind](/validmind/validmind.qmd).datasets"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Example datasets that can be used with the ValidMind Library.
+
+- [classification](datasets/classification.qmd)
+- [credit_risk](datasets/credit_risk.qmd)
+- [nlp](datasets/nlp.qmd)
+- [regression](datasets/regression.qmd)
diff --git a/docs/validmind/datasets/classification.qmd b/docs/validmind/datasets/classification.qmd
new file mode 100644
index 000000000..9b40ca7cd
--- /dev/null
+++ b/docs/validmind/datasets/classification.qmd
@@ -0,0 +1,14 @@
+---
+title: "[validmind](/validmind/validmind.qmd).classification"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Entrypoint for classification datasets.
+
+- [customer_churn](classification/customer_churn.qmd)
+- [taiwan_credit](classification/taiwan_credit.qmd)
diff --git a/docs/validmind/datasets/classification/customer_churn.qmd b/docs/validmind/datasets/classification/customer_churn.qmd
new file mode 100644
index 000000000..64b4ebfc9
--- /dev/null
+++ b/docs/validmind/datasets/classification/customer_churn.qmd
@@ -0,0 +1,62 @@
+---
+title: "[validmind](/validmind/validmind.qmd).customer_churn"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## get_demo_test_config
+
+
+
+::: {.signature}
+
+defget_demo_test_config(test_suite=None):
+
+:::
+
+
+
+Returns input configuration for the default documentation template assigned to this demo model
+
+The default documentation template uses the following inputs:
+
+- raw_dataset
+- train_dataset
+- test_dataset
+- model
+
+We assign the following inputs depending on the input config expected by each test:
+
+- When a test expects a "dataset" we use the raw_dataset
+- When a tets expects "datasets" we use the train_dataset and test_dataset
+- When a test expects a "model" we use the model
+- When a test expects "model" and "dataset" we use the model and test_dataset
+- The only exception is ClassifierPerformance since that runs twice: once with the train_dataset (in sample) and once with the test_dataset (out of sample)
+
+
+
+## load_data
+
+
+
+::: {.signature}
+
+defload_data(full_dataset=False):
+
+:::
+
+
+
+## preprocess
+
+
+
+::: {.signature}
+
+defpreprocess(df):
+
+:::
diff --git a/docs/validmind/datasets/classification/taiwan_credit.qmd b/docs/validmind/datasets/classification/taiwan_credit.qmd
new file mode 100644
index 000000000..f94d93b2e
--- /dev/null
+++ b/docs/validmind/datasets/classification/taiwan_credit.qmd
@@ -0,0 +1,31 @@
+---
+title: "[validmind](/validmind/validmind.qmd).taiwan_credit"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## load_data
+
+
+
+::: {.signature}
+
+defload_data():
+
+:::
+
+
+
+## preprocess
+
+
+
+::: {.signature}
+
+defpreprocess(df):
+
+:::
diff --git a/docs/validmind/datasets/credit_risk.qmd b/docs/validmind/datasets/credit_risk.qmd
new file mode 100644
index 000000000..2ca1b4563
--- /dev/null
+++ b/docs/validmind/datasets/credit_risk.qmd
@@ -0,0 +1,14 @@
+---
+title: "[validmind](/validmind/validmind.qmd).credit_risk"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Entrypoint for credit risk datasets.
+
+- [lending_club](credit_risk/lending_club.qmd)
+- [lending_club_bias](credit_risk/lending_club_bias.qmd)
diff --git a/docs/validmind/datasets/credit_risk/lending_club.qmd b/docs/validmind/datasets/credit_risk/lending_club.qmd
new file mode 100644
index 000000000..391d594ab
--- /dev/null
+++ b/docs/validmind/datasets/credit_risk/lending_club.qmd
@@ -0,0 +1,167 @@
+---
+title: "[validmind](/validmind/validmind.qmd).lending_club"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## compute_scores
+
+
+
+::: {.signature}
+
+defcompute_scores(probabilities:np.ndarray) → np.ndarray:
+
+:::
+
+
+
+## feature_engineering
+
+
+
+::: {.signature}
+
+deffeature_engineering(df:pd.DataFrame,verbose:bool=True) → pd.DataFrame:
+
+:::
+
+
+
+## get_demo_test_config
+
+
+
+::: {.signature}
+
+defget_demo_test_config(x_test:Optional\[np.ndarray\]=None,y_test:Optional\[np.ndarray\]=None) → Dict\[str, Any\]:
+
+:::
+
+
+
+Get demo test configuration.
+
+**Arguments**
+
+- `x_test`: Test features DataFrame
+- `y_test`: Test target Series
+
+**Returns**
+
+- Test configuration dictionary
+
+
+
+## init_vm_objects
+
+
+
+::: {.signature}
+
+definit_vm_objects(scorecard):
+
+:::
+
+
+
+## load_data
+
+
+
+::: {.signature}
+
+defload_data(source:str='online',verbose:bool=True) → pd.DataFrame:
+
+:::
+
+
+
+Load data from either an online source or offline files, automatically dropping specified columns for offline data.
+
+**Arguments**
+
+- `source`: 'online' for online data, 'offline' for offline files. Defaults to 'online'.
+
+**Returns**
+
+- DataFrame containing the loaded data.
+
+
+
+## load_scorecard
+
+
+
+::: {.signature}
+
+defload_scorecard():
+
+:::
+
+
+
+## load_test_config
+
+
+
+::: {.signature}
+
+defload_test_config(scorecard):
+
+:::
+
+
+
+## preprocess
+
+
+
+::: {.signature}
+
+defpreprocess(df:pd.DataFrame,verbose:bool=True) → pd.DataFrame:
+
+:::
+
+
+
+## split
+
+
+
+::: {.signature}
+
+defsplit(df:pd.DataFrame,validation_split:Optional\[float\]=None,test_size:float=0.2,add_constant:bool=False,verbose:bool=True) → Tuple\[np.ndarray, np.ndarray, np.ndarray, np.ndarray\]:
+
+:::
+
+
+
+Split dataset into train, validation (optional), and test sets.
+
+**Arguments**
+
+- `df`: Input DataFrame
+- `validation_split`: If None, returns train/test split. If float, returns train/val/test split
+- `test_size`: Proportion of data for test set (default: 0.2)
+- `add_constant`: Whether to add constant column for statsmodels (default: False)
+
+**Returns**
+
+- If validation_size is None: train_df, test_df If validation_size is float: train_df, validation_df, test_df
+
+
+
+## woe_encoding
+
+
+
+::: {.signature}
+
+defwoe_encoding(df:pd.DataFrame,verbose:bool=True) → pd.DataFrame:
+
+:::
diff --git a/docs/validmind/datasets/credit_risk/lending_club_bias.qmd b/docs/validmind/datasets/credit_risk/lending_club_bias.qmd
new file mode 100644
index 000000000..ee010aa95
--- /dev/null
+++ b/docs/validmind/datasets/credit_risk/lending_club_bias.qmd
@@ -0,0 +1,61 @@
+---
+title: "[validmind](/validmind/validmind.qmd).lending_club_bias"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## compute_scores
+
+
+
+::: {.signature}
+
+defcompute_scores(probabilities):
+
+:::
+
+
+
+## load_data
+
+
+
+::: {.signature}
+
+defload_data():
+
+:::
+
+
+
+Load data from the specified CSV file.
+
+:return: DataFrame containing the loaded data.
+
+
+
+## preprocess
+
+
+
+::: {.signature}
+
+defpreprocess(df):
+
+:::
+
+
+
+## split
+
+
+
+::: {.signature}
+
+defsplit(df,test_size=0.3):
+
+:::
diff --git a/docs/validmind/datasets/nlp.qmd b/docs/validmind/datasets/nlp.qmd
new file mode 100644
index 000000000..d0dc65ca8
--- /dev/null
+++ b/docs/validmind/datasets/nlp.qmd
@@ -0,0 +1,14 @@
+---
+title: "[validmind](/validmind/validmind.qmd).nlp"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Example datasets that can be used with the ValidMind Library.
+
+- [cnn_dailymail](nlp/cnn_dailymail.qmd)
+- [twitter_covid_19](nlp/twitter_covid_19.qmd)
diff --git a/docs/validmind/datasets/nlp/cnn_dailymail.qmd b/docs/validmind/datasets/nlp/cnn_dailymail.qmd
new file mode 100644
index 000000000..074c38eff
--- /dev/null
+++ b/docs/validmind/datasets/nlp/cnn_dailymail.qmd
@@ -0,0 +1,48 @@
+---
+title: "[validmind](/validmind/validmind.qmd).cnn_dailymail"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## display_nice
+
+
+
+::: {.signature}
+
+defdisplay_nice(df,num_rows=None):
+
+:::
+
+
+
+Primary function to format and display a DataFrame.
+
+
+
+## load_data
+
+
+
+::: {.signature}
+
+defload_data(source:str='online',dataset_size:Optional\[str\]=None) → Tuple\[pd.DataFrame, pd.DataFrame\]:
+
+:::
+
+
+
+Load data from either online source or offline files.
+
+**Arguments**
+
+- `source`: 'online' for online data, 'offline' for offline data. Defaults to 'online'.
+- `dataset_size`: Applicable if source is 'offline'. '300k' or '500k' for dataset size. Defaults to None.
+
+**Returns**
+
+- Tuple containing (train_df, test_df) DataFrames with the loaded data.
diff --git a/docs/validmind/datasets/nlp/twitter_covid_19.qmd b/docs/validmind/datasets/nlp/twitter_covid_19.qmd
new file mode 100644
index 000000000..076b11c3b
--- /dev/null
+++ b/docs/validmind/datasets/nlp/twitter_covid_19.qmd
@@ -0,0 +1,19 @@
+---
+title: "[validmind](/validmind/validmind.qmd).twitter_covid_19"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## load_data
+
+
+
+::: {.signature}
+
+defload_data(full_dataset=False):
+
+:::
diff --git a/docs/validmind/datasets/regression.qmd b/docs/validmind/datasets/regression.qmd
new file mode 100644
index 000000000..6b0288573
--- /dev/null
+++ b/docs/validmind/datasets/regression.qmd
@@ -0,0 +1,14 @@
+---
+title: "[validmind](/validmind/validmind.qmd).regression"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Entrypoint for regression datasets
+
+- [fred](regression/fred.qmd)
+- [lending_club](regression/lending_club.qmd)
diff --git a/docs/validmind/datasets/regression/fred.qmd b/docs/validmind/datasets/regression/fred.qmd
new file mode 100644
index 000000000..1e0426241
--- /dev/null
+++ b/docs/validmind/datasets/regression/fred.qmd
@@ -0,0 +1,118 @@
+---
+title: "[validmind](/validmind/validmind.qmd).fred"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## load_all_data
+
+
+
+::: {.signature}
+
+defload_all_data():
+
+:::
+
+
+
+## load_data
+
+
+
+::: {.signature}
+
+defload_data():
+
+:::
+
+
+
+## load_model
+
+
+
+::: {.signature}
+
+defload_model(model_name):
+
+:::
+
+
+
+## load_processed_data
+
+
+
+::: {.signature}
+
+defload_processed_data():
+
+:::
+
+
+
+## load_test_dataset
+
+
+
+::: {.signature}
+
+defload_test_dataset(model_name):
+
+:::
+
+
+
+## load_train_dataset
+
+
+
+::: {.signature}
+
+defload_train_dataset(model_path):
+
+:::
+
+
+
+## preprocess
+
+
+
+::: {.signature}
+
+defpreprocess(df,split_option='train_test_val',train_size=0.6,test_size=0.2):
+
+:::
+
+
+
+Split a time series DataFrame into train, validation, and test sets.
+
+**Arguments**
+
+- `df (pandas.DataFrame)`: The time series DataFrame to be split.
+- `split_option (str)`: The split option to choose from: 'train_test_val' (default) or 'train_test'.
+- `train_size (float)`: The proportion of the dataset to include in the training set. Default is 0.6.
+- `test_size (float)`: The proportion of the dataset to include in the test set. Default is 0.2.
+
+**Returns**
+
+- train_df (pandas.DataFrame): The training set. validation_df (pandas.DataFrame): The validation set (only returned if split_option is 'train_test_val'). test_df (pandas.DataFrame): The test set.
+
+
+
+## transform
+
+
+
+::: {.signature}
+
+deftransform(df,transform_func='diff'):
+
+:::
diff --git a/docs/validmind/datasets/regression/lending_club.qmd b/docs/validmind/datasets/regression/lending_club.qmd
new file mode 100644
index 000000000..0aae5ecc4
--- /dev/null
+++ b/docs/validmind/datasets/regression/lending_club.qmd
@@ -0,0 +1,58 @@
+---
+title: "[validmind](/validmind/validmind.qmd).lending_club"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## load_data
+
+
+
+::: {.signature}
+
+defload_data():
+
+:::
+
+
+
+## preprocess
+
+
+
+::: {.signature}
+
+defpreprocess(df,split_option='train_test_val',train_size=0.6,test_size=0.2):
+
+:::
+
+
+
+Split a time series DataFrame into train, validation, and test sets.
+
+**Arguments**
+
+- `df (pandas.DataFrame)`: The time series DataFrame to be split.
+- `split_option (str)`: The split option to choose from: 'train_test_val' (default) or 'train_test'.
+- `train_size (float)`: The proportion of the dataset to include in the training set. Default is 0.6.
+- `test_size (float)`: The proportion of the dataset to include in the test set. Default is 0.2.
+
+**Returns**
+
+- train_df (pandas.DataFrame): The training set. validation_df (pandas.DataFrame): The validation set (only returned if split_option is 'train_test_val'). test_df (pandas.DataFrame): The test set.
+
+
+
+## transform
+
+
+
+::: {.signature}
+
+deftransform(df,transform_func='diff'):
+
+:::
diff --git a/docs/validmind/errors.qmd b/docs/validmind/errors.qmd
new file mode 100644
index 000000000..8754de29c
--- /dev/null
+++ b/docs/validmind/errors.qmd
@@ -0,0 +1,1004 @@
+---
+title: "[validmind](/validmind/validmind.qmd).errors"
+sidebar: validmind-reference
+# errors.qmd.jinja2
+---
+
+
+
+This module contains all the custom errors that are used in the ValidMind Library.
+
+The following base errors are defined for others:
+
+- BaseError
+- APIRequestError
+
+## Base errors
+
+### BaseError
+
+
+
+::: {.signature}
+
+classBaseError(Exception):
+
+:::
+
+
+
+Common base class for all non-exit exceptions.
+
+#### BaseError
+
+
+
+::: {.signature}
+
+BaseError(message='')
+
+:::
+
+#### description
+
+
+
+::: {.signature}
+
+defdescription(self,\*args,\*\*kwargs):
+
+:::
+
+
+
+**Inherited members**
+
+- builtins.BaseException with_traceback, add_note
+
+### APIRequestError
+
+
+
+::: {.signature}
+
+classAPIRequestError(BaseError):
+
+:::
+
+
+
+Generic error for API request errors that are not known.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+## API errors
+
+### InvalidAPICredentialsError
+
+
+
+::: {.signature}
+
+classInvalidAPICredentialsError(APIRequestError):
+
+:::
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+### MissingAPICredentialsError
+
+
+
+::: {.signature}
+
+classMissingAPICredentialsError(BaseError):
+
+:::
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+## Model errors
+
+### InvalidXGBoostTrainedModelError
+
+
+
+::: {.signature}
+
+classInvalidXGBoostTrainedModelError(BaseError):
+
+:::
+
+
+
+When an invalid XGBoost trained model is used when calling init_r_model.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingModelIdError
+
+
+
+::: {.signature}
+
+classMissingModelIdError(BaseError):
+
+:::
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingOrInvalidModelPredictFnError
+
+
+
+::: {.signature}
+
+classMissingOrInvalidModelPredictFnError(BaseError):
+
+:::
+
+
+
+When the PyTorch model is missing a predict function or its predict method does not have the expected arguments.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### UnsupportedModelError
+
+
+
+::: {.signature}
+
+classUnsupportedModelError(BaseError):
+
+:::
+
+
+
+When an unsupported model is used.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### UnsupportedModelForSHAPError
+
+
+
+::: {.signature}
+
+classUnsupportedModelForSHAPError(BaseError):
+
+:::
+
+
+
+When an unsupported model is used for SHAP importance.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### UnsupportedRModelError
+
+
+
+::: {.signature}
+
+classUnsupportedRModelError(BaseError):
+
+:::
+
+
+
+When an unsupported R model is used.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+## Test errors
+
+### GetTestSuiteError
+
+
+
+::: {.signature}
+
+classGetTestSuiteError(BaseError):
+
+:::
+
+
+
+When the test suite could not be found.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### InitializeTestSuiteError
+
+
+
+::: {.signature}
+
+classInitializeTestSuiteError(BaseError):
+
+:::
+
+
+
+When the test suite was found but could not be initialized.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidTestParametersError
+
+
+
+::: {.signature}
+
+classInvalidTestParametersError(BaseError):
+
+:::
+
+
+
+When invalid parameters are provided for the test.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidTestResultsError
+
+
+
+::: {.signature}
+
+classInvalidTestResultsError(APIRequestError):
+
+:::
+
+
+
+When an invalid test results object is sent to the API.
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+### LoadTestError
+
+
+
+::: {.signature}
+
+classLoadTestError(BaseError):
+
+:::
+
+
+
+Exception raised when an error occurs while loading a test.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingRequiredTestInputError
+
+
+
+::: {.signature}
+
+classMissingRequiredTestInputError(BaseError):
+
+:::
+
+
+
+When a required test context variable is missing.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### SkipTestError
+
+
+
+::: {.signature}
+
+classSkipTestError(BaseError):
+
+:::
+
+
+
+Useful error to throw when a test cannot be executed.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### TestInputInvalidDatasetError
+
+
+
+::: {.signature}
+
+classTestInputInvalidDatasetError(BaseError):
+
+:::
+
+
+
+When an invalid dataset is used in a test context.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+## Input validation errors
+
+### InvalidXGBoostTrainedModelError
+
+
+
+::: {.signature}
+
+classInvalidXGBoostTrainedModelError(BaseError):
+
+:::
+
+
+
+When an invalid XGBoost trained model is used when calling init_r_model.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingModelIdError
+
+
+
+::: {.signature}
+
+classMissingModelIdError(BaseError):
+
+:::
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingOrInvalidModelPredictFnError
+
+
+
+::: {.signature}
+
+classMissingOrInvalidModelPredictFnError(BaseError):
+
+:::
+
+
+
+When the PyTorch model is missing a predict function or its predict method does not have the expected arguments.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidTestParametersError
+
+
+
+::: {.signature}
+
+classInvalidTestParametersError(BaseError):
+
+:::
+
+
+
+When invalid parameters are provided for the test.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidTestResultsError
+
+
+
+::: {.signature}
+
+classInvalidTestResultsError(APIRequestError):
+
+:::
+
+
+
+When an invalid test results object is sent to the API.
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+### MissingRequiredTestInputError
+
+
+
+::: {.signature}
+
+classMissingRequiredTestInputError(BaseError):
+
+:::
+
+
+
+When a required test context variable is missing.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidAPICredentialsError
+
+
+
+::: {.signature}
+
+classInvalidAPICredentialsError(APIRequestError):
+
+:::
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidContentIdPrefixError
+
+
+
+::: {.signature}
+
+classInvalidContentIdPrefixError(APIRequestError):
+
+:::
+
+
+
+When an invalid text content_id is sent to the API.
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidInputError
+
+
+
+::: {.signature}
+
+classInvalidInputError(BaseError):
+
+:::
+
+
+
+When an invalid input object is provided.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidMetricResultsError
+
+
+
+::: {.signature}
+
+classInvalidMetricResultsError(APIRequestError):
+
+:::
+
+
+
+When an invalid metric results object is sent to the API.
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidParameterError
+
+
+
+::: {.signature}
+
+classInvalidParameterError(BaseError):
+
+:::
+
+
+
+When an invalid parameter is provided.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidProjectError
+
+
+
+::: {.signature}
+
+classInvalidProjectError(APIRequestError):
+
+:::
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidRequestBodyError
+
+
+
+::: {.signature}
+
+classInvalidRequestBodyError(APIRequestError):
+
+:::
+
+
+
+When a POST/PUT request is made with an invalid request body.
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidTextObjectError
+
+
+
+::: {.signature}
+
+classInvalidTextObjectError(APIRequestError):
+
+:::
+
+
+
+When an invalid Metadata (Text) object is sent to the API.
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+### InvalidValueFormatterError
+
+
+
+::: {.signature}
+
+classInvalidValueFormatterError(BaseError):
+
+:::
+
+
+
+When an invalid value formatter is provided when serializing results.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingAPICredentialsError
+
+
+
+::: {.signature}
+
+classMissingAPICredentialsError(BaseError):
+
+:::
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingCacheResultsArgumentsError
+
+
+
+::: {.signature}
+
+classMissingCacheResultsArgumentsError(BaseError):
+
+:::
+
+
+
+When the cache_results function is missing arguments.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingClassLabelError
+
+
+
+::: {.signature}
+
+classMissingClassLabelError(BaseError):
+
+:::
+
+
+
+When the one or more class labels are missing from provided dataset targets.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingDependencyError
+
+
+
+::: {.signature}
+
+classMissingDependencyError(BaseError):
+
+:::
+
+
+
+When a required dependency is missing.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingDocumentationTemplate
+
+
+
+::: {.signature}
+
+classMissingDocumentationTemplate(BaseError):
+
+:::
+
+
+
+When the client config is missing the documentation template.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingRExtrasError
+
+
+
+::: {.signature}
+
+classMissingRExtrasError(BaseError):
+
+:::
+
+
+
+When the R extras have not been installed.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### MissingTextContentIdError
+
+
+
+::: {.signature}
+
+classMissingTextContentIdError(APIRequestError):
+
+:::
+
+
+
+When a Text object is sent to the API without a content_id.
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+### MissingTextContentsError
+
+
+
+::: {.signature}
+
+classMissingTextContentsError(APIRequestError):
+
+:::
+
+
+
+When a Text object is sent to the API without a "text" attribute.
+
+
+
+**Inherited members**
+
+- [APIRequestError](#apirequesterror)
+- builtins.BaseException with_traceback, add_note
+
+## Unsupported feature errors
+
+### UnsupportedModelError
+
+
+
+::: {.signature}
+
+classUnsupportedModelError(BaseError):
+
+:::
+
+
+
+When an unsupported model is used.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### UnsupportedModelForSHAPError
+
+
+
+::: {.signature}
+
+classUnsupportedModelForSHAPError(BaseError):
+
+:::
+
+
+
+When an unsupported model is used for SHAP importance.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### UnsupportedRModelError
+
+
+
+::: {.signature}
+
+classUnsupportedRModelError(BaseError):
+
+:::
+
+
+
+When an unsupported R model is used.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### UnsupportedColumnTypeError
+
+
+
+::: {.signature}
+
+classUnsupportedColumnTypeError(BaseError):
+
+:::
+
+
+
+When an unsupported column type is found on a dataset.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### UnsupportedDatasetError
+
+
+
+::: {.signature}
+
+classUnsupportedDatasetError(BaseError):
+
+:::
+
+
+
+When an unsupported dataset is used.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
+
+### UnsupportedFigureError
+
+
+
+::: {.signature}
+
+classUnsupportedFigureError(BaseError):
+
+:::
+
+
+
+When an unsupported figure object is constructed.
+
+
+
+**Inherited members**
+
+- [BaseError](#baseerror), [description](#description)
+- builtins.BaseException with_traceback, add_note
diff --git a/docs/validmind/test_suites.qmd b/docs/validmind/test_suites.qmd
new file mode 100644
index 000000000..296174a0f
--- /dev/null
+++ b/docs/validmind/test_suites.qmd
@@ -0,0 +1,101 @@
+---
+title: "[validmind](/validmind/validmind.qmd).test_suites"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Entrypoint for test suites.
+
+- [classifier](test_suites/classifier.qmd)
+- [cluster](test_suites/cluster.qmd)
+- [embeddings](test_suites/embeddings.qmd)
+- [llm](test_suites/llm.qmd)
+- [nlp](test_suites/nlp.qmd)
+- [parameters_optimization](test_suites/parameters_optimization.qmd)
+- [regression](test_suites/regression.qmd)
+- [statsmodels_timeseries](test_suites/statsmodels_timeseries.qmd)
+- [summarization](test_suites/summarization.qmd)
+- [tabular_datasets](test_suites/tabular_datasets.qmd)
+- [text_data](test_suites/text_data.qmd)
+- [time_series](test_suites/time_series.qmd)
+
+## describe_test_suite
+
+*This function is an alias for [describe_suite](#describe_suite).*
+
+
+
+## describe_suite
+
+
+
+::: {.signature}
+
+defdescribe_suite(test_suite_id:str,verbose:bool=False) → pd.DataFrame:
+
+:::
+
+
+
+Describes a Test Suite by ID
+
+**Arguments**
+
+- `test_suite_id`: Test Suite ID
+- `verbose`: If True, describe all plans and tests in the Test Suite
+
+**Returns**
+
+- A formatted table with the Test Suite description
+
+
+
+## get_by_id
+
+
+
+::: {.signature}
+
+defget_by_id(test_suite_id:str):
+
+:::
+
+
+
+Returns the test suite by ID
+
+
+
+## list_suites
+
+
+
+::: {.signature}
+
+deflist_suites(pretty:bool=True):
+
+:::
+
+
+
+Returns a list of all available test suites
+
+
+
+## register_test_suite
+
+
+
+::: {.signature}
+
+defregister_test_suite(suite_id:str,suite:validmind.vm_models.TestSuite):
+
+:::
+
+
+
+Registers a custom test suite
diff --git a/docs/validmind/test_suites/classifier.qmd b/docs/validmind/test_suites/classifier.qmd
new file mode 100644
index 000000000..9d5cfabf9
--- /dev/null
+++ b/docs/validmind/test_suites/classifier.qmd
@@ -0,0 +1,93 @@
+---
+title: "[validmind](/validmind/validmind.qmd).classifier"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Test suites for sklearn-compatible classifier models
+
+Ideal setup is to have the API client to read a custom test suite from the project's configuration
+
+
+
+## ClassifierDiagnosis
+
+
+
+::: {.signature}
+
+classClassifierDiagnosis(TestSuite):
+
+:::
+
+
+
+Test suite for sklearn classifier model diagnosis tests
+
+
+
+## ClassifierFullSuite
+
+
+
+::: {.signature}
+
+classClassifierFullSuite(TestSuite):
+
+:::
+
+
+
+Full test suite for binary classification models.
+
+
+
+## ClassifierMetrics
+
+
+
+::: {.signature}
+
+classClassifierMetrics(TestSuite):
+
+:::
+
+
+
+Test suite for sklearn classifier metrics
+
+
+
+## ClassifierModelValidation
+
+
+
+::: {.signature}
+
+classClassifierModelValidation(TestSuite):
+
+:::
+
+
+
+Test suite for binary classification models.
+
+
+
+## ClassifierPerformance
+
+
+
+::: {.signature}
+
+classClassifierPerformance(TestSuite):
+
+:::
+
+
+
+Test suite for sklearn classifier models
diff --git a/docs/validmind/test_suites/cluster.qmd b/docs/validmind/test_suites/cluster.qmd
new file mode 100644
index 000000000..b1c6a4ce9
--- /dev/null
+++ b/docs/validmind/test_suites/cluster.qmd
@@ -0,0 +1,61 @@
+---
+title: "[validmind](/validmind/validmind.qmd).cluster"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Test suites for sklearn-compatible clustering models
+
+Ideal setup is to have the API client to read a custom test suite from the project's configuration
+
+
+
+## ClusterFullSuite
+
+
+
+::: {.signature}
+
+classClusterFullSuite(TestSuite):
+
+:::
+
+
+
+Full test suite for clustering models.
+
+
+
+## ClusterMetrics
+
+
+
+::: {.signature}
+
+classClusterMetrics(TestSuite):
+
+:::
+
+
+
+Test suite for sklearn clustering metrics
+
+
+
+## ClusterPerformance
+
+
+
+::: {.signature}
+
+classClusterPerformance(TestSuite):
+
+:::
+
+
+
+Test suite for sklearn cluster performance
diff --git a/docs/validmind/test_suites/embeddings.qmd b/docs/validmind/test_suites/embeddings.qmd
new file mode 100644
index 000000000..e724651d3
--- /dev/null
+++ b/docs/validmind/test_suites/embeddings.qmd
@@ -0,0 +1,61 @@
+---
+title: "[validmind](/validmind/validmind.qmd).embeddings"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Test suites for embeddings models
+
+Ideal setup is to have the API client to read a custom test suite from the project's configuration
+
+
+
+## EmbeddingsFullSuite
+
+
+
+::: {.signature}
+
+classEmbeddingsFullSuite(TestSuite):
+
+:::
+
+
+
+Full test suite for embeddings models.
+
+
+
+## EmbeddingsMetrics
+
+
+
+::: {.signature}
+
+classEmbeddingsMetrics(TestSuite):
+
+:::
+
+
+
+Test suite for embeddings metrics
+
+
+
+## EmbeddingsPerformance
+
+
+
+::: {.signature}
+
+classEmbeddingsPerformance(TestSuite):
+
+:::
+
+
+
+Test suite for embeddings model performance
diff --git a/docs/validmind/test_suites/llm.qmd b/docs/validmind/test_suites/llm.qmd
new file mode 100644
index 000000000..87587c207
--- /dev/null
+++ b/docs/validmind/test_suites/llm.qmd
@@ -0,0 +1,43 @@
+---
+title: "[validmind](/validmind/validmind.qmd).llm"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Test suites for LLMs
+
+
+
+## LLMClassifierFullSuite
+
+
+
+::: {.signature}
+
+classLLMClassifierFullSuite(TestSuite):
+
+:::
+
+
+
+Full test suite for LLM classification models.
+
+
+
+## PromptValidation
+
+
+
+::: {.signature}
+
+classPromptValidation(TestSuite):
+
+:::
+
+
+
+Test suite for prompt validation
diff --git a/docs/validmind/test_suites/nlp.qmd b/docs/validmind/test_suites/nlp.qmd
new file mode 100644
index 000000000..c9c3a17ac
--- /dev/null
+++ b/docs/validmind/test_suites/nlp.qmd
@@ -0,0 +1,27 @@
+---
+title: "[validmind](/validmind/validmind.qmd).nlp"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Test suites for NLP models
+
+
+
+## NLPClassifierFullSuite
+
+
+
+::: {.signature}
+
+classNLPClassifierFullSuite(TestSuite):
+
+:::
+
+
+
+Full test suite for NLP classification models.
diff --git a/docs/validmind/test_suites/parameters_optimization.qmd b/docs/validmind/test_suites/parameters_optimization.qmd
new file mode 100644
index 000000000..b93d2bc71
--- /dev/null
+++ b/docs/validmind/test_suites/parameters_optimization.qmd
@@ -0,0 +1,29 @@
+---
+title: "[validmind](/validmind/validmind.qmd).parameters_optimization"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Test suites for sklearn-compatible hyper parameters tunning
+
+Ideal setup is to have the API client to read a custom test suite from the project's configuration
+
+
+
+## KmeansParametersOptimization
+
+
+
+::: {.signature}
+
+classKmeansParametersOptimization(TestSuite):
+
+:::
+
+
+
+Test suite for sklearn hyperparameters optimization
diff --git a/docs/validmind/test_suites/regression.qmd b/docs/validmind/test_suites/regression.qmd
new file mode 100644
index 000000000..b19fa9563
--- /dev/null
+++ b/docs/validmind/test_suites/regression.qmd
@@ -0,0 +1,55 @@
+---
+title: "[validmind](/validmind/validmind.qmd).regression"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionFullSuite
+
+
+
+::: {.signature}
+
+classRegressionFullSuite(TestSuite):
+
+:::
+
+
+
+Full test suite for regression models.
+
+
+
+## RegressionMetrics
+
+
+
+::: {.signature}
+
+classRegressionMetrics(TestSuite):
+
+:::
+
+
+
+Test suite for performance metrics of regression metrics
+
+
+
+## RegressionPerformance
+
+
+
+::: {.signature}
+
+classRegressionPerformance(TestSuite):
+
+:::
+
+
+
+Test suite for regression model performance
diff --git a/docs/validmind/test_suites/statsmodels_timeseries.qmd b/docs/validmind/test_suites/statsmodels_timeseries.qmd
new file mode 100644
index 000000000..bcfb3fb2a
--- /dev/null
+++ b/docs/validmind/test_suites/statsmodels_timeseries.qmd
@@ -0,0 +1,43 @@
+---
+title: "[validmind](/validmind/validmind.qmd).statsmodels_timeseries"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Time Series Test Suites from statsmodels
+
+
+
+## RegressionModelDescription
+
+
+
+::: {.signature}
+
+classRegressionModelDescription(TestSuite):
+
+:::
+
+
+
+Test suite for performance metric of regression model of statsmodels library
+
+
+
+## RegressionModelsEvaluation
+
+
+
+::: {.signature}
+
+classRegressionModelsEvaluation(TestSuite):
+
+:::
+
+
+
+Test suite for metrics comparison of regression model of statsmodels library
diff --git a/docs/validmind/test_suites/summarization.qmd b/docs/validmind/test_suites/summarization.qmd
new file mode 100644
index 000000000..3af91eb8d
--- /dev/null
+++ b/docs/validmind/test_suites/summarization.qmd
@@ -0,0 +1,27 @@
+---
+title: "[validmind](/validmind/validmind.qmd).summarization"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Test suites for llm summarization models
+
+
+
+## SummarizationMetrics
+
+
+
+::: {.signature}
+
+classSummarizationMetrics(TestSuite):
+
+:::
+
+
+
+Test suite for Summarization metrics
diff --git a/docs/validmind/test_suites/tabular_datasets.qmd b/docs/validmind/test_suites/tabular_datasets.qmd
new file mode 100644
index 000000000..5901d7c3c
--- /dev/null
+++ b/docs/validmind/test_suites/tabular_datasets.qmd
@@ -0,0 +1,59 @@
+---
+title: "[validmind](/validmind/validmind.qmd).tabular_datasets"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Test suites for tabular datasets
+
+
+
+## TabularDataQuality
+
+
+
+::: {.signature}
+
+classTabularDataQuality(TestSuite):
+
+:::
+
+
+
+Test suite for data quality on tabular datasets
+
+
+
+## TabularDataset
+
+
+
+::: {.signature}
+
+classTabularDataset(TestSuite):
+
+:::
+
+
+
+Test suite for tabular datasets.
+
+
+
+## TabularDatasetDescription
+
+
+
+::: {.signature}
+
+classTabularDatasetDescription(TestSuite):
+
+:::
+
+
+
+Test suite to extract metadata and descriptive statistics from a tabular dataset
diff --git a/docs/validmind/test_suites/text_data.qmd b/docs/validmind/test_suites/text_data.qmd
new file mode 100644
index 000000000..60594ad6e
--- /dev/null
+++ b/docs/validmind/test_suites/text_data.qmd
@@ -0,0 +1,27 @@
+---
+title: "[validmind](/validmind/validmind.qmd).text_data"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Test suites for text datasets
+
+
+
+## TextDataQuality
+
+
+
+::: {.signature}
+
+classTextDataQuality(TestSuite):
+
+:::
+
+
+
+Test suite for data quality on text data
diff --git a/docs/validmind/test_suites/time_series.qmd b/docs/validmind/test_suites/time_series.qmd
new file mode 100644
index 000000000..b4cd65c7c
--- /dev/null
+++ b/docs/validmind/test_suites/time_series.qmd
@@ -0,0 +1,93 @@
+---
+title: "[validmind](/validmind/validmind.qmd).time_series"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Time Series Test Suites
+
+
+
+## TimeSeriesDataQuality
+
+
+
+::: {.signature}
+
+classTimeSeriesDataQuality(TestSuite):
+
+:::
+
+
+
+Test suite for data quality on time series datasets
+
+
+
+## TimeSeriesDataset
+
+
+
+::: {.signature}
+
+classTimeSeriesDataset(TestSuite):
+
+:::
+
+
+
+Test suite for time series datasets.
+
+
+
+## TimeSeriesModelValidation
+
+
+
+::: {.signature}
+
+classTimeSeriesModelValidation(TestSuite):
+
+:::
+
+
+
+Test suite for time series model validation.
+
+
+
+## TimeSeriesMultivariate
+
+
+
+::: {.signature}
+
+classTimeSeriesMultivariate(TestSuite):
+
+:::
+
+
+
+This test suite provides a preliminary understanding of the features and relationship in multivariate dataset. It presents various multivariate visualizations that can help identify patterns, trends, and relationships between pairs of variables. The visualizations are designed to explore the relationships between multiple features simultaneously. They allow you to quickly identify any patterns or trends in the data, as well as any potential outliers or anomalies. The individual feature distribution can also be explored to provide insight into the range and frequency of values observed in the data. This multivariate analysis test suite aims to provide an overview of the data structure and guide further exploration and modeling.
+
+
+
+## TimeSeriesUnivariate
+
+
+
+::: {.signature}
+
+classTimeSeriesUnivariate(TestSuite):
+
+:::
+
+
+
+This test suite provides a preliminary understanding of the target variable(s) used in the time series dataset. It visualizations that present the raw time series data and a histogram of the target variable(s).
+
+The raw time series data provides a visual inspection of the target variable's behavior over time. This helps to identify any patterns or trends in the data, as well as any potential outliers or anomalies. The histogram of the target variable displays the distribution of values, providing insight into the range and frequency of values observed in the data.
diff --git a/docs/validmind/tests.qmd b/docs/validmind/tests.qmd
new file mode 100644
index 000000000..31cb60882
--- /dev/null
+++ b/docs/validmind/tests.qmd
@@ -0,0 +1,459 @@
+---
+title: "[validmind](/validmind/validmind.qmd).tests"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+ValidMind Tests Module
+
+- [data_validation](tests/data_validation.qmd)
+- [model_validation](tests/model_validation.qmd)
+- [prompt_validation](tests/prompt_validation.qmd)
+
+## list_tests
+
+
+
+::: {.signature}
+
+deflist_tests(filter:Optional\[str\]=None,task:Optional\[str\]=None,tags:Optional\[List\[str\]\]=None,pretty:bool=True,truncate:bool=True) → Union\[List\[str\], None\]:
+
+:::
+
+
+
+List all tests in the tests directory.
+
+**Arguments**
+
+- `filter (str, optional)`: Find tests where the ID, tasks or tags match the filter string. Defaults to None.
+- `task (str, optional)`: Find tests that match the task. Can be used to narrow down matches from the filter string. Defaults to None.
+- `tags (list, optional)`: Find tests that match list of tags. Can be used to narrow down matches from the filter string. Defaults to None.
+- `pretty (bool, optional)`: If True, returns a pandas DataFrame with a formatted table. Defaults to True.
+- `truncate (bool, optional)`: If True, truncates the test description to the first line. Defaults to True. (only used if pretty=True)
+
+## load_test
+
+
+
+::: {.signature}
+
+defload_test(test_id:str,test_func:Optional\[Callable\[..., Any\]\]=None,reload:bool=False) → Callable\[..., Any\]:
+
+:::
+
+
+
+Load a test by test ID
+
+Test IDs are in the format `namespace.path_to_module.TestClassOrFuncName[:tag]`. The tag is optional and is used to distinguish between multiple results from the same test.
+
+**Arguments**
+
+- `test_id (str)`: The test ID in the format `namespace.path_to_module.TestName[:tag]`
+- `test_func (callable, optional)`: The test function to load. If not provided, the test will be loaded from the test provider. Defaults to None.
+- `reload (bool, optional)`: If True, reload the test even if it's already loaded. Defaults to False.
+
+## describe_test
+
+
+
+::: {.signature}
+
+defdescribe_test(test_id:Optional\[TestID (Union of validmind.data_validation.\*, validmind.model_validation.\*, validmind.prompt_validation.\* and str)\]=None,raw:bool=False,show:bool=True) → Union\[str, HTML, Dict\[str, Any\]\]:
+
+:::
+
+
+
+Get or show details about the test
+
+This function can be used to see test details including the test name, description, required inputs and default params. It can also be used to get a dictionary of the above information for programmatic use.
+
+**Arguments**
+
+- `test_id (str, optional)`: The test ID. Defaults to None.
+- `raw (bool, optional)`: If True, returns a dictionary with the test details. Defaults to False.
+
+## run_test
+
+
+
+::: {.signature}
+
+defrun_test(test_id:Union\[TestID (Union of validmind.data_validation.\*, validmind.model_validation.\*, validmind.prompt_validation.\* and str), None\]=None,name:Union\[str, None\]=None,unit_metrics:Union\[List\[TestID (Unit metrics from validmind.unit_metrics.\*)\], None\]=None,inputs:Union\[Dict\[str, Any\], None\]=None,input_grid:Union\[Dict\[str, List\[Any\]\], List\[Dict\[str, Any\]\], None\]=None,params:Union\[Dict\[str, Any\], None\]=None,param_grid:Union\[Dict\[str, List\[Any\]\], List\[Dict\[str, Any\]\], None\]=None,show:bool=True,generate_description:bool=True,title:Optional\[str\]=None,post_process_fn:Union\[Callable\[\[validmind.vm_models.TestResult\], None\], None\]=None,show_params:bool=True,\*\*kwargs) → validmind.vm_models.TestResult:
+
+:::
+
+
+
+Run a ValidMind or custom test
+
+This function is the main entry point for running tests. It can run simple unit metrics, ValidMind and custom tests, composite tests made up of multiple unit metrics and comparison tests made up of multiple tests.
+
+**Arguments**
+
+- `test_id (TestID)`: Test ID to run. Not required if `name` and `unit_metrics` provided.
+- `params (dict)`: Parameters to customize test behavior. See test details for available parameters.
+- `param_grid (Union[Dict[str, List[Any]], List[Dict[str, Any]]])`: For comparison tests, either:
+- Dict mapping parameter names to lists of values (creates Cartesian product)
+- List of parameter dictionaries to test
+- `inputs (Dict[str, Any])`: Test inputs (models/datasets initialized with vm.init_model/dataset)
+- `input_grid (Union[Dict[str, List[Any]], List[Dict[str, Any]]])`: For comparison tests, either:
+- Dict mapping input names to lists of values (creates Cartesian product)
+- List of input dictionaries to test
+- `name (str)`: Test name (required for composite metrics)
+- `unit_metrics (list)`: Unit metric IDs to run as composite metric
+- `show (bool, optional)`: Whether to display results. Defaults to True.
+- `generate_description (bool, optional)`: Whether to generate a description. Defaults to True.
+- `title (str)`: Custom title for the test result
+- `post_process_fn (Callable[[TestResult], None])`: Function to post-process the test result
+- `show_params (bool, optional)`: Whether to include parameter values in figure titles for comparison tests. Defaults to True.
+
+**Returns**
+
+- A TestResult object containing the test results
+
+**Raises**
+
+- `ValueError`: If the test inputs are invalid
+- `LoadTestError`: If the test class fails to load
+
+## list_tags
+
+
+
+::: {.signature}
+
+deflist_tags() → List\[str\]:
+
+:::
+
+
+
+List all unique available tags
+
+## list_tasks
+
+
+
+::: {.signature}
+
+deflist_tasks() → List\[str\]:
+
+:::
+
+
+
+List all unique available tasks
+
+## list_tasks_and_tags
+
+
+
+::: {.signature}
+
+deflist_tasks_and_tags(as_json:bool=False) → Union\[str, Dict\[str, List\[str\]\]\]:
+
+:::
+
+
+
+List all task types and their associated tags, with one row per task type and all tags for a task type in one row.
+
+**Returns**
+
+- A DataFrame with 'Task Type' and concatenated 'Tags'.
+
+## test
+
+
+
+::: {.signature}
+
+deftest(func_or_id:Union\[Callable\[..., Any\], str, None\]):
+
+:::
+
+
+
+Decorator for creating and registering custom tests
+
+This decorator registers the function it wraps as a test function within ValidMind under the provided ID. Once decorated, the function can be run using the `validmind.tests.run_test` function.
+
+The function can take two different types of arguments:
+
+- Inputs: ValidMind model or dataset (or list of models/datasets). These arguments must use the following names: `model`, `models`, `dataset`, `datasets`.
+- Parameters: Any additional keyword arguments of any type (must have a default value) that can have any name.
+
+The function should return one of the following types:
+
+- Table: Either a list of dictionaries or a pandas DataFrame
+- Plot: Either a matplotlib figure or a plotly figure
+- Scalar: A single number (int or float)
+- Boolean: A single boolean value indicating whether the test passed or failed
+
+The function may also include a docstring. This docstring will be used and logged as the metric's description.
+
+**Arguments**
+
+- `func_or_id (Union[Callable[..., Any], str, None])`: Either the function to decorate or the test ID. If None, the function name is used.
+
+**Returns**
+
+- The decorated function.
+
+## tags
+
+
+
+::: {.signature}
+
+deftags(\*tags:str):
+
+:::
+
+
+
+Decorator for specifying tags for a test.
+
+**Arguments**
+
+- `*tags`: The tags to apply to the test.
+
+## tasks
+
+
+
+::: {.signature}
+
+deftasks(\*tasks:str):
+
+:::
+
+
+
+Decorator for specifying the task types that a test is designed for.
+
+**Arguments**
+
+- `*tasks`: The task types that the test is designed for.
+
+
+
+## register_test_provider
+
+
+
+::: {.signature}
+
+defregister_test_provider(namespace:str,test_provider:validmind.vm_models.TestProvider):
+
+:::
+
+
+
+Register an external test provider
+
+**Arguments**
+
+- `namespace (str)`: The namespace of the test provider
+- `test_provider (TestProvider)`: The test provider
+
+
+
+## LoadTestError
+
+
+
+::: {.signature}
+
+classLoadTestError(BaseError):
+
+:::
+
+
+
+Exception raised when an error occurs while loading a test.
+
+**Inherited members**
+
+- **From BaseError**: [class BaseError](#baseerror), [description](#description)
+- **From builtins.BaseException**: with_traceback, add_note
+
+### LoadTestError
+
+
+
+::: {.signature}
+
+LoadTestError(message:str,original_error:Optional\[validmind.vm_models.Exception\]=None)
+
+:::
+
+
+
+## LocalTestProvider
+
+
+
+::: {.signature}
+
+classLocalTestProvider:
+
+:::
+
+
+
+Test providers in ValidMind are responsible for loading tests from different sources, such as local files, databases, or remote services. The LocalTestProvider specifically loads tests from the local file system.
+
+To use the LocalTestProvider, you need to provide the root_folder, which is the root directory for local tests. The test_id is a combination of the namespace (set when registering the test provider) and the path to the test class module, where slashes are replaced by dots and the .py extension is left out.
+
+Example usage:
+
+```
+# Create an instance of LocalTestProvider with the root folder
+test_provider = LocalTestProvider("/path/to/tests/folder")
+
+# Register the test provider with a namespace
+register_test_provider("my_namespace", test_provider)
+
+# List all tests in the namespace (returns a list of test IDs)
+test_provider.list_tests()
+# this is used by the validmind.tests.list_tests() function to aggregate all tests
+# from all test providers
+
+# Load a test using the test_id (namespace + path to test class module)
+test = test_provider.load_test("my_namespace.my_test_class")
+# full path to the test class module is /path/to/tests/folder/my_test_class.py
+```
+
+**Arguments**
+
+- `root_folder (str)`: The root directory for local tests.
+
+### LocalTestProvider
+
+
+
+::: {.signature}
+
+LocalTestProvider(root_folder:str)
+
+:::
+
+
+
+Initialize the LocalTestProvider with the given root_folder (see class docstring for details)
+
+**Arguments**
+
+- `root_folder (str)`: The root directory for local tests.
+
+### list_tests
+
+
+
+::: {.signature}
+
+deflist_tests(self) → List\[str\]:
+
+:::
+
+
+
+List all tests in the given namespace
+
+**Returns**
+
+- A list of test IDs
+
+### load_test
+
+
+
+::: {.signature}
+
+defload_test(self,test_id:str) → Callable\[..., Any\]:
+
+:::
+
+
+
+Load the test function identified by the given test_id
+
+**Arguments**
+
+- `test_id (str)`: The test ID (does not contain the namespace under which the test is registered)
+
+**Returns**
+
+- The test function
+
+**Raises**
+
+- `FileNotFoundError`: If the test is not found
+
+
+
+## TestProvider
+
+
+
+::: {.signature}
+
+classTestProvider(Protocol):
+
+:::
+
+
+
+Protocol for user-defined test providers
+
+### list_tests
+
+
+
+::: {.signature}
+
+deflist_tests(self) → List\[str\]:
+
+:::
+
+
+
+List all tests in the given namespace
+
+**Returns**
+
+- A list of test IDs
+
+### load_test
+
+
+
+::: {.signature}
+
+defload_test(self,test_id:str) → callable:
+
+:::
+
+
+
+Load the test function identified by the given test_id
+
+**Arguments**
+
+- `test_id (str)`: The test ID (does not contain the namespace under which the test is registered)
+
+**Returns**
+
+- The test function
+
+**Raises**
+
+- `FileNotFoundError`: If the test is not found
diff --git a/docs/validmind/tests/data_validation.qmd b/docs/validmind/tests/data_validation.qmd
new file mode 100644
index 000000000..88cc8bde8
--- /dev/null
+++ b/docs/validmind/tests/data_validation.qmd
@@ -0,0 +1,68 @@
+---
+title: "[validmind](/validmind/validmind.qmd).data_validation"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+- [ACFandPACFPlot](data_validation/ACFandPACFPlot.qmd)
+- [ADF](data_validation/ADF.qmd)
+- [AutoAR](data_validation/AutoAR.qmd)
+- [AutoMA](data_validation/AutoMA.qmd)
+- [AutoStationarity](data_validation/AutoStationarity.qmd)
+- [BivariateScatterPlots](data_validation/BivariateScatterPlots.qmd)
+- [BoxPierce](data_validation/BoxPierce.qmd)
+- [ChiSquaredFeaturesTable](data_validation/ChiSquaredFeaturesTable.qmd)
+- [ClassImbalance](data_validation/ClassImbalance.qmd)
+- [DatasetDescription](data_validation/DatasetDescription.qmd)
+- [DatasetSplit](data_validation/DatasetSplit.qmd)
+- [DescriptiveStatistics](data_validation/DescriptiveStatistics.qmd)
+- [DickeyFullerGLS](data_validation/DickeyFullerGLS.qmd)
+- [Duplicates](data_validation/Duplicates.qmd)
+- [EngleGrangerCoint](data_validation/EngleGrangerCoint.qmd)
+- [FeatureTargetCorrelationPlot](data_validation/FeatureTargetCorrelationPlot.qmd)
+- [HighCardinality](data_validation/HighCardinality.qmd)
+- [HighPearsonCorrelation](data_validation/HighPearsonCorrelation.qmd)
+- [IQROutliersBarPlot](data_validation/IQROutliersBarPlot.qmd)
+- [IQROutliersTable](data_validation/IQROutliersTable.qmd)
+- [IsolationForestOutliers](data_validation/IsolationForestOutliers.qmd)
+- [JarqueBera](data_validation/JarqueBera.qmd)
+- [KPSS](data_validation/KPSS.qmd)
+- [LaggedCorrelationHeatmap](data_validation/LaggedCorrelationHeatmap.qmd)
+- [LJungBox](data_validation/LJungBox.qmd)
+- [MissingValues](data_validation/MissingValues.qmd)
+- [MissingValuesBarPlot](data_validation/MissingValuesBarPlot.qmd)
+- [MutualInformation](data_validation/MutualInformation.qmd)
+- [nlp](data_validation/nlp.qmd)
+- [PearsonCorrelationMatrix](data_validation/PearsonCorrelationMatrix.qmd)
+- [PhillipsPerronArch](data_validation/PhillipsPerronArch.qmd)
+- [ProtectedClassesCombination](data_validation/ProtectedClassesCombination.qmd)
+- [ProtectedClassesDescription](data_validation/ProtectedClassesDescription.qmd)
+- [ProtectedClassesDisparity](data_validation/ProtectedClassesDisparity.qmd)
+- [ProtectedClassesThresholdOptimizer](data_validation/ProtectedClassesThresholdOptimizer.qmd)
+- [RollingStatsPlot](data_validation/RollingStatsPlot.qmd)
+- [RunsTest](data_validation/RunsTest.qmd)
+- [ScatterPlot](data_validation/ScatterPlot.qmd)
+- [ScoreBandDefaultRates](data_validation/ScoreBandDefaultRates.qmd)
+- [SeasonalDecompose](data_validation/SeasonalDecompose.qmd)
+- [ShapiroWilk](data_validation/ShapiroWilk.qmd)
+- [Skewness](data_validation/Skewness.qmd)
+- [SpreadPlot](data_validation/SpreadPlot.qmd)
+- [TabularCategoricalBarPlots](data_validation/TabularCategoricalBarPlots.qmd)
+- [TabularDateTimeHistograms](data_validation/TabularDateTimeHistograms.qmd)
+- [TabularDescriptionTables](data_validation/TabularDescriptionTables.qmd)
+- [TabularNumericalHistograms](data_validation/TabularNumericalHistograms.qmd)
+- [TargetRateBarPlots](data_validation/TargetRateBarPlots.qmd)
+- [TimeSeriesDescription](data_validation/TimeSeriesDescription.qmd)
+- [TimeSeriesDescriptiveStatistics](data_validation/TimeSeriesDescriptiveStatistics.qmd)
+- [TimeSeriesFrequency](data_validation/TimeSeriesFrequency.qmd)
+- [TimeSeriesHistogram](data_validation/TimeSeriesHistogram.qmd)
+- [TimeSeriesLinePlot](data_validation/TimeSeriesLinePlot.qmd)
+- [TimeSeriesMissingValues](data_validation/TimeSeriesMissingValues.qmd)
+- [TimeSeriesOutliers](data_validation/TimeSeriesOutliers.qmd)
+- [TooManyZeroValues](data_validation/TooManyZeroValues.qmd)
+- [UniqueRows](data_validation/UniqueRows.qmd)
+- [WOEBinPlots](data_validation/WOEBinPlots.qmd)
+- [WOEBinTable](data_validation/WOEBinTable.qmd)
+- [ZivotAndrewsArch](data_validation/ZivotAndrewsArch.qmd)
diff --git a/docs/validmind/tests/data_validation/ACFandPACFPlot.qmd b/docs/validmind/tests/data_validation/ACFandPACFPlot.qmd
new file mode 100644
index 000000000..e0a4387a6
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ACFandPACFPlot.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ACFandPACFPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ACFandPACFPlot
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'forecasting', 'statistical_test', 'visualization')
+
+@tasks('regression')
+
+defACFandPACFPlot(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Analyzes time series data using Autocorrelation Function (ACF) and Partial Autocorrelation Function (PACF) plots to reveal trends and correlations.
+
+### Purpose
+
+The ACF (Autocorrelation Function) and PACF (Partial Autocorrelation Function) plot test is employed to analyze time series data in machine learning models. It illuminates the correlation of the data over time by plotting the correlation of the series with its own lags (ACF), and the correlations after removing effects already accounted for by earlier lags (PACF). This information can identify trends, such as seasonality, degrees of autocorrelation, and inform the selection of order parameters for AutoRegressive Integrated Moving Average (ARIMA) models.
+
+### Test Mechanism
+
+The `ACFandPACFPlot` test accepts a dataset with a time-based index. It first confirms the index is of a datetime type, then handles any NaN values. The test subsequently generates ACF and PACF plots for each column in the dataset, producing a subplot for each. If the dataset doesn't include key columns, an error is returned.
+
+### Signs of High Risk
+
+- Sudden drops in the correlation at a specific lag might signal a model at high risk.
+- Consistent high correlation across multiple lags could also indicate non-stationarity in the data, which may suggest that a model estimated on this data won't generalize well to future, unknown data.
+
+### Strengths
+
+- ACF and PACF plots offer clear graphical representations of the correlations in time series data.
+- These plots are effective at revealing important data characteristics such as seasonality, trends, and correlation patterns.
+- The insights from these plots aid in better model configuration, particularly in the selection of ARIMA model parameters.
+
+### Limitations
+
+- ACF and PACF plots are exclusively for time series data and hence, can't be applied to all ML models.
+- These plots require large, consistent datasets as gaps could lead to misleading results.
+- The plots can only represent linear correlations and fail to capture any non-linear relationships within the data.
+- The plots might be difficult for non-experts to interpret and should not replace more advanced analyses.
diff --git a/docs/validmind/tests/data_validation/ADF.qmd b/docs/validmind/tests/data_validation/ADF.qmd
new file mode 100644
index 000000000..cf1271f51
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ADF.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ADF"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ADF
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'statsmodels', 'forecasting', 'statistical_test', 'stationarity')
+
+@tasks('regression')
+
+defADF(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Assesses the stationarity of a time series dataset using the Augmented Dickey-Fuller (ADF) test.
+
+### Purpose
+
+The Augmented Dickey-Fuller (ADF) test metric is used to determine the order of integration, i.e., the stationarity of a given time series dataset. The stationary property of data is pivotal in many machine learning models as it impacts the reliability and effectiveness of predictions and forecasts.
+
+### Test Mechanism
+
+The ADF test is executed using the `adfuller` function from the `statsmodels` library on each feature of the dataset. Multiple outputs are generated for each run, including the ADF test statistic and p-value, count of lags used, the number of observations considered in the test, critical values at various confidence levels, and the information criterion. These results are stored for each feature for subsequent analysis.
+
+### Signs of High Risk
+
+- An inflated ADF statistic and high p-value (generally above 0.05) indicate a high risk to the model's performance due to the presence of a unit root indicating non-stationarity.
+- Non-stationarity might result in untrustworthy or insufficient forecasts.
+
+### Strengths
+
+- The ADF test is robust to sophisticated correlations within the data, making it suitable for settings where data displays complex stochastic behavior.
+- It provides explicit outputs like test statistics, critical values, and information criterion, enhancing understanding and transparency in the model validation process.
+
+### Limitations
+
+- The ADF test might demonstrate low statistical power, making it challenging to differentiate between a unit root and near-unit-root processes, potentially causing false negatives.
+- It assumes the data follows an autoregressive process, which might not always be the case.
+- The test struggles with time series data that have structural breaks.
diff --git a/docs/validmind/tests/data_validation/AutoAR.qmd b/docs/validmind/tests/data_validation/AutoAR.qmd
new file mode 100644
index 000000000..d08f176be
--- /dev/null
+++ b/docs/validmind/tests/data_validation/AutoAR.qmd
@@ -0,0 +1,55 @@
+---
+title: "[validmind](/validmind/validmind.qmd).AutoAR"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## AutoAR
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'statsmodels', 'forecasting', 'statistical_test')
+
+@tasks('regression')
+
+defAutoAR(dataset:validmind.vm_models.VMDataset,max_ar_order:int=3):
+
+:::
+
+
+
+Automatically identifies the optimal Autoregressive (AR) order for a time series using BIC and AIC criteria.
+
+### Purpose
+
+The AutoAR test is intended to automatically identify the Autoregressive (AR) order of a time series by utilizing the Bayesian Information Criterion (BIC) and Akaike Information Criterion (AIC). AR order is crucial in forecasting tasks as it dictates the quantity of prior terms in the sequence to use for predicting the current term. The objective is to select the most fitting AR model that encapsulates the trend and seasonality in the time series data.
+
+### Test Mechanism
+
+The test mechanism operates by iterating through a possible range of AR orders up to a defined maximum. An AR model is fitted for each order, and the corresponding BIC and AIC are computed. BIC and AIC statistical measures are designed to penalize models for complexity, preferring simpler models that fit the data proficiently. To verify the stationarity of the time series, the Augmented Dickey-Fuller test is executed. The AR order, BIC, and AIC findings are compiled into a dataframe for effortless comparison. Then, the AR order with the smallest BIC is established as the desirable order for each variable.
+
+### Signs of High Risk
+
+- An augmented Dickey Fuller test p-value > 0.05, indicating the time series isn't stationary, may lead to inaccurate results.
+- Problems with the model fitting procedure, such as computational or convergence issues.
+- Continuous selection of the maximum specified AR order may suggest an insufficient set limit.
+
+### Strengths
+
+- The test independently pinpoints the optimal AR order, thereby reducing potential human bias.
+- It strikes a balance between model simplicity and goodness-of-fit to avoid overfitting.
+- Has the capability to account for stationarity in a time series, an essential aspect for dependable AR modeling.
+- The results are aggregated into a comprehensive table, enabling an easy interpretation.
+
+### Limitations
+
+- The tests need a stationary time series input.
+- They presume a linear relationship between the series and its lags.
+- The search for the best model is constrained by the maximum AR order supplied in the parameters. Therefore, a low max_ar_order could result in subpar outcomes.
+- AIC and BIC may not always agree on the selection of the best model. This potentially requires the user to juggle interpretational choices.
diff --git a/docs/validmind/tests/data_validation/AutoMA.qmd b/docs/validmind/tests/data_validation/AutoMA.qmd
new file mode 100644
index 000000000..20a4cb238
--- /dev/null
+++ b/docs/validmind/tests/data_validation/AutoMA.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).AutoMA"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## AutoMA
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'statsmodels', 'forecasting', 'statistical_test')
+
+@tasks('regression')
+
+defAutoMA(dataset:validmind.vm_models.VMDataset,max_ma_order:int=3):
+
+:::
+
+
+
+Automatically selects the optimal Moving Average (MA) order for each variable in a time series dataset based on minimal BIC and AIC values.
+
+### Purpose
+
+The `AutoMA` metric serves an essential role of automated decision-making for selecting the optimal Moving Average (MA) order for every variable in a given time series dataset. The selection is dependent on the minimalization of BIC (Bayesian Information Criterion) and AIC (Akaike Information Criterion); these are established statistical tools used for model selection. Furthermore, prior to the commencement of the model fitting process, the algorithm conducts a stationarity test (Augmented Dickey-Fuller test) on each series.
+
+### Test Mechanism
+
+Starting off, the `AutoMA` algorithm checks whether the `max_ma_order` parameter has been provided. It consequently loops through all variables in the dataset, carrying out the Dickey-Fuller test for stationarity. For each stationary variable, it fits an ARIMA model for orders running from 0 to `max_ma_order`. The result is a list showcasing the BIC and AIC values of the ARIMA models based on different orders. The MA order, which yields the smallest BIC, is chosen as the 'best MA order' for every single variable. The final results include a table summarizing the auto MA analysis and another table listing the best MA order for each variable.
+
+### Signs of High Risk
+
+- When a series is non-stationary (p-value>0.05 in the Dickey-Fuller test), the produced result could be inaccurate.
+- Any error that arises in the process of fitting the ARIMA models, especially with a higher MA order, can potentially indicate risks and might need further investigation.
+
+### Strengths
+
+- The metric facilitates automation in the process of selecting the MA order for time series forecasting. This significantly saves time and reduces efforts conventionally necessary for manual hyperparameter tuning.
+- The use of both BIC and AIC enhances the likelihood of selecting the most suitable model.
+- The metric ascertains the stationarity of the series prior to model fitting, thus ensuring that the underlying assumptions of the MA model are fulfilled.
+
+### Limitations
+
+- If the time series fails to be stationary, the metric may yield inaccurate results. Consequently, it necessitates pre-processing steps to stabilize the series before fitting the ARIMA model.
+- The metric adopts a rudimentary model selection process based on BIC and doesn't consider other potential model selection strategies. Depending on the specific dataset, other strategies could be more appropriate.
+- The 'max_ma_order' parameter must be manually input which doesn't always guarantee optimal performance, especially when configured too low.
+- The computation time increases with the rise in `max_ma_order`, hence, the metric may become computationally costly for larger values.
diff --git a/docs/validmind/tests/data_validation/AutoStationarity.qmd b/docs/validmind/tests/data_validation/AutoStationarity.qmd
new file mode 100644
index 000000000..c76731266
--- /dev/null
+++ b/docs/validmind/tests/data_validation/AutoStationarity.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).AutoStationarity"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## AutoStationarity
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'statsmodels', 'forecasting', 'statistical_test')
+
+@tasks('regression')
+
+defAutoStationarity(dataset:validmind.vm_models.VMDataset,max_order:int=5,threshold:float=0.05):
+
+:::
+
+
+
+Automates Augmented Dickey-Fuller test to assess stationarity across multiple time series in a DataFrame.
+
+### Purpose
+
+The AutoStationarity metric is intended to automatically detect and evaluate the stationary nature of each time series in a DataFrame. It incorporates the Augmented Dickey-Fuller (ADF) test, a statistical approach used to assess stationarity. Stationarity is a fundamental property suggesting that statistic features like mean and variance remain unchanged over time. This is necessary for many time-series models.
+
+### Test Mechanism
+
+The mechanism for the AutoStationarity test involves applying the Augmented Dicky-Fuller test to each time series within the given dataframe to assess if they are stationary. Every series in the dataframe is looped, using the ADF test up to a defined maximum order (configurable and by default set to 5). The p-value resulting from the ADF test is compared against a predetermined threshold (also configurable and by default set to 0.05). The time series is deemed stationary at its current differencing order if the p-value is less than the threshold.
+
+### Signs of High Risk
+
+- A significant number of series not achieving stationarity even at the maximum order of differencing can indicate high risk or potential failure in the model.
+- This could suggest the series may not be appropriately modeled by a stationary process, hence other modeling approaches might be required.
+
+### Strengths
+
+- The key strength in this metric lies in the automation of the ADF test, enabling mass stationarity analysis across various time series and boosting the efficiency and credibility of the analysis.
+- The utilization of the ADF test, a widely accepted method for testing stationarity, lends authenticity to the results derived.
+- The introduction of the max order and threshold parameters give users the autonomy to determine their preferred levels of stringency in the tests.
+
+### Limitations
+
+- The Augmented Dickey-Fuller test and the stationarity test are not without their limitations. These tests are premised on the assumption that the series can be modeled by an autoregressive process, which may not always hold true.
+- The stationarity check is highly sensitive to the choice of threshold for the significance level; an extremely high or low threshold could lead to incorrect results regarding the stationarity properties.
+- There's also a risk of over-differencing if the maximum order is set too high, which could induce unnecessary cycles.
diff --git a/docs/validmind/tests/data_validation/BivariateScatterPlots.qmd b/docs/validmind/tests/data_validation/BivariateScatterPlots.qmd
new file mode 100644
index 000000000..aed572dd6
--- /dev/null
+++ b/docs/validmind/tests/data_validation/BivariateScatterPlots.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).BivariateScatterPlots"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## BivariateScatterPlots
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'numerical_data', 'visualization')
+
+@tasks('classification')
+
+defBivariateScatterPlots(dataset):
+
+:::
+
+
+
+Generates bivariate scatterplots to visually inspect relationships between pairs of numerical predictor variables in machine learning classification tasks.
+
+### Purpose
+
+This function is intended for visual inspection and monitoring of relationships between pairs of numerical variables in a machine learning model targeting classification tasks. It helps in understanding how predictor variables (features) interact with each other, which can inform feature selection, model-building strategies, and identify potential biases or irregularities in the data.
+
+### Test Mechanism
+
+The function creates scatter plots for each pair of numerical features in the dataset. It first filters out non-numerical and binary features, ensuring the plots focus on meaningful numerical relationships. The resulting scatterplots are color-coded uniformly to avoid visual distraction, and the function returns a tuple of Plotly figure objects, each representing a scatter plot for a pair of features.
+
+### Signs of High Risk
+
+- Visual patterns suggesting non-linear relationships, multicollinearity, clustering, or outlier points in the scatter plots.
+- Such issues could affect the assumptions and performance of certain models, especially those assuming linearity, like logistic regression.
+
+### Strengths
+
+- Scatterplots provide an intuitive and visual tool to explore relationships between two variables.
+- They are useful for identifying outliers, variable associations, and trends, including non-linear patterns.
+- Supports visualization of binary or multi-class classification datasets, focusing on numerical features.
+
+### Limitations
+
+- Scatterplots are limited to bivariate analysis, showing relationships between only two variables at a time.
+- Not ideal for very large datasets where overlapping points can reduce the clarity of the visualization.
+- Scatterplots are exploratory tools and do not provide quantitative measures of model quality or performance.
+- Interpretation is subjective and relies on the domain knowledge and judgment of the viewer.
diff --git a/docs/validmind/tests/data_validation/BoxPierce.qmd b/docs/validmind/tests/data_validation/BoxPierce.qmd
new file mode 100644
index 000000000..0f8d08564
--- /dev/null
+++ b/docs/validmind/tests/data_validation/BoxPierce.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).BoxPierce"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## BoxPierce
+
+
+
+::: {.signature}
+
+@tasks('regression')
+
+@tags('time_series_data', 'forecasting', 'statistical_test', 'statsmodels')
+
+defBoxPierce(dataset):
+
+:::
+
+
+
+Detects autocorrelation in time-series data through the Box-Pierce test to validate model performance.
+
+### Purpose
+
+The Box-Pierce test is utilized to detect the presence of autocorrelation in a time-series dataset. Autocorrelation, or serial correlation, refers to the degree of similarity between observations based on the temporal spacing between them. This test is essential for affirming the quality of a time-series model by ensuring that the error terms in the model are random and do not adhere to a specific pattern.
+
+### Test Mechanism
+
+The implementation of the Box-Pierce test involves calculating a test statistic along with a corresponding p-value derived from the dataset features. These quantities are used to test the null hypothesis that posits the data to be independently distributed. This is achieved by iterating over every feature column in the time-series data and applying the `acorr_ljungbox` function of the statsmodels library. The function yields the Box-Pierce test statistic as well as the respective p-value, all of which are cached as test results.
+
+### Signs of High Risk
+
+- A low p-value, typically under 0.05 as per statistical convention, throws the null hypothesis of independence into question. This implies that the dataset potentially houses autocorrelations, thus indicating a high-risk scenario concerning model performance.
+- Large Box-Pierce test statistic values may indicate the presence of autocorrelation.
+
+### Strengths
+
+- Detects patterns in data that are supposed to be random, thereby ensuring no underlying autocorrelation.
+- Can be computed efficiently given its low computational complexity.
+- Can be widely applied to most regression problems, making it very versatile.
+
+### Limitations
+
+- Assumes homoscedasticity (constant variance) and normality of residuals, which may not always be the case in real-world datasets.
+- May exhibit reduced power for detecting complex autocorrelation schemes such as higher-order or negative correlations.
+- It only provides a general indication of the existence of autocorrelation, without providing specific insights into the nature or patterns of the detected autocorrelation.
+- In the presence of trends or seasonal patterns, the Box-Pierce test may yield misleading results.
+- Applicability is limited to time-series data, which limits its overall utility.
diff --git a/docs/validmind/tests/data_validation/ChiSquaredFeaturesTable.qmd b/docs/validmind/tests/data_validation/ChiSquaredFeaturesTable.qmd
new file mode 100644
index 000000000..e71f3e3f2
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ChiSquaredFeaturesTable.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ChiSquaredFeaturesTable"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ChiSquaredFeaturesTable
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'categorical_data', 'statistical_test')
+
+@tasks('classification')
+
+defChiSquaredFeaturesTable(dataset,p_threshold=0.05):
+
+:::
+
+
+
+Assesses the statistical association between categorical features and a target variable using the Chi-Squared test.
+
+### Purpose
+
+The `ChiSquaredFeaturesTable` function is designed to evaluate the relationship between categorical features and a target variable in a dataset. It performs a Chi-Squared test of independence for each categorical feature to determine whether a statistically significant association exists with the target variable. This is particularly useful in Model Risk Management for understanding the relevance of features and identifying potential biases in a classification model.
+
+### Test Mechanism
+
+The function creates a contingency table for each categorical feature and the target variable, then applies the Chi-Squared test to compute the Chi-squared statistic and the p-value. The results for each feature include the variable name, Chi-squared statistic, p-value, p-value threshold, and a pass/fail status based on whether the p-value is below the specified threshold. The output is a DataFrame summarizing these results, sorted by p-value to highlight the most statistically significant associations.
+
+### Signs of High Risk
+
+- High p-values (greater than the set threshold) indicate a lack of significant association between a feature and the target variable, resulting in a 'Fail' status.
+- Features with a 'Fail' status might not be relevant for the model, which could negatively impact model performance.
+
+### Strengths
+
+- Provides a clear, statistical assessment of the relationship between categorical features and the target variable.
+- Produces an easily interpretable summary with a 'Pass/Fail' outcome for each feature, helping in feature selection.
+- The p-value threshold is adjustable, allowing for flexibility in statistical rigor.
+
+### Limitations
+
+- Assumes the dataset is tabular and consists of categorical variables, which may not be suitable for all datasets.
+- The test is designed for classification tasks and is not applicable to regression problems.
+- As with all hypothesis tests, the Chi-Squared test can only detect associations, not causal relationships.
+- The choice of p-value threshold can affect the interpretation of feature relevance, and different thresholds may lead to different conclusions.
diff --git a/docs/validmind/tests/data_validation/ClassImbalance.qmd b/docs/validmind/tests/data_validation/ClassImbalance.qmd
new file mode 100644
index 000000000..4506dd76d
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ClassImbalance.qmd
@@ -0,0 +1,61 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ClassImbalance"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Threshold based tests
+
+
+
+## ClassImbalance
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'binary_classification', 'multiclass_classification', 'data_quality')
+
+@tasks('classification')
+
+defClassImbalance(dataset:validmind.vm_models.VMDataset,min_percent_threshold:int=10) → Tuple\[Dict\[str, Any\], go.Figure, bool\]:
+
+:::
+
+
+
+Evaluates and quantifies class distribution imbalance in a dataset used by a machine learning model.
+
+### Purpose
+
+The Class Imbalance test is designed to evaluate the distribution of target classes in a dataset that's utilized by a machine learning model. Specifically, it aims to ensure that the classes aren't overly skewed, which could lead to bias in the model's predictions. It's crucial to have a balanced training dataset to avoid creating a model that's biased with high accuracy for the majority class and low accuracy for the minority class.
+
+### Test Mechanism
+
+This Class Imbalance test operates by calculating the frequency (expressed as a percentage) of each class in the target column of the dataset. It then checks whether each class appears in at least a set minimum percentage of the total records. This minimum percentage is a modifiable parameter, but the default value is set to 10%.
+
+### Signs of High Risk
+
+- Any class that represents less than the pre-set minimum percentage threshold is marked as high risk, implying a potential class imbalance.
+- The function provides a pass/fail outcome for each class based on this criterion.
+- Fundamentally, if any class fails this test, it's highly likely that the dataset possesses imbalanced class distribution.
+
+### Strengths
+
+- The test can spot under-represented classes that could affect the efficiency of a machine learning model.
+- The calculation is straightforward and swift.
+- The test is highly informative because it not only spots imbalance, but it also quantifies the degree of imbalance.
+- The adjustable threshold enables flexibility and adaptation to differing use-cases or domain-specific needs.
+- The test creates a visually insightful plot showing the classes and their corresponding proportions, enhancing interpretability and comprehension of the data.
+
+### Limitations
+
+- The test might struggle to perform well or provide vital insights for datasets with a high number of classes. In such cases, the imbalance could be inevitable due to the inherent class distribution.
+- Sensitivity to the threshold value might result in faulty detection of imbalance if the threshold is set excessively high.
+- Regardless of the percentage threshold, it doesn't account for varying costs or impacts of misclassifying different classes, which might fluctuate based on specific applications or domains.
+- While it can identify imbalances in class distribution, it doesn't provide direct methods to address or correct these imbalances.
+- The test is only applicable for classification operations and unsuitable for regression or clustering tasks.
diff --git a/docs/validmind/tests/data_validation/DatasetDescription.qmd b/docs/validmind/tests/data_validation/DatasetDescription.qmd
new file mode 100644
index 000000000..c3c8e31fc
--- /dev/null
+++ b/docs/validmind/tests/data_validation/DatasetDescription.qmd
@@ -0,0 +1,107 @@
+---
+title: "[validmind](/validmind/validmind.qmd).DatasetDescription"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## DatasetDescription
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'time_series_data', 'text_data')
+
+@tasks('classification', 'regression', 'text_classification', 'text_summarization')
+
+defDatasetDescription(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Provides comprehensive analysis and statistical summaries of each column in a machine learning model's dataset.
+
+### Purpose
+
+The test depicted in the script is meant to run a comprehensive analysis on a Machine Learning model's datasets. The test or metric is implemented to obtain a complete summary of the columns in the dataset, including vital statistics of each column such as count, distinct values, missing values, histograms for numerical, categorical, boolean, and text columns. This summary gives a comprehensive overview of the dataset to better understand the characteristics of the data that the model is trained on or evaluates.
+
+### Test Mechanism
+
+The DatasetDescription class accomplishes the purpose as follows: firstly, the test method "run" infers the data type of each column in the dataset and stores the details (id, column type). For each column, the "describe_column" method is invoked to collect statistical information about the column, including count, missing value count and its proportion to the total, unique value count, and its proportion to the total. Depending on the data type of a column, histograms are generated that reflect the distribution of data within the column. Numerical columns use the "get_numerical_histograms" method to calculate histogram distribution, whereas for categorical, boolean and text columns, a histogram is computed with frequencies of each unique value in the datasets. For unsupported types, an error is raised. Lastly, a summary table is built to aggregate all the statistical insights and histograms of the columns in a dataset.
+
+### Signs of High Risk
+
+- High ratio of missing values to total values in one or more columns which may impact the quality of the predictions.
+- Unsupported data types in dataset columns.
+- Large number of unique values in the dataset's columns which might make it harder for the model to establish patterns.
+- Extreme skewness or irregular distribution of data as reflected in the histograms.
+
+### Strengths
+
+- Provides a detailed analysis of the dataset with versatile summaries like count, unique values, histograms, etc.
+- Flexibility in handling different types of data: numerical, categorical, boolean, and text.
+- Useful in detecting problems in the dataset like missing values, unsupported data types, irregular data distribution, etc.
+- The summary gives a comprehensive understanding of dataset features allowing developers to make informed decisions.
+
+### Limitations
+
+- The computation can be expensive from a resource standpoint, particularly for large datasets with numerous columns.
+- The histograms use an arbitrary number of bins which may not be the optimal number of bins for specific data distribution.
+- Unsupported data types for columns will raise an error which may limit evaluating the dataset.
+- Columns with all null or missing values are not included in histogram computation.
+- This test only validates the quality of the dataset but doesn't address the model's performance directly.
+
+
+
+## describe_column
+
+
+
+::: {.signature}
+
+defdescribe_column(df,column):
+
+:::
+
+
+
+Gets descriptive statistics for a single column in a Pandas DataFrame.
+
+
+
+## get_column_histograms
+
+
+
+::: {.signature}
+
+defget_column_histograms(df,column,type\_):
+
+:::
+
+
+
+Returns a collection of histograms for a numerical or categorical column. We store different combinations of bin sizes to allow analyzing the data better
+
+Will be used in favor of \_get_histogram in the future
+
+
+
+## get_numerical_histograms
+
+
+
+::: {.signature}
+
+defget_numerical_histograms(df,column):
+
+:::
+
+
+
+Returns a collection of histograms for a numerical column, each one with a different bin size
diff --git a/docs/validmind/tests/data_validation/DatasetSplit.qmd b/docs/validmind/tests/data_validation/DatasetSplit.qmd
new file mode 100644
index 000000000..dd682148f
--- /dev/null
+++ b/docs/validmind/tests/data_validation/DatasetSplit.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).DatasetSplit"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## DatasetSplit
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'time_series_data', 'text_data')
+
+@tasks('classification', 'regression', 'text_classification', 'text_summarization')
+
+defDatasetSplit(datasets:List\[validmind.vm_models.VMDataset\]):
+
+:::
+
+
+
+Evaluates and visualizes the distribution proportions among training, testing, and validation datasets of an ML model.
+
+### Purpose
+
+The DatasetSplit test is designed to evaluate and visualize the distribution of data among training, testing, and validation datasets, if available, within a given machine learning model. The main purpose is to assess whether the model's datasets are split appropriately, as an imbalanced split might affect the model's ability to learn from the data and generalize to unseen data.
+
+### Test Mechanism
+
+The DatasetSplit test first calculates the total size of all available datasets in the model. Then, for each individual dataset, the methodology involves determining the size of the dataset and its proportion relative to the total size. The results are then conveniently summarized in a table that shows dataset names, sizes, and proportions. Absolute size and proportion of the total dataset size are displayed for each individual dataset.
+
+### Signs of High Risk
+
+- A very small training dataset, which may result in the model not learning enough from the data.
+- A very large training dataset and a small test dataset, which may lead to model overfitting and poor generalization to unseen data.
+- A small or non-existent validation dataset, which might complicate the model's performance assessment.
+
+### Strengths
+
+- The DatasetSplit test provides a clear, understandable visualization of dataset split proportions, which can highlight any potential imbalance in dataset splits quickly.
+- It covers a wide range of task types including classification, regression, and text-related tasks.
+- The metric is not tied to any specific data type and is applicable to tabular data, time series data, or text data.
+
+### Limitations
+
+- The DatasetSplit test does not provide any insight into the quality or diversity of the data within each split, just the size and proportion.
+- The test does not give any recommendations or adjustments for imbalanced datasets.
+- Potential lack of compatibility with more complex modes of data splitting (for example, stratified or time-based splits) could limit the applicability of this test.
diff --git a/docs/validmind/tests/data_validation/DescriptiveStatistics.qmd b/docs/validmind/tests/data_validation/DescriptiveStatistics.qmd
new file mode 100644
index 000000000..d2fa820db
--- /dev/null
+++ b/docs/validmind/tests/data_validation/DescriptiveStatistics.qmd
@@ -0,0 +1,77 @@
+---
+title: "[validmind](/validmind/validmind.qmd).DescriptiveStatistics"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## DescriptiveStatistics
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'time_series_data', 'data_quality')
+
+@tasks('classification', 'regression')
+
+defDescriptiveStatistics(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Performs a detailed descriptive statistical analysis of both numerical and categorical data within a model's dataset.
+
+### Purpose
+
+The purpose of the Descriptive Statistics metric is to provide a comprehensive summary of both numerical and categorical data within a dataset. This involves statistics such as count, mean, standard deviation, minimum and maximum values for numerical data. For categorical data, it calculates the count, number of unique values, most common value and its frequency, and the proportion of the most frequent value relative to the total. The goal is to visualize the overall distribution of the variables in the dataset, aiding in understanding the model's behavior and predicting its performance.
+
+### Test Mechanism
+
+The testing mechanism utilizes two in-built functions of pandas dataframes: `describe()` for numerical fields and `value_counts()` for categorical fields. The `describe()` function pulls out several summary statistics, while `value_counts()` accounts for unique values. The resulting data is formatted into two distinct tables, one for numerical and another for categorical variable summaries. These tables provide a clear summary of the main characteristics of the variables, which can be instrumental in assessing the model's performance.
+
+### Signs of High Risk
+
+- Skewed data or significant outliers can represent high risk. For numerical data, this may be reflected via a significant difference between the mean and median (50% percentile).
+- For categorical data, a lack of diversity (low count of unique values), or overdominance of a single category (high frequency of the top value) can indicate high risk.
+
+### Strengths
+
+- Provides a comprehensive summary of the dataset, shedding light on the distribution and characteristics of the variables under consideration.
+- It is a versatile and robust method, applicable to both numerical and categorical data.
+- Helps highlight crucial anomalies such as outliers, extreme skewness, or lack of diversity, which are vital in understanding model behavior during testing and validation.
+
+### Limitations
+
+- While this metric offers a high-level overview of the data, it may fail to detect subtle correlations or complex patterns.
+- Does not offer any insights on the relationship between variables.
+- Alone, descriptive statistics cannot be used to infer properties about future unseen data.
+- Should be used in conjunction with other statistical tests to provide a comprehensive understanding of the model's data.
+
+
+
+## get_summary_statistics_categorical
+
+
+
+::: {.signature}
+
+defget_summary_statistics_categorical(df,categorical_fields):
+
+:::
+
+
+
+## get_summary_statistics_numerical
+
+
+
+::: {.signature}
+
+defget_summary_statistics_numerical(df,numerical_fields):
+
+:::
diff --git a/docs/validmind/tests/data_validation/DickeyFullerGLS.qmd b/docs/validmind/tests/data_validation/DickeyFullerGLS.qmd
new file mode 100644
index 000000000..f7d9c1e07
--- /dev/null
+++ b/docs/validmind/tests/data_validation/DickeyFullerGLS.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).DickeyFullerGLS"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## DickeyFullerGLS
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'forecasting', 'unit_root_test')
+
+@tasks('regression')
+
+defDickeyFullerGLS(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Assesses stationarity in time series data using the Dickey-Fuller GLS test to determine the order of integration.
+
+### Purpose
+
+The Dickey-Fuller GLS (DFGLS) test is utilized to determine the order of integration in time series data. For machine learning models dealing with time series and forecasting, this metric evaluates the existence of a unit root, thereby checking whether a time series is non-stationary. This analysis is a crucial initial step when dealing with time series data.
+
+### Test Mechanism
+
+This code implements the Dickey-Fuller GLS unit root test on each attribute of the dataset. This process involves iterating through every column of the dataset and applying the DFGLS test to assess the presence of a unit root. The resulting information, including the test statistic ('stat'), the p-value ('pvalue'), the quantity of lagged differences utilized in the regression ('usedlag'), and the number of observations ('nobs'), is subsequently stored.
+
+### Signs of High Risk
+
+- A high p-value for the DFGLS test represents a high risk. Specifically, a p-value above a typical threshold of 0.05 suggests that the time series data is quite likely to be non-stationary, thus presenting a high risk for generating unreliable forecasts.
+
+### Strengths
+
+- The Dickey-Fuller GLS test is a potent tool for checking the stationarity of time series data.
+- It helps to verify the assumptions of the models before the actual construction of the machine learning models proceeds.
+- The results produced by this metric offer a clear insight into whether the data is appropriate for specific machine learning models, especially those demanding the stationarity of time series data.
+
+### Limitations
+
+- Despite its benefits, the DFGLS test does present some drawbacks. It can potentially lead to inaccurate conclusions if the time series data incorporates a structural break.
+- If the time series tends to follow a trend while still being stationary, the test might misinterpret it, necessitating further detrending.
+- The test also presents challenges when dealing with shorter time series data or volatile data, not producing reliable results in these cases.
diff --git a/docs/validmind/tests/data_validation/Duplicates.qmd b/docs/validmind/tests/data_validation/Duplicates.qmd
new file mode 100644
index 000000000..682fa9eeb
--- /dev/null
+++ b/docs/validmind/tests/data_validation/Duplicates.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Duplicates"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Duplicates
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'data_quality', 'text_data')
+
+@tasks('classification', 'regression')
+
+defDuplicates(dataset,min_threshold=1):
+
+:::
+
+
+
+Tests dataset for duplicate entries, ensuring model reliability via data quality verification.
+
+### Purpose
+
+The 'Duplicates' test is designed to check for duplicate rows within the dataset provided to the model. It serves as a measure of data quality, ensuring that the model isn't merely memorizing duplicate entries or being swayed by redundant information. This is an important step in the pre-processing of data for both classification and regression tasks.
+
+### Test Mechanism
+
+This test operates by checking each row for duplicates in the dataset. If a text column is specified in the dataset, the test is conducted on this column; if not, the test is run on all feature columns. The number and percentage of duplicates are calculated and returned in a DataFrame. Additionally, a test is passed if the total count of duplicates falls below a specified minimum threshold.
+
+### Signs of High Risk
+
+- A high number of duplicate rows in the dataset, which can lead to overfitting where the model performs well on the training data but poorly on unseen data.
+- A high percentage of duplicate rows in the dataset, indicating potential problems with data collection or processing.
+
+### Strengths
+
+- Assists in improving the reliability of the model's training process by ensuring the training data is not contaminated with duplicate entries, which can distort statistical analyses.
+- Provides both absolute numbers and percentage values of duplicate rows, giving a thorough overview of data quality.
+- Highly customizable as it allows for setting a user-defined minimum threshold to determine if the test has been passed.
+
+### Limitations
+
+- Does not distinguish between benign duplicates (i.e., coincidental identical entries in different rows) and problematic duplicates originating from data collection or processing errors.
+- The test becomes more computationally intensive as the size of the dataset increases, which might not be suitable for very large datasets.
+- Can only check for exact duplicates and may miss semantically similar information packaged differently.
diff --git a/docs/validmind/tests/data_validation/EngleGrangerCoint.qmd b/docs/validmind/tests/data_validation/EngleGrangerCoint.qmd
new file mode 100644
index 000000000..98121b529
--- /dev/null
+++ b/docs/validmind/tests/data_validation/EngleGrangerCoint.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).EngleGrangerCoint"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## EngleGrangerCoint
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'statistical_test', 'forecasting')
+
+@tasks('regression')
+
+defEngleGrangerCoint(dataset:validmind.vm_models.VMDataset,threshold:float=0.05):
+
+:::
+
+
+
+Assesses the degree of co-movement between pairs of time series data using the Engle-Granger cointegration test.
+
+### Purpose
+
+The intent of this Engle-Granger cointegration test is to explore and quantify the degree of co-movement between pairs of time series variables in a dataset. This is particularly useful in enhancing the accuracy of predictive regressions whenever the underlying variables are co-integrated, i.e., they move together over time.
+
+### Test Mechanism
+
+The test first drops any non-applicable values from the input dataset and then iterates over each pair of variables to apply the Engle-Granger cointegration test. The test generates a 'p' value, which is then compared against a pre-specified threshold (0.05 by default). The pair is labeled as 'Cointegrated' if the 'p' value is less than or equal to the threshold or 'Not cointegrated' otherwise. A summary table is returned by the metric showing cointegration results for each variable pair.
+
+### Signs of High Risk
+
+- A significant number of hypothesized cointegrated variables do not pass the test.
+- A considerable number of 'p' values are close to the threshold, indicating minor data fluctuations can switch the decision between 'Cointegrated' and 'Not cointegrated'.
+
+### Strengths
+
+- Provides an effective way to analyze relationships between time series, particularly in contexts where it's essential to check if variables move together in a statistically significant manner.
+- Useful in various domains, especially finance or economics, where predictive models often hinge on understanding how different variables move together over time.
+
+### Limitations
+
+- Assumes that the time series are integrated of the same order, which isn't always true in multivariate time series datasets.
+- The presence of non-stationary characteristics in the series or structural breaks can result in falsely positive or negative cointegration results.
+- May not perform well for small sample sizes due to lack of statistical power and should be supplemented with other predictive indicators for a more robust model evaluation.
diff --git a/docs/validmind/tests/data_validation/FeatureTargetCorrelationPlot.qmd b/docs/validmind/tests/data_validation/FeatureTargetCorrelationPlot.qmd
new file mode 100644
index 000000000..6300fb261
--- /dev/null
+++ b/docs/validmind/tests/data_validation/FeatureTargetCorrelationPlot.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).FeatureTargetCorrelationPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## FeatureTargetCorrelationPlot
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'visualization', 'correlation')
+
+@tasks('classification', 'regression')
+
+defFeatureTargetCorrelationPlot(dataset,fig_height=600):
+
+:::
+
+
+
+Visualizes the correlation between input features and the model's target output in a color-coded horizontal bar plot.
+
+### Purpose
+
+This test is designed to graphically illustrate the correlations between distinct input features and the target output of a Machine Learning model. Understanding how each feature influences the model's predictions is crucial—a higher correlation indicates a stronger influence of the feature on the target variable. This correlation study is especially advantageous during feature selection and for comprehending the model's operation.
+
+### Test Mechanism
+
+This FeatureTargetCorrelationPlot test computes and presents the correlations between the features and the target variable using a specific dataset. These correlations are calculated and are then graphically represented in a horizontal bar plot, color-coded based on the strength of the correlation. A hovering template can also be utilized for informative tooltips. It is possible to specify the features to be analyzed and adjust the graph's height according to need.
+
+### Signs of High Risk
+
+- There are no strong correlations (either positive or negative) between features and the target variable. This could suggest high risk as the supplied features do not appear to significantly impact the prediction output.
+- The presence of duplicated correlation values might hint at redundancy in the feature set.
+
+### Strengths
+
+- Provides visual assistance to interpreting correlations more effectively.
+- Gives a clear and simple tour of how each feature affects the model's target variable.
+- Beneficial for feature selection and grasping the model's prediction nature.
+- Precise correlation values for each feature are offered by the hover template, contributing to a granular-level comprehension.
+
+### Limitations
+
+- The test only accepts numerical data, meaning variables of other types need to be prepared beforehand.
+- The plot assumes all correlations to be linear, thus non-linear relationships might not be captured effectively.
+- Not apt for models that employ complex feature interactions, like Decision Trees or Neural Networks, as the test may not accurately reflect their importance.
diff --git a/docs/validmind/tests/data_validation/HighCardinality.qmd b/docs/validmind/tests/data_validation/HighCardinality.qmd
new file mode 100644
index 000000000..9bae65aaa
--- /dev/null
+++ b/docs/validmind/tests/data_validation/HighCardinality.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).HighCardinality"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## HighCardinality
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'data_quality', 'categorical_data')
+
+@tasks('classification', 'regression')
+
+defHighCardinality(dataset:validmind.vm_models.VMDataset,num_threshold:int=100,percent_threshold:float=0.1,threshold_type:str='percent'):
+
+:::
+
+
+
+Assesses the number of unique values in categorical columns to detect high cardinality and potential overfitting.
+
+### Purpose
+
+The “High Cardinality” test is used to evaluate the number of unique values present in the categorical columns of a dataset. In this context, high cardinality implies the presence of a large number of unique, non-repetitive values in the dataset.
+
+### Test Mechanism
+
+The test first infers the dataset's type and then calculates an initial numeric threshold based on the test parameters. It only considers columns classified as "Categorical". For each of these columns, the number of distinct values (n_distinct) and the percentage of distinct values (p_distinct) are calculated. The test will pass if n_distinct is less than the calculated numeric threshold. Lastly, the results, which include details such as column name, number of distinct values, and pass/fail status, are compiled into a table.
+
+### Signs of High Risk
+
+- A large number of distinct values (high cardinality) in one or more categorical columns implies a high risk.
+- A column failing the test (n_distinct >= num_threshold) is another indicator of high risk.
+
+### Strengths
+
+- The High Cardinality test is effective in early detection of potential overfitting and unwanted noise.
+- It aids in identifying potential outliers and inconsistencies, thereby improving data quality.
+- The test can be applied to both classification and regression task types, demonstrating its versatility.
+
+### Limitations
+
+- The test is restricted to only "Categorical" data types and is thus not suitable for numerical or continuous features, limiting its scope.
+- The test does not consider the relevance or importance of unique values in categorical features, potentially causing it to overlook critical data points.
+- The threshold (both number and percent) used for the test is static and may not be optimal for diverse datasets and varied applications. Further mechanisms to adjust and refine this threshold could enhance its effectiveness.
diff --git a/docs/validmind/tests/data_validation/HighPearsonCorrelation.qmd b/docs/validmind/tests/data_validation/HighPearsonCorrelation.qmd
new file mode 100644
index 000000000..f7becf783
--- /dev/null
+++ b/docs/validmind/tests/data_validation/HighPearsonCorrelation.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).HighPearsonCorrelation"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## HighPearsonCorrelation
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'data_quality', 'correlation')
+
+@tasks('classification', 'regression')
+
+defHighPearsonCorrelation(dataset:validmind.vm_models.VMDataset,max_threshold:float=0.3,top_n_correlations:int=10,feature_columns:list=None):
+
+:::
+
+
+
+Identifies highly correlated feature pairs in a dataset suggesting feature redundancy or multicollinearity.
+
+### Purpose
+
+The High Pearson Correlation test measures the linear relationship between features in a dataset, with the main goal of identifying high correlations that might indicate feature redundancy or multicollinearity. Identification of such issues allows developers and risk management teams to properly deal with potential impacts on the machine learning model's performance and interpretability.
+
+### Test Mechanism
+
+The test works by generating pairwise Pearson correlations for all features in the dataset, then sorting and eliminating duplicate and self-correlations. It assigns a Pass or Fail based on whether the absolute value of the correlation coefficient surpasses a pre-set threshold (defaulted at 0.3). It lastly returns the top n strongest correlations regardless of passing or failing status (where n is 10 by default but can be configured by passing the `top_n_correlations` parameter).
+
+### Signs of High Risk
+
+- A high risk indication would be the presence of correlation coefficients exceeding the threshold.
+- If the features share a strong linear relationship, this could lead to potential multicollinearity and model overfitting.
+- Redundancy of variables can undermine the interpretability of the model due to uncertainty over the authenticity of individual variable's predictive power.
+
+### Strengths
+
+- Provides a quick and simple means of identifying relationships between feature pairs.
+- Generates a transparent output that displays pairs of correlated variables, the Pearson correlation coefficient, and a Pass or Fail status for each.
+- Aids in early identification of potential multicollinearity issues that may disrupt model training.
+
+### Limitations
+
+- Can only delineate linear relationships, failing to shed light on nonlinear relationships or dependencies.
+- Sensitive to outliers where a few outliers could notably affect the correlation coefficient.
+- Limited to identifying redundancy only within feature pairs; may fail to spot more complex relationships among three or more variables.
diff --git a/docs/validmind/tests/data_validation/IQROutliersBarPlot.qmd b/docs/validmind/tests/data_validation/IQROutliersBarPlot.qmd
new file mode 100644
index 000000000..ca5ed977d
--- /dev/null
+++ b/docs/validmind/tests/data_validation/IQROutliersBarPlot.qmd
@@ -0,0 +1,72 @@
+---
+title: "[validmind](/validmind/validmind.qmd).IQROutliersBarPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## compute_outliers
+
+
+
+::: {.signature}
+
+defcompute_outliers(series,threshold):
+
+:::
+
+
+
+## IQROutliersBarPlot
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'visualization', 'numerical_data')
+
+@tasks('classification', 'regression')
+
+defIQROutliersBarPlot(dataset:validmind.vm_models.VMDataset,threshold:float=1.5,fig_width:int=800):
+
+:::
+
+
+
+Visualizes outlier distribution across percentiles in numerical data using the Interquartile Range (IQR) method.
+
+### Purpose
+
+The InterQuartile Range Outliers Bar Plot (IQROutliersBarPlot) metric aims to visually analyze and evaluate the extent of outliers in numeric variables based on percentiles. Its primary purpose is to clarify the dataset's distribution, flag possible abnormalities in it, and gauge potential risks associated with processing potentially skewed data, which can affect the machine learning model's predictive prowess.
+
+### Test Mechanism
+
+The examination invokes a series of steps:
+
+1. For every numeric feature in the dataset, the 25th percentile (Q1) and 75th percentile (Q3) are calculated before deriving the Interquartile Range (IQR), the difference between Q1 and Q3.
+1. Subsequently, the metric calculates the lower and upper thresholds by subtracting Q1 from the `threshold` times IQR and adding Q3 to `threshold` times IQR, respectively. The default `threshold` is set at 1.5.
+1. Any value in the feature that falls below the lower threshold or exceeds the upper threshold is labeled as an outlier.
+1. The number of outliers are tallied for different percentiles, such as \[0-25\], \[25-50\], \[50-75\], and \[75-100\].
+1. These counts are employed to construct a bar plot for the feature, showcasing the distribution of outliers across different percentiles.
+
+### Signs of High Risk
+
+- A prevalence of outliers in the data, potentially skewing its distribution.
+- Outliers dominating higher percentiles (75-100) which implies the presence of extreme values, capable of severely influencing the model's performance.
+- Certain features harboring most of their values as outliers, which signifies that these features might not contribute positively to the model's forecasting ability.
+
+### Strengths
+
+- Effectively identifies outliers in the data through visual means, facilitating easier comprehension and offering insights into the outliers' possible impact on the model.
+- Provides flexibility by accommodating all numeric features or a chosen subset.
+- Task-agnostic in nature; it is viable for both classification and regression tasks.
+- Can handle large datasets as its operation does not hinge on computationally heavy operations.
+
+### Limitations
+
+- Its application is limited to numerical variables and does not extend to categorical ones.
+- Only reveals the presence and distribution of outliers and does not provide insights into how these outliers might affect the model's predictive performance.
+- The assumption that data is unimodal and symmetric may not always hold true. In cases with non-normal distributions, the results can be misleading.
diff --git a/docs/validmind/tests/data_validation/IQROutliersTable.qmd b/docs/validmind/tests/data_validation/IQROutliersTable.qmd
new file mode 100644
index 000000000..824bdcb20
--- /dev/null
+++ b/docs/validmind/tests/data_validation/IQROutliersTable.qmd
@@ -0,0 +1,66 @@
+---
+title: "[validmind](/validmind/validmind.qmd).IQROutliersTable"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## compute_outliers
+
+
+
+::: {.signature}
+
+defcompute_outliers(series,threshold=1.5):
+
+:::
+
+
+
+## IQROutliersTable
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'numerical_data')
+
+@tasks('classification', 'regression')
+
+defIQROutliersTable(dataset:validmind.vm_models.VMDataset,threshold:float=1.5):
+
+:::
+
+
+
+Determines and summarizes outliers in numerical features using the Interquartile Range method.
+
+### Purpose
+
+The "Interquartile Range Outliers Table" (IQROutliersTable) metric is designed to identify and summarize outliers within numerical features of a dataset using the Interquartile Range (IQR) method. This exercise is crucial in the pre-processing of data because outliers can substantially distort statistical analysis and impact the performance of machine learning models.
+
+### Test Mechanism
+
+The IQR, which is the range separating the first quartile (25th percentile) from the third quartile (75th percentile), is calculated for each numerical feature within the dataset. An outlier is defined as a data point falling below the "Q1 - 1.5 * IQR" or above "Q3 + 1.5 * IQR" range. The test computes the number of outliers and their summary statistics (minimum, 25th percentile, median, 75th percentile, and maximum values) for each numerical feature. If no specific features are chosen, the test applies to all numerical features in the dataset. The default outlier threshold is set to 1.5 but can be customized by the user.
+
+### Signs of High Risk
+
+- A large number of outliers in multiple features.
+- Outliers significantly distanced from the mean value of variables.
+- Extremely high or low outlier values indicative of data entry errors or other data quality issues.
+
+### Strengths
+
+- Provides a comprehensive summary of outliers for each numerical feature, helping pinpoint features with potential quality issues.
+- The IQR method is robust to extremely high or low outlier values as it is based on quartile calculations.
+- Can be customized to work on selected features and set thresholds for outliers.
+
+### Limitations
+
+- Might cause false positives if the variable deviates from a normal or near-normal distribution, especially for skewed distributions.
+- Does not provide interpretation or recommendations for addressing outliers, relying on further analysis by users or data scientists.
+- Only applicable to numerical features, not categorical data.
+- Default thresholds may not be optimal for data with heavy pre-processing, manipulation, or inherently high kurtosis (heavy tails).
diff --git a/docs/validmind/tests/data_validation/IsolationForestOutliers.qmd b/docs/validmind/tests/data_validation/IsolationForestOutliers.qmd
new file mode 100644
index 000000000..2c2659aa9
--- /dev/null
+++ b/docs/validmind/tests/data_validation/IsolationForestOutliers.qmd
@@ -0,0 +1,57 @@
+---
+title: "[validmind](/validmind/validmind.qmd).IsolationForestOutliers"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## IsolationForestOutliers
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'anomaly_detection')
+
+@tasks('classification')
+
+defIsolationForestOutliers(dataset:validmind.vm_models.VMDataset,random_state:int=0,contamination:float=0.1,feature_columns:list=None):
+
+:::
+
+
+
+Detects outliers in a dataset using the Isolation Forest algorithm and visualizes results through scatter plots.
+
+### Purpose
+
+The IsolationForestOutliers test is designed to identify anomalies or outliers in the model's dataset using the isolation forest algorithm. This algorithm assumes that anomalous data points can be isolated more quickly due to their distinctive properties. By creating isolation trees and identifying instances with shorter average path lengths, the test is able to pick out data points that differ from the majority.
+
+### Test Mechanism
+
+The test uses the isolation forest algorithm, which builds an ensemble of isolation trees by randomly selecting features and splitting the data based on random thresholds. It isolates anomalies rather than focusing on normal data points. For each pair of variables, a scatter plot is generated which distinguishes the identified outliers from the inliers. The results of the test can be visualized using these scatter plots, illustrating the distinction between outliers and inliers.
+
+### Signs of High Risk
+
+- The presence of high contamination, indicating a large number of anomalies
+- Inability to detect clusters of anomalies that are close in the feature space
+- Misclassifying normal instances as anomalies
+- Failure to detect actual anomalies
+
+### Strengths
+
+- Ability to handle large, high-dimensional datasets
+- Efficiency in isolating anomalies instead of normal instances
+- Insensitivity to the underlying distribution of data
+- Ability to recognize anomalies even when they are not separated from the main data cloud through identifying distinctive properties
+- Visually presents the test results for better understanding and interpretability
+
+### Limitations
+
+- Difficult to detect anomalies that are close to each other or prevalent in datasets
+- Dependency on the contamination parameter which may need fine-tuning to be effective
+- Potential failure in detecting collective anomalies if they behave similarly to normal data
+- Potential lack of precision in identifying which features contribute most to the anomalous behavior
diff --git a/docs/validmind/tests/data_validation/JarqueBera.qmd b/docs/validmind/tests/data_validation/JarqueBera.qmd
new file mode 100644
index 000000000..2a1ae958f
--- /dev/null
+++ b/docs/validmind/tests/data_validation/JarqueBera.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).JarqueBera"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## JarqueBera
+
+
+
+::: {.signature}
+
+@tasks('classification', 'regression')
+
+@tags('tabular_data', 'data_distribution', 'statistical_test', 'statsmodels')
+
+defJarqueBera(dataset):
+
+:::
+
+
+
+Assesses normality of dataset features in an ML model using the Jarque-Bera test.
+
+### Purpose
+
+The purpose of the Jarque-Bera test as implemented in this metric is to determine if the features in the dataset of a given Machine Learning model follow a normal distribution. This is crucial for understanding the distribution and behavior of the model's features, as numerous statistical methods assume normal distribution of the data.
+
+### Test Mechanism
+
+The test mechanism involves computing the Jarque-Bera statistic, p-value, skew, and kurtosis for each feature in the dataset. It utilizes the 'jarque_bera' function from the 'statsmodels' library in Python, storing the results in a dictionary. The test evaluates the skewness and kurtosis to ascertain whether the dataset follows a normal distribution. A significant p-value (typically less than 0.05) implies that the data does not possess normal distribution.
+
+### Signs of High Risk
+
+- A high Jarque-Bera statistic and a low p-value (usually less than 0.05) indicate high-risk conditions.
+- Such results suggest the data significantly deviates from a normal distribution. If a machine learning model expects feature data to be normally distributed, these findings imply that it may not function as intended.
+
+### Strengths
+
+- Provides insights into the shape of the data distribution, helping determine whether a given set of data follows a normal distribution.
+- Particularly useful for risk assessment for models that assume a normal distribution of data.
+- By measuring skewness and kurtosis, it provides additional insights into the nature and magnitude of a distribution's deviation.
+
+### Limitations
+
+- Only checks for normality in the data distribution. It cannot provide insights into other types of distributions.
+- Datasets that aren't normally distributed but follow some other distribution might lead to inaccurate risk assessments.
+- Highly sensitive to large sample sizes, often rejecting the null hypothesis (that data is normally distributed) even for minor deviations in larger datasets.
diff --git a/docs/validmind/tests/data_validation/KPSS.qmd b/docs/validmind/tests/data_validation/KPSS.qmd
new file mode 100644
index 000000000..4e3db67fd
--- /dev/null
+++ b/docs/validmind/tests/data_validation/KPSS.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).KPSS"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## KPSS
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'stationarity', 'unit_root_test', 'statsmodels')
+
+@tasks('data_validation')
+
+defKPSS(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Assesses the stationarity of time-series data in a machine learning model using the KPSS unit root test.
+
+### Purpose
+
+The KPSS (Kwiatkowski-Phillips-Schmidt-Shin) unit root test is utilized to ensure the stationarity of data within a machine learning model. It specifically works on time-series data to establish the order of integration, which is essential for accurate forecasting. A fundamental requirement for any time series model is that the series should be stationary.
+
+### Test Mechanism
+
+This test calculates the KPSS score for each feature in the dataset. The KPSS score includes a statistic, a p-value, a used lag, and critical values. The core principle behind the KPSS test is to evaluate the hypothesis that an observable time series is stationary around a deterministic trend. If the computed statistic exceeds the critical value, the null hypothesis (that the series is stationary) is rejected, indicating that the series is non-stationary.
+
+### Signs of High Risk
+
+- High KPSS score, particularly if the calculated statistic is higher than the critical value.
+- Rejection of the null hypothesis, indicating that the series is recognized as non-stationary, can severely affect the model's forecasting capability.
+
+### Strengths
+
+- Directly measures the stationarity of a series, fulfilling a key prerequisite for many time-series models.
+- The underlying logic of the test is intuitive and simple, making it easy to understand and accessible for both developers and risk management teams.
+
+### Limitations
+
+- Assumes the absence of a unit root in the series and doesn't differentiate between series that are stationary and those border-lining stationarity.
+- The test may have restricted power against certain alternatives.
+- The reliability of the test is contingent on the number of lags selected, which introduces potential bias in the measurement.
diff --git a/docs/validmind/tests/data_validation/LJungBox.qmd b/docs/validmind/tests/data_validation/LJungBox.qmd
new file mode 100644
index 000000000..867019263
--- /dev/null
+++ b/docs/validmind/tests/data_validation/LJungBox.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).LJungBox"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## LJungBox
+
+
+
+::: {.signature}
+
+@tasks('regression')
+
+@tags('time_series_data', 'forecasting', 'statistical_test', 'statsmodels')
+
+defLJungBox(dataset):
+
+:::
+
+
+
+Assesses autocorrelations in dataset features by performing a Ljung-Box test on each feature.
+
+### Purpose
+
+The Ljung-Box test is a type of statistical test utilized to ascertain whether there are autocorrelations within a given dataset that differ significantly from zero. In the context of a machine learning model, this test is primarily used to evaluate data utilized in regression tasks, especially those involving time series and forecasting.
+
+### Test Mechanism
+
+The test operates by iterating over each feature within the dataset and applying the `acorr_ljungbox` function from the `statsmodels.stats.diagnostic` library. This function calculates the Ljung-Box statistic and p-value for each feature. These results are then stored in a pandas DataFrame where the columns are the feature names, statistic, and p-value respectively. Generally, a lower p-value indicates a higher likelihood of significant autocorrelations within the feature.
+
+### Signs of High Risk
+
+- High Ljung-Box statistic values or low p-values.
+- Presence of significant autocorrelations in the respective features.
+- Potential for negative impact on model performance or bias if autocorrelations are not properly handled.
+
+### Strengths
+
+- Powerful tool for detecting autocorrelations within datasets, especially in time series data.
+- Provides quantitative measures (statistic and p-value) for precise evaluation.
+- Helps avoid issues related to autoregressive residuals and other challenges in regression models.
+
+### Limitations
+
+- Cannot detect all types of non-linearity or complex interrelationships among variables.
+- Testing individual features may not fully encapsulate the dynamics of the data if features interact with each other.
+- Designed more for traditional statistical models and may not be fully compatible with certain types of complex machine learning models.
diff --git a/docs/validmind/tests/data_validation/LaggedCorrelationHeatmap.qmd b/docs/validmind/tests/data_validation/LaggedCorrelationHeatmap.qmd
new file mode 100644
index 000000000..e96b90f10
--- /dev/null
+++ b/docs/validmind/tests/data_validation/LaggedCorrelationHeatmap.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).LaggedCorrelationHeatmap"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## LaggedCorrelationHeatmap
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'visualization')
+
+@tasks('regression')
+
+defLaggedCorrelationHeatmap(dataset:validmind.vm_models.VMDataset,num_lags:int=10):
+
+:::
+
+
+
+Assesses and visualizes correlation between target variable and lagged independent variables in a time-series dataset.
+
+### Purpose
+
+The LaggedCorrelationHeatmap metric is utilized to appraise and illustrate the correlation between the target variable and delayed copies (lags) of independent variables in a time-series dataset. It assists in revealing relationships in time-series data where the influence of an independent variable on the dependent variable is not immediate but occurs after a period (lags).
+
+### Test Mechanism
+
+To execute this test, Python's Pandas library pairs with Plotly to perform computations and present the visualization in the form of a heatmap. The test begins by extracting the target variable and corresponding independent variables from the dataset. Then, generation of lags of independent variables takes place, followed by the calculation of correlation between these lagged variables and the target variable. The outcome is a correlation matrix that gets recorded and illustrated as a heatmap, where different color intensities represent the strength of the correlation, making patterns easier to identify.
+
+### Signs of High Risk
+
+- Insignificant correlations across the heatmap, indicating a lack of noteworthy relationships between variables.
+- Correlations that break intuition or previous understanding, suggesting potential issues with the dataset or the model.
+
+### Strengths
+
+- This metric serves as an exceptional tool for exploring and visualizing time-dependent relationships between features and the target variable in a time-series dataset.
+- It aids in identifying delayed effects that might go unnoticed with other correlation measures.
+- The heatmap offers an intuitive visual representation of time-dependent correlations and influences.
+
+### Limitations
+
+- The metric presumes linear relationships between variables, potentially ignoring non-linear relationships.
+- The correlation considered is linear; therefore, intricate non-linear interactions might be overlooked.
+- The metric is only applicable for time-series data, limiting its utility outside of this context.
+- The number of lags chosen can significantly influence the results; too many lags can render the heatmap difficult to interpret, while too few might overlook delayed effects.
+- This metric does not take into account any causal relationships, but merely demonstrates correlation.
diff --git a/docs/validmind/tests/data_validation/MissingValues.qmd b/docs/validmind/tests/data_validation/MissingValues.qmd
new file mode 100644
index 000000000..da471664a
--- /dev/null
+++ b/docs/validmind/tests/data_validation/MissingValues.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).MissingValues"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## MissingValues
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'data_quality')
+
+@tasks('classification', 'regression')
+
+defMissingValues(dataset:validmind.vm_models.VMDataset,min_threshold:int=1):
+
+:::
+
+
+
+Evaluates dataset quality by ensuring missing value ratio across all features does not exceed a set threshold.
+
+### Purpose
+
+The Missing Values test is designed to evaluate the quality of a dataset by measuring the number of missing values across all features. The objective is to ensure that the ratio of missing data to total data is less than a predefined threshold, defaulting to 1, in order to maintain the data quality necessary for reliable predictive strength in a machine learning model.
+
+### Test Mechanism
+
+The mechanism for this test involves iterating through each column of the dataset, counting missing values (represented as NaNs), and calculating the percentage they represent against the total number of rows. The test then checks if these missing value counts are less than the predefined `min_threshold`. The results are shown in a table summarizing each column, the number of missing values, the percentage of missing values in each column, and a Pass/Fail status based on the threshold comparison.
+
+### Signs of High Risk
+
+- When the number of missing values in any column exceeds the `min_threshold` value.
+- Presence of missing values across many columns, leading to multiple instances of failing the threshold.
+
+### Strengths
+
+- Quick and granular identification of missing data across each feature in the dataset.
+- Provides an effective and straightforward means of maintaining data quality, essential for constructing efficient machine learning models.
+
+### Limitations
+
+- Does not suggest the root causes of the missing values or recommend ways to impute or handle them.
+- May overlook features with significant missing data but still less than the `min_threshold`, potentially impacting the model.
+- Does not account for data encoded as values like "-999" or "None," which might not technically classify as missing but could bear similar implications.
diff --git a/docs/validmind/tests/data_validation/MissingValuesBarPlot.qmd b/docs/validmind/tests/data_validation/MissingValuesBarPlot.qmd
new file mode 100644
index 000000000..8f85c8448
--- /dev/null
+++ b/docs/validmind/tests/data_validation/MissingValuesBarPlot.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).MissingValuesBarPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## MissingValuesBarPlot
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'data_quality', 'visualization')
+
+@tasks('classification', 'regression')
+
+defMissingValuesBarPlot(dataset:validmind.vm_models.VMDataset,threshold:int=80,fig_height:int=600):
+
+:::
+
+
+
+Assesses the percentage and distribution of missing values in the dataset via a bar plot, with emphasis on identifying high-risk columns based on a user-defined threshold.
+
+### Purpose
+
+The 'MissingValuesBarPlot' metric provides a color-coded visual representation of the percentage of missing values for each column in an ML model's dataset. The primary purpose of this metric is to easily identify and quantify missing data, which are essential steps in data preprocessing. The presence of missing data can potentially skew the model's predictions and decrease its accuracy. Additionally, this metric uses a pre-set threshold to categorize various columns into ones that contain missing data above the threshold (high risk) and below the threshold (less risky).
+
+### Test Mechanism
+
+The test mechanism involves scanning each column in the input dataset and calculating the percentage of missing values. It then compares each column's missing data percentage with the predefined threshold, categorizing columns with missing data above the threshold as high-risk. The test generates a bar plot in which columns with missing data are represented on the y-axis and their corresponding missing data percentages are displayed on the x-axis. The color of each bar reflects the missing data percentage in relation to the threshold: grey for values below the threshold and light coral for those exceeding it. The user-defined threshold is represented by a red dashed line on the plot.
+
+### Signs of High Risk
+
+- Columns with higher percentages of missing values beyond the threshold are high-risk. These are visually represented by light coral bars on the bar plot.
+
+### Strengths
+
+- Helps in quickly identifying and quantifying missing data across all columns of the dataset.
+- Facilitates pattern recognition through visual representation.
+- Enables customization of the level of risk tolerance via a user-defined threshold.
+- Supports both classification and regression tasks, sharing its versatility.
+
+### Limitations
+
+- It only considers the quantity of missing values, not differentiating between different types of missingness (Missing completely at random - MCAR, Missing at random - MAR, Not Missing at random - NMAR).
+- It doesn't offer insights into potential approaches for handling missing entries, such as various imputation strategies.
+- The metric does not consider possible impacts of the missing data on the model's accuracy or precision.
+- Interpretation of the findings and the next steps might require an expert understanding of the field.
diff --git a/docs/validmind/tests/data_validation/MutualInformation.qmd b/docs/validmind/tests/data_validation/MutualInformation.qmd
new file mode 100644
index 000000000..f75ad9464
--- /dev/null
+++ b/docs/validmind/tests/data_validation/MutualInformation.qmd
@@ -0,0 +1,68 @@
+---
+title: "[validmind](/validmind/validmind.qmd).MutualInformation"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## MutualInformation
+
+
+
+::: {.signature}
+
+@tags('feature_selection', 'data_analysis')
+
+@tasks('classification', 'regression')
+
+defMutualInformation(dataset:validmind.vm_models.VMDataset,min_threshold:float=0.01,task:str='classification'):
+
+:::
+
+
+
+Calculates mutual information scores between features and target variable to evaluate feature relevance.
+
+### Purpose
+
+The Mutual Information test quantifies the predictive power of each feature by measuring its statistical dependency with the target variable. This helps identify relevant features for model training and detect potential redundant or irrelevant variables, supporting feature selection decisions and model interpretability.
+
+### Test Mechanism
+
+The test employs sklearn's mutual_info_classif/mutual_info_regression functions to compute mutual information between each feature and the target. It produces a normalized score (0 to 1) for each feature, where higher scores indicate stronger relationships. Results are presented in both tabular format and visualized through a bar plot with a configurable threshold line.
+
+### Signs of High Risk
+
+- Many features showing very low mutual information scores
+- Key business features exhibiting unexpectedly low scores
+- All features showing similar, low information content
+- Large discrepancy between business importance and MI scores
+- Highly skewed distribution of MI scores
+- Critical features below the minimum threshold
+- Unexpected zero or near-zero scores for known important features
+- Inconsistent scores across different data samples
+
+### Strengths
+
+- Captures non-linear relationships between features and target
+- Scale-invariant measurement of feature relevance
+- Works for both classification and regression tasks
+- Provides interpretable scores (0 to 1 scale)
+- Supports automated feature selection
+- No assumptions about data distribution
+- Handles numerical and categorical features
+- Computationally efficient for most datasets
+
+### Limitations
+
+- Requires sufficient data for reliable estimates
+- May be computationally intensive for very large datasets
+- Cannot detect redundant features (pairwise relationships)
+- Sensitive to feature discretization for continuous variables
+- Does not account for feature interactions
+- May underestimate importance of rare but crucial events
+- Cannot handle missing values directly
+- May be affected by extreme class imbalance
diff --git a/docs/validmind/tests/data_validation/PearsonCorrelationMatrix.qmd b/docs/validmind/tests/data_validation/PearsonCorrelationMatrix.qmd
new file mode 100644
index 000000000..15d513cb8
--- /dev/null
+++ b/docs/validmind/tests/data_validation/PearsonCorrelationMatrix.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).PearsonCorrelationMatrix"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## PearsonCorrelationMatrix
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'numerical_data', 'correlation')
+
+@tasks('classification', 'regression')
+
+defPearsonCorrelationMatrix(dataset):
+
+:::
+
+
+
+Evaluates linear dependency between numerical variables in a dataset via a Pearson Correlation coefficient heat map.
+
+### Purpose
+
+This test is intended to evaluate the extent of linear dependency between all pairs of numerical variables in the given dataset. It provides the Pearson Correlation coefficient, which reveals any high correlations present. The purpose of doing this is to identify potential redundancy, as variables that are highly correlated can often be removed to reduce the dimensionality of the dataset without significantly impacting the model's performance.
+
+### Test Mechanism
+
+This metric test generates a correlation matrix for all numerical variables in the dataset using the Pearson correlation formula. A heat map is subsequently created to visualize this matrix effectively. The color of each point on the heat map corresponds to the magnitude and direction (positive or negative) of the correlation, with a range from -1 (perfect negative correlation) to 1 (perfect positive correlation). Any correlation coefficients higher than 0.7 (in absolute terms) are indicated in white in the heat map, suggesting a high degree of correlation.
+
+### Signs of High Risk
+
+- A large number of variables in the dataset showing a high degree of correlation (coefficients approaching ±1). This indicates redundancy within the dataset, suggesting that some variables may not be contributing new information to the model.
+- Potential risk of overfitting.
+
+### Strengths
+
+- Detects and quantifies the linearity of relationships between variables, aiding in identifying redundant variables to simplify models and potentially improve performance.
+- The heatmap visualization provides an easy-to-understand overview of correlations, beneficial for users not comfortable with numerical matrices.
+
+### Limitations
+
+- Limited to detecting linear relationships, potentially missing non-linear relationships which impede opportunities for dimensionality reduction.
+- Measures only the degree of linear relationship, not the strength of one variable's effect on another.
+- The 0.7 correlation threshold is arbitrary and might exclude valid dependencies with lower coefficients.
diff --git a/docs/validmind/tests/data_validation/PhillipsPerronArch.qmd b/docs/validmind/tests/data_validation/PhillipsPerronArch.qmd
new file mode 100644
index 000000000..2bbcc79c5
--- /dev/null
+++ b/docs/validmind/tests/data_validation/PhillipsPerronArch.qmd
@@ -0,0 +1,57 @@
+---
+title: "[validmind](/validmind/validmind.qmd).PhillipsPerronArch"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## PhillipsPerronArch
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'forecasting', 'statistical_test', 'unit_root_test')
+
+@tasks('regression')
+
+defPhillipsPerronArch(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Assesses the stationarity of time series data in each feature of the ML model using the Phillips-Perron test.
+
+### Purpose
+
+The Phillips-Perron (PP) test is used to determine the stationarity of time series data for each feature in a dataset, which is crucial for forecasting tasks. It tests the null hypothesis that a time series is unit-root non-stationary. This is vital for understanding the stochastic behavior of the data and ensuring the robustness and validity of predictions generated by regression analysis models.
+
+### Test Mechanism
+
+The PP test is conducted for each feature in the dataset as follows:
+
+- A data frame is created from the dataset.
+- For each column, the Phillips-Perron method calculates the test statistic, p-value, lags used, and number of observations.
+- The results are then stored for each feature, providing a metric that indicates the stationarity of the time series data.
+
+### Signs of High Risk
+
+- A high p-value, indicating that the series has a unit root and is non-stationary.
+- Test statistic values exceeding critical values, suggesting non-stationarity.
+- High 'usedlag' value, pointing towards autocorrelation issues that may degrade model performance.
+
+### Strengths
+
+- Resilience against heteroskedasticity in the error term.
+- Effective for long time series data.
+- Helps in determining whether the time series is stationary, aiding in the selection of suitable forecasting models.
+
+### Limitations
+
+- Applicable only within a univariate time series framework.
+- Relies on asymptotic theory, which may reduce the test’s power for small sample sizes.
+- Non-stationary time series must be converted to stationary series through differencing, potentially leading to loss of important data points.
diff --git a/docs/validmind/tests/data_validation/ProtectedClassesCombination.qmd b/docs/validmind/tests/data_validation/ProtectedClassesCombination.qmd
new file mode 100644
index 000000000..dae223206
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ProtectedClassesCombination.qmd
@@ -0,0 +1,57 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ProtectedClassesCombination"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ProtectedClassesCombination
+
+
+
+::: {.signature}
+
+@tags('bias_and_fairness')
+
+@tasks('classification', 'regression')
+
+defProtectedClassesCombination(dataset,model,protected_classes=None):
+
+:::
+
+
+
+Visualizes combinations of protected classes and their corresponding error metric differences.
+
+### Purpose
+
+This test aims to provide insights into how different combinations of protected classes affect various error metrics, particularly the false negative rate (FNR) and false positive rate (FPR). By visualizing these combinations, it helps identify potential biases or disparities in model performance across different intersectional groups.
+
+### Test Mechanism
+
+The test performs the following steps:
+
+1. Combines the specified protected class columns to create a single multi-class category.
+1. Calculates error metrics (FNR, FPR, etc.) for each combination of protected classes.
+1. Generates visualizations showing the distribution of these metrics across all class combinations.
+
+### Signs of High Risk
+
+- Large disparities in FNR or FPR across different protected class combinations.
+- Consistent patterns of higher error rates for specific combinations of protected attributes.
+- Unexpected or unexplainable variations in error metrics between similar group combinations.
+
+### Strengths
+
+- Provides a comprehensive view of intersectional fairness across multiple protected attributes.
+- Allows for easy identification of potentially problematic combinations of protected classes.
+- Visualizations make it easier to spot patterns or outliers in model performance across groups.
+
+### Limitations
+
+- May become complex and difficult to interpret with a large number of protected classes or combinations.
+- Does not provide statistical significance of observed differences.
+- Visualization alone may not capture all nuances of intersectional fairness.
diff --git a/docs/validmind/tests/data_validation/ProtectedClassesDescription.qmd b/docs/validmind/tests/data_validation/ProtectedClassesDescription.qmd
new file mode 100644
index 000000000..41d15fc9b
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ProtectedClassesDescription.qmd
@@ -0,0 +1,63 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ProtectedClassesDescription"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ProtectedClassesDescription
+
+
+
+::: {.signature}
+
+@tags('bias_and_fairness', 'descriptive_statistics')
+
+@tasks('classification', 'regression')
+
+defProtectedClassesDescription(dataset,protected_classes=None):
+
+:::
+
+
+
+Visualizes the distribution of protected classes in the dataset relative to the target variable and provides descriptive statistics.
+
+### Purpose
+
+The ProtectedClassesDescription test aims to identify potential biases or significant differences in the distribution of target outcomes across different protected classes. This visualization and statistical summary help in understanding the relationship between protected attributes and the target variable, which is crucial for assessing fairness in machine learning models.
+
+### Test Mechanism
+
+The function creates interactive stacked bar charts for each specified protected class using Plotly. Additionally, it generates a single table of descriptive statistics for all protected classes, including:
+
+- Protected class and category
+- Count and percentage of each category within the protected class
+- Mean, median, and mode of the target variable for each category
+- Standard deviation of the target variable for each category
+- Minimum and maximum values of the target variable for each category
+
+### Signs of High Risk
+
+- Significant imbalances in the distribution of target outcomes across different categories of a protected class.
+- Large disparities in mean, median, or mode of the target variable across categories.
+- Underrepresentation or overrepresentation of certain groups within protected classes.
+- High standard deviations in certain categories, indicating potential volatility or outliers.
+
+### Strengths
+
+- Provides both visual and statistical representation of potential biases in the dataset.
+- Allows for easy identification of imbalances in target variable distribution across protected classes.
+- Interactive plots enable detailed exploration of the data.
+- Consolidated statistical summary provides quantitative measures to complement visual analysis.
+- Applicable to both classification and regression tasks.
+
+### Limitations
+
+- Does not provide advanced statistical measures of bias or fairness.
+- May become cluttered if there are many categories within a protected class or many unique target values.
+- Interpretation may require domain expertise to understand the implications of observed disparities.
+- Does not account for intersectionality or complex interactions between multiple protected attributes.
diff --git a/docs/validmind/tests/data_validation/ProtectedClassesDisparity.qmd b/docs/validmind/tests/data_validation/ProtectedClassesDisparity.qmd
new file mode 100644
index 000000000..aa8efafc2
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ProtectedClassesDisparity.qmd
@@ -0,0 +1,59 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ProtectedClassesDisparity"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ProtectedClassesDisparity
+
+
+
+::: {.signature}
+
+@tags('bias_and_fairness')
+
+@tasks('classification', 'regression')
+
+defProtectedClassesDisparity(dataset,model,protected_classes=None,disparity_tolerance=1.25,metrics=\['fnr', 'fpr', 'tpr'\]):
+
+:::
+
+
+
+Investigates disparities in model performance across different protected class segments.
+
+### Purpose
+
+This test aims to identify and quantify potential biases in model outcomes by comparing various performance metrics across different segments of protected classes. It helps in assessing whether the model produces discriminatory outcomes for certain groups, which is crucial for ensuring fairness in machine learning models.
+
+### Test Mechanism
+
+The test performs the following steps:
+
+1. Calculates performance metrics (e.g., false negative rate, false positive rate, true positive rate) for each segment of the specified protected classes.
+1. Computes disparity ratios by comparing these metrics between different segments and a reference group.
+1. Generates visualizations showing the disparities and their relation to a user-defined disparity tolerance threshold.
+1. Produces a comprehensive table with various disparity metrics for detailed analysis.
+
+### Signs of High Risk
+
+- Disparity ratios exceeding the specified disparity tolerance threshold.
+- Consistent patterns of higher error rates or lower performance for specific protected class segments.
+- Statistically significant differences in performance metrics across segments.
+
+### Strengths
+
+- Provides a comprehensive view of model fairness across multiple protected attributes and metrics.
+- Allows for easy identification of problematic disparities through visual and tabular representations.
+- Customizable disparity tolerance threshold to align with specific use-case requirements.
+- Applicable to various performance metrics, offering a multi-faceted analysis of model fairness.
+
+### Limitations
+
+- Relies on a predefined reference group for each protected class, which may not always be the most appropriate choice.
+- Does not account for intersectionality between different protected attributes.
+- The interpretation of results may require domain expertise to understand the implications of observed disparities.
diff --git a/docs/validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.qmd b/docs/validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.qmd
new file mode 100644
index 000000000..42a2e8bd6
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.qmd
@@ -0,0 +1,130 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ProtectedClassesThresholdOptimizer"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## calculate_fairness_metrics
+
+
+
+::: {.signature}
+
+defcalculate_fairness_metrics(test_df,target,y_pred_opt,protected_classes):
+
+:::
+
+
+
+## calculate_group_metrics
+
+
+
+::: {.signature}
+
+defcalculate_group_metrics(test_df,target,y_pred_opt,protected_classes):
+
+:::
+
+
+
+## get_thresholds_by_group
+
+
+
+::: {.signature}
+
+defget_thresholds_by_group(threshold_optimizer):
+
+:::
+
+
+
+## initialize_and_fit_optimizer
+
+
+
+::: {.signature}
+
+definitialize_and_fit_optimizer(pipeline,X_train,y_train,protected_classes_df):
+
+:::
+
+
+
+## make_predictions
+
+
+
+::: {.signature}
+
+defmake_predictions(threshold_optimizer,test_df,protected_classes):
+
+:::
+
+
+
+## plot_thresholds
+
+
+
+::: {.signature}
+
+defplot_thresholds(threshold_optimizer):
+
+:::
+
+
+
+## ProtectedClassesThresholdOptimizer
+
+
+
+::: {.signature}
+
+@tags('bias_and_fairness')
+
+@tasks('classification', 'regression')
+
+defProtectedClassesThresholdOptimizer(dataset,pipeline=None,protected_classes=None,X_train=None,y_train=None):
+
+:::
+
+
+
+Obtains a classifier by applying group-specific thresholds to the provided estimator.
+
+### Purpose
+
+This test aims to optimize the fairness of a machine learning model by applying different classification thresholds for different protected groups. It helps in mitigating bias and achieving more equitable outcomes across different demographic groups.
+
+### Test Mechanism
+
+The test uses Fairlearn's ThresholdOptimizer to:
+
+1. Fit an optimizer on the training data, considering protected classes.
+1. Apply optimized thresholds to make predictions on the test data.
+1. Calculate and report various fairness metrics.
+1. Visualize the optimized thresholds.
+
+### Signs of High Risk
+
+- Large disparities in fairness metrics (e.g., Demographic Parity Ratio, Equalized Odds Ratio) across different protected groups.
+- Significant differences in False Positive Rates (FPR) or True Positive Rates (TPR) between groups.
+- Thresholds that vary widely across different protected groups.
+
+### Strengths
+
+- Provides a post-processing method to improve model fairness without modifying the original model.
+- Allows for balancing multiple fairness criteria simultaneously.
+- Offers visual insights into the threshold optimization process.
+
+### Limitations
+
+- May lead to a decrease in overall model performance while improving fairness.
+- Requires access to protected attribute information at prediction time.
+- The effectiveness can vary depending on the chosen fairness constraint and objective.
diff --git a/docs/validmind/tests/data_validation/RollingStatsPlot.qmd b/docs/validmind/tests/data_validation/RollingStatsPlot.qmd
new file mode 100644
index 000000000..6e432c13c
--- /dev/null
+++ b/docs/validmind/tests/data_validation/RollingStatsPlot.qmd
@@ -0,0 +1,66 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RollingStatsPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## plot_rolling_statistics
+
+
+
+::: {.signature}
+
+defplot_rolling_statistics(df,col,window_size):
+
+:::
+
+
+
+## RollingStatsPlot
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'visualization', 'stationarity')
+
+@tasks('regression')
+
+defRollingStatsPlot(dataset:validmind.vm_models.VMDataset,window_size:int=12):
+
+:::
+
+
+
+Evaluates the stationarity of time series data by plotting its rolling mean and standard deviation over a specified window.
+
+### Purpose
+
+The `RollingStatsPlot` metric is employed to gauge the stationarity of time series data in a given dataset. This metric specifically evaluates the rolling mean and rolling standard deviation of the dataset over a pre-specified window size. The rolling mean provides an understanding of the average trend in the data, while the rolling standard deviation gauges the volatility of the data within the window. It is critical in preparing time series data for modeling as it reveals key insights into data behavior across time.
+
+### Test Mechanism
+
+This mechanism is comprised of two steps. Initially, the rolling mean and standard deviation for each of the dataset's columns are calculated over a window size, which can be user-specified or by default set to 12 data points. Then, the calculated rolling mean and standard deviation are visualized via separate plots, illustrating the trends and volatility in the dataset. A straightforward check is conducted to ensure the existence of columns in the dataset, and to verify that the given dataset has been indexed by its date and time—a necessary prerequisite for time series analysis.
+
+### Signs of High Risk
+
+- The presence of non-stationary patterns in either the rolling mean or the rolling standard deviation plots, which could indicate trends or seasonality in the data that may affect the performance of time series models.
+- Missing columns in the dataset, which would prevent the execution of this metric correctly.
+- The detection of NaN values in the dataset, which may need to be addressed before the metric can proceed successfully.
+
+### Strengths
+
+- Offers visualizations of trending behavior and volatility within the data, facilitating a broader understanding of the dataset's inherent characteristics.
+- Checks of the dataset's integrity, such as the existence of all required columns and the availability of a datetime index.
+- Adjusts to accommodate various window sizes, thus allowing accurate analysis of data with differing temporal granularities.
+- Considers each column of the data individually, thereby accommodating multi-feature datasets.
+
+### Limitations
+
+- For all columns, a fixed-size window is utilized. This may not accurately capture patterns in datasets where different features may require different optimal window sizes.
+- Requires the dataset to be indexed by date and time, hence it may not be usable for datasets without a timestamp index.
+- Primarily serves for data visualization as it does not facilitate any quantitative measures for stationarity, such as through statistical tests. Therefore, the interpretation is subjective and depends heavily on modeler discretion.
diff --git a/docs/validmind/tests/data_validation/RunsTest.qmd b/docs/validmind/tests/data_validation/RunsTest.qmd
new file mode 100644
index 000000000..61b6ec3e1
--- /dev/null
+++ b/docs/validmind/tests/data_validation/RunsTest.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RunsTest"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RunsTest
+
+
+
+::: {.signature}
+
+@tasks('classification', 'regression')
+
+@tags('tabular_data', 'statistical_test', 'statsmodels')
+
+defRunsTest(dataset):
+
+:::
+
+
+
+Executes Runs Test on ML model to detect non-random patterns in output data sequence.
+
+### Purpose
+
+The Runs Test is a statistical procedure used to determine whether the sequence of data extracted from the ML model behaves randomly or not. Specifically, it analyzes runs, sequences of consecutive positives or negatives, in the data to check if there are more or fewer runs than expected under the assumption of randomness. This can be an indication of some pattern, trend, or cycle in the model's output which may need attention.
+
+### Test Mechanism
+
+The testing mechanism applies the Runs Test from the statsmodels module on each column of the training dataset. For every feature in the dataset, a Runs Test is executed, whose output includes a Runs Statistic and P-value. A low P-value suggests that data arrangement in the feature is not likely to be random. The results are stored in a dictionary where the keys are the feature names, and the values are another dictionary storing the test statistic and the P-value for each feature.
+
+### Signs of High Risk
+
+- High risk is indicated when the P-value is close to zero.
+- If the P-value is less than a predefined significance level (like 0.05), it suggests that the runs (series of positive or negative values) in the model's output are not random and are longer or shorter than what is expected under a random scenario.
+- This would mean there's a high risk of non-random distribution of errors or model outcomes, suggesting potential issues with the model.
+
+### Strengths
+
+- Straightforward and fast for detecting non-random patterns in data sequence.
+- Validates assumptions of randomness, which is valuable for checking error distributions in regression models, trendless time series data, and ensuring a classifier doesn't favor one class over another.
+- Can be applied to both classification and regression tasks, making it versatile.
+
+### Limitations
+
+- Assumes that the data is independently and identically distributed (i.i.d.), which might not be the case for many real-world datasets.
+- The conclusion drawn from the low P-value indicating non-randomness does not provide information about the type or the source of the detected pattern.
+- Sensitive to extreme values (outliers), and overly large or small run sequences can influence the results.
+- Does not provide model performance evaluation; it is used to detect patterns in the sequence of outputs only.
diff --git a/docs/validmind/tests/data_validation/ScatterPlot.qmd b/docs/validmind/tests/data_validation/ScatterPlot.qmd
new file mode 100644
index 000000000..0da71096d
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ScatterPlot.qmd
@@ -0,0 +1,57 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ScatterPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ScatterPlot
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'visualization')
+
+@tasks('classification', 'regression')
+
+defScatterPlot(dataset):
+
+:::
+
+
+
+Assesses visual relationships, patterns, and outliers among features in a dataset through scatter plot matrices.
+
+### Purpose
+
+The ScatterPlot test aims to visually analyze a given dataset by constructing a scatter plot matrix of its numerical features. The primary goal is to uncover relationships, patterns, and outliers across different features to provide both quantitative and qualitative insights into multidimensional relationships within the dataset. This visual assessment aids in understanding the efficacy of the chosen features for model training and their suitability.
+
+### Test Mechanism
+
+Using the Seaborn library, the ScatterPlot function creates the scatter plot matrix. The process involves retrieving all numerical columns from the dataset and generating a scatter matrix for these columns. The resulting scatter plot provides visual representations of feature relationships. The function also adjusts axis labels for readability and returns the final plot as a Matplotlib Figure object for further analysis and visualization.
+
+### Signs of High Risk
+
+- The emergence of non-linear or random patterns across different feature pairs, suggesting complex relationships unsuitable for linear assumptions.
+- Lack of clear patterns or clusters, indicating weak or non-existent correlations among features, which could challenge certain model types.
+- Presence of outliers, as visual outliers can adversely influence the model's performance.
+
+### Strengths
+
+- Provides insight into the multidimensional relationships among multiple features.
+- Assists in identifying trends, correlations, and outliers that could affect model performance.
+- Validates assumptions made during model creation, such as linearity.
+- Versatile for application in both regression and classification tasks.
+- Using Seaborn facilitates an intuitive and detailed visual exploration of data.
+
+### Limitations
+
+- Scatter plot matrices may become cluttered and hard to decipher as the number of features increases.
+- Primarily reveals pairwise relationships and may fail to illuminate complex interactions involving three or more features.
+- Being a visual tool, precision in quantitative analysis might be compromised.
+- Outliers not clearly visible in plots can be missed, affecting model performance.
+- Assumes that the dataset can fit into the computer's memory, which might not be valid for extremely large datasets.
diff --git a/docs/validmind/tests/data_validation/ScoreBandDefaultRates.qmd b/docs/validmind/tests/data_validation/ScoreBandDefaultRates.qmd
new file mode 100644
index 000000000..7369e8ad6
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ScoreBandDefaultRates.qmd
@@ -0,0 +1,72 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ScoreBandDefaultRates"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ScoreBandDefaultRates
+
+
+
+::: {.signature}
+
+@tags('visualization', 'credit_risk', 'scorecard')
+
+@tasks('classification')
+
+defScoreBandDefaultRates(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel,score_column:str='score',score_bands:list=None):
+
+:::
+
+
+
+Analyzes default rates and population distribution across credit score bands.
+
+### Purpose
+
+The Score Band Default Rates test evaluates the discriminatory power of credit scores by analyzing default rates across different score bands. This helps validate score effectiveness, supports policy decisions, and provides insights into portfolio risk distribution.
+
+### Test Mechanism
+
+The test segments the score distribution into bands and calculates key metrics for each band:
+
+1. Population count and percentage in each band
+1. Default rate within each band
+1. Cumulative statistics across bands The results show how well the scores separate good and bad accounts.
+
+### Signs of High Risk
+
+- Non-monotonic default rates across score bands
+- Insufficient population in critical score bands
+- Unexpected default rates for score ranges
+- High concentration in specific score bands
+- Similar default rates across adjacent bands
+- Unstable default rates in key decision bands
+- Extreme population skewness
+- Poor risk separation between bands
+
+### Strengths
+
+- Clear view of score effectiveness
+- Supports policy threshold decisions
+- Easy to interpret and communicate
+- Directly links to business decisions
+- Shows risk segmentation power
+- Identifies potential score issues
+- Helps validate scoring model
+- Supports portfolio monitoring
+
+### Limitations
+
+- Sensitive to band definition choices
+- May mask within-band variations
+- Requires sufficient data in each band
+- Cannot capture non-linear patterns
+- Point-in-time analysis only
+- No temporal trend information
+- Assumes band boundaries are appropriate
+- May oversimplify risk patterns
diff --git a/docs/validmind/tests/data_validation/SeasonalDecompose.qmd b/docs/validmind/tests/data_validation/SeasonalDecompose.qmd
new file mode 100644
index 000000000..ab267d4da
--- /dev/null
+++ b/docs/validmind/tests/data_validation/SeasonalDecompose.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).SeasonalDecompose"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## SeasonalDecompose
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'seasonality', 'statsmodels')
+
+@tasks('regression')
+
+defSeasonalDecompose(dataset:validmind.vm_models.VMDataset,seasonal_model:str='additive'):
+
+:::
+
+
+
+Assesses patterns and seasonality in a time series dataset by decomposing its features into foundational components.
+
+### Purpose
+
+The Seasonal Decompose test aims to decompose the features of a time series dataset into their fundamental components: observed, trend, seasonal, and residuals. By utilizing the Seasonal Decomposition of Time Series by Loess (STL) method, the test identifies underlying patterns, predominantly seasonality, in the dataset's features. This aids in developing a more comprehensive understanding of the dataset, which in turn facilitates more effective model validation.
+
+### Test Mechanism
+
+The testing process leverages the `seasonal_decompose` function from the `statsmodels.tsa.seasonal` library to evaluate each feature in the dataset. It isolates each feature into four components—observed, trend, seasonal, and residuals—and generates six subplot graphs per feature for visual interpretation. Prior to decomposition, the test scrutinizes and removes any non-finite values, ensuring the reliability of the analysis.
+
+### Signs of High Risk
+
+- **Non-Finiteness**: Datasets with a high number of non-finite values may flag as high risk since these values are omitted before conducting the seasonal decomposition.
+- **Frequent Warnings**: Chronic failure to infer the frequency for a scrutinized feature indicates high risk.
+- **High Seasonality**: A significant seasonal component could potentially render forecasts unreliable due to overwhelming seasonal variation.
+
+### Strengths
+
+- **Seasonality Detection**: Accurately discerns hidden seasonality patterns in dataset features.
+- **Visualization**: Facilitates interpretation and comprehension through graphical representations.
+- **Unrestricted Usage**: Not confined to any specific regression model, promoting wide-ranging applicability.
+
+### Limitations
+
+- **Dependence on Assumptions**: Assumes that dataset features are periodically distributed. Features with no inferable frequency are excluded from the test.
+- **Handling Non-Finite Values**: Disregards non-finite values during analysis, potentially resulting in an incomplete understanding of the dataset.
+- **Unreliability with Noisy Datasets**: Produces unreliable results when used with datasets that contain heavy noise.
diff --git a/docs/validmind/tests/data_validation/ShapiroWilk.qmd b/docs/validmind/tests/data_validation/ShapiroWilk.qmd
new file mode 100644
index 000000000..33806279a
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ShapiroWilk.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ShapiroWilk"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ShapiroWilk
+
+
+
+::: {.signature}
+
+@tasks('classification', 'regression')
+
+@tags('tabular_data', 'data_distribution', 'statistical_test')
+
+defShapiroWilk(dataset):
+
+:::
+
+
+
+Evaluates feature-wise normality of training data using the Shapiro-Wilk test.
+
+### Purpose
+
+The Shapiro-Wilk test is utilized to investigate whether a particular dataset conforms to the standard normal distribution. This analysis is crucial in machine learning modeling because the normality of the data can profoundly impact the performance of the model. This metric is especially useful in evaluating various features of the dataset in both classification and regression tasks.
+
+### Test Mechanism
+
+The Shapiro-Wilk test is conducted on each feature column of the training dataset to determine if the data contained fall within the normal distribution. The test presents a statistic and a p-value, with the p-value serving to validate or repudiate the null hypothesis, which is that the tested data is normally distributed.
+
+### Signs of High Risk
+
+- A p-value that falls below 0.05 signifies a high risk as it discards the null hypothesis, indicating that the data does not adhere to the normal distribution.
+- For machine learning models built on the presumption of data normality, such an outcome could result in subpar performance or incorrect predictions.
+
+### Strengths
+
+- The Shapiro-Wilk test is esteemed for its level of accuracy, thereby making it particularly well-suited to datasets of small to moderate sizes.
+- It proves its versatility through its efficient functioning in both classification and regression tasks.
+- By separately testing each feature column, the Shapiro-Wilk test can raise an alarm if a specific feature does not comply with the normality.
+
+### Limitations
+
+- The Shapiro-Wilk test's sensitivity can be a disadvantage as it often rejects the null hypothesis (i.e., data is normally distributed), even for minor deviations, especially in large datasets. This may lead to unwarranted 'false alarms' of high risk by deeming the data as not normally distributed even if it approximates normal distribution.
+- Exceptional care must be taken in managing missing data or outliers prior to testing as these can greatly skew the results.
+- Lastly, the Shapiro-Wilk test is not optimally suited for processing data with pronounced skewness or kurtosis.
diff --git a/docs/validmind/tests/data_validation/Skewness.qmd b/docs/validmind/tests/data_validation/Skewness.qmd
new file mode 100644
index 000000000..114c52794
--- /dev/null
+++ b/docs/validmind/tests/data_validation/Skewness.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Skewness"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Skewness
+
+
+
+::: {.signature}
+
+@tags('data_quality', 'tabular_data')
+
+@tasks('classification', 'regression')
+
+defSkewness(dataset,max_threshold=1):
+
+:::
+
+
+
+Evaluates the skewness of numerical data in a dataset to check against a defined threshold, aiming to ensure data quality and optimize model performance.
+
+### Purpose
+
+The purpose of the Skewness test is to measure the asymmetry in the distribution of data within a predictive machine learning model. Specifically, it evaluates the divergence of said distribution from a normal distribution. Understanding the level of skewness helps identify data quality issues, which are crucial for optimizing the performance of traditional machine learning models in both classification and regression settings.
+
+### Test Mechanism
+
+This test calculates the skewness of numerical columns in the dataset, focusing specifically on numerical data types. The calculated skewness value is then compared against a predetermined maximum threshold, which is set by default to 1. If the skewness value is less than this maximum threshold, the test passes; otherwise, it fails. The test results, along with the skewness values and column names, are then recorded for further analysis.
+
+### Signs of High Risk
+
+- Substantial skewness levels that significantly exceed the maximum threshold.
+- Persistent skewness in the data, indicating potential issues with the foundational assumptions of the machine learning model.
+- Subpar model performance, erroneous predictions, or biased inferences due to skewed data distributions.
+
+### Strengths
+
+- Fast and efficient identification of unequal data distributions within a machine learning model.
+- Adjustable maximum threshold parameter, allowing for customization based on user needs.
+- Provides a clear quantitative measure to mitigate model risks related to data skewness.
+
+### Limitations
+
+- Only evaluates numeric columns, potentially missing skewness or bias in non-numeric data.
+- Assumes that data should follow a normal distribution, which may not always be applicable to real-world data.
+- Subjective threshold for risk grading, requiring expert input and recurrent iterations for refinement.
diff --git a/docs/validmind/tests/data_validation/SpreadPlot.qmd b/docs/validmind/tests/data_validation/SpreadPlot.qmd
new file mode 100644
index 000000000..9868e269a
--- /dev/null
+++ b/docs/validmind/tests/data_validation/SpreadPlot.qmd
@@ -0,0 +1,55 @@
+---
+title: "[validmind](/validmind/validmind.qmd).SpreadPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## SpreadPlot
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'visualization')
+
+@tasks('regression')
+
+defSpreadPlot(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Assesses potential correlations between pairs of time series variables through visualization to enhance understanding of their relationships.
+
+### Purpose
+
+The SpreadPlot test aims to graphically illustrate and analyze the relationships between pairs of time series variables within a given dataset. This facilitated understanding helps in identifying and assessing potential time series correlations, such as cointegration, between the variables.
+
+### Test Mechanism
+
+The SpreadPlot test computes and represents the spread between each pair of time series variables in the dataset. Specifically, the difference between two variables is calculated and presented as a line graph. This process is iterated for each unique pair of variables in the dataset, allowing for comprehensive visualization of their relationships.
+
+### Signs of High Risk
+
+- Large fluctuations in the spread over a given timespan.
+- Unexpected patterns or trends that may signal potential risks in the underlying correlations between the variables.
+- Presence of significant missing data or extreme outlier values, which could potentially skew the spread and indicate high risk.
+
+### Strengths
+
+- Allows for thorough visual examination and interpretation of the correlations between time-series pairs.
+- Aids in revealing complex relationships like cointegration.
+- Enhances interpretability by visualizing the relationships, thereby helping in spotting outliers and trends.
+- Capable of handling numerous variable pairs from the dataset through a versatile and adaptable process.
+
+### Limitations
+
+- Primarily serves as a visualization tool and does not offer quantitative measurements or statistics to objectively determine relationships.
+- Heavily relies on the quality and granularity of the data—missing data or outliers can notably disturb the interpretation of relationships.
+- Can become inefficient or difficult to interpret with a high number of variables due to the profuse number of plots.
+- Might not completely capture intricate non-linear relationships between the variables.
diff --git a/docs/validmind/tests/data_validation/TabularCategoricalBarPlots.qmd b/docs/validmind/tests/data_validation/TabularCategoricalBarPlots.qmd
new file mode 100644
index 000000000..237869d09
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TabularCategoricalBarPlots.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TabularCategoricalBarPlots"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TabularCategoricalBarPlots
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'visualization')
+
+@tasks('classification', 'regression')
+
+defTabularCategoricalBarPlots(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Generates and visualizes bar plots for each category in categorical features to evaluate the dataset's composition.
+
+### Purpose
+
+The purpose of this metric is to visually analyze categorical data using bar plots. It is intended to evaluate the dataset's composition by displaying the counts of each category in each categorical feature.
+
+### Test Mechanism
+
+The provided dataset is first checked to determine if it contains any categorical variables. If no categorical columns are found, the tool raises a ValueError. For each categorical variable in the dataset, a separate bar plot is generated. The number of occurrences for each category is calculated and displayed on the plot. If a dataset contains multiple categorical columns, multiple bar plots are produced.
+
+### Signs of High Risk
+
+- High risk could occur if the categorical variables exhibit an extreme imbalance, with categories having very few instances possibly being underrepresented in the model, which could affect the model's performance and its ability to generalize.
+- Another sign of risk is if there are too many categories in a single variable, which could lead to overfitting and make the model complex.
+
+### Strengths
+
+- Provides a visual and intuitively understandable representation of categorical data.
+- Aids in the analysis of variable distributions.
+- Helps in easily identifying imbalances or rare categories that could affect the model's performance.
+
+### Limitations
+
+- This method only works with categorical data and won't apply to numerical variables.
+- It does not provide informative value when there are too many categories, as the bar chart could become cluttered and hard to interpret.
+- Offers no insights into the model's performance or precision, but rather provides a descriptive analysis of the input.
diff --git a/docs/validmind/tests/data_validation/TabularDateTimeHistograms.qmd b/docs/validmind/tests/data_validation/TabularDateTimeHistograms.qmd
new file mode 100644
index 000000000..5469fb6ff
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TabularDateTimeHistograms.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TabularDateTimeHistograms"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TabularDateTimeHistograms
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'visualization')
+
+@tasks('classification', 'regression')
+
+defTabularDateTimeHistograms(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Generates histograms to provide graphical insight into the distribution of time intervals in a model's datetime data.
+
+### Purpose
+
+The `TabularDateTimeHistograms` metric is designed to provide graphical insight into the distribution of time intervals in a machine learning model's datetime data. By plotting histograms of differences between consecutive date entries in all datetime variables, it enables an examination of the underlying pattern of time series data and identification of anomalies.
+
+### Test Mechanism
+
+This test operates by first identifying all datetime columns and extracting them from the dataset. For each datetime column, it next computes the differences (in days) between consecutive dates, excluding zero values, and visualizes these differences in a histogram. The Plotly library's histogram function is used to generate histograms, which are labeled appropriately and provide a graphical representation of the frequency of different day intervals in the dataset.
+
+### Signs of High Risk
+
+- If no datetime columns are detected in the dataset, this would lead to a ValueError. Hence, the absence of datetime columns signifies a high risk.
+- A severely skewed or irregular distribution depicted in the histogram may indicate possible complications with the data, such as faulty timestamps or abnormalities.
+
+### Strengths
+
+- The metric offers a visual overview of time interval frequencies within the dataset, supporting the recognition of inherent patterns.
+- Histogram plots can aid in the detection of potential outliers and data anomalies, contributing to an assessment of data quality.
+- The metric is versatile, compatible with a range of task types, including classification and regression, and can work with multiple datetime variables if present.
+
+### Limitations
+
+- A major weakness of this metric is its dependence on the visual examination of data, as it does not provide a measurable evaluation of the model.
+- The metric might overlook complex or multi-dimensional trends in the data.
+- The test is only applicable to datasets containing datetime columns and will fail if such columns are unavailable.
+- The interpretation of the histograms relies heavily on the domain expertise and experience of the reviewer.
diff --git a/docs/validmind/tests/data_validation/TabularDescriptionTables.qmd b/docs/validmind/tests/data_validation/TabularDescriptionTables.qmd
new file mode 100644
index 000000000..7b1c86ee7
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TabularDescriptionTables.qmd
@@ -0,0 +1,132 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TabularDescriptionTables"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## get_categorical_columns
+
+
+
+::: {.signature}
+
+defget_categorical_columns(dataset):
+
+:::
+
+
+
+## get_datetime_columns
+
+
+
+::: {.signature}
+
+defget_datetime_columns(dataset):
+
+:::
+
+
+
+## get_numerical_columns
+
+
+
+::: {.signature}
+
+defget_numerical_columns(dataset):
+
+:::
+
+
+
+## get_summary_statistics_categorical
+
+
+
+::: {.signature}
+
+defget_summary_statistics_categorical(dataset,categorical_fields):
+
+:::
+
+
+
+## get_summary_statistics_datetime
+
+
+
+::: {.signature}
+
+defget_summary_statistics_datetime(dataset,datetime_fields):
+
+:::
+
+
+
+## get_summary_statistics_numerical
+
+
+
+::: {.signature}
+
+defget_summary_statistics_numerical(dataset,numerical_fields):
+
+:::
+
+
+
+## TabularDescriptionTables
+
+
+
+::: {.signature}
+
+@tags('tabular_data')
+
+@tasks('classification', 'regression')
+
+defTabularDescriptionTables(dataset):
+
+:::
+
+
+
+Summarizes key descriptive statistics for numerical, categorical, and datetime variables in a dataset.
+
+### Purpose
+
+The main purpose of this metric is to gather and present the descriptive statistics of numerical, categorical, and datetime variables present in a dataset. The attributes it measures include the count, mean, minimum and maximum values, percentage of missing values, data types of fields, and unique values for categorical fields, among others.
+
+### Test Mechanism
+
+The test first segregates the variables in the dataset according to their data types (numerical, categorical, or datetime). Then, it compiles summary statistics for each type of variable. The specifics of these statistics vary depending on the type of variable:
+
+- For numerical variables, the metric extracts descriptors like count, mean, minimum and maximum values, count of missing values, and data types.
+- For categorical variables, it counts the number of unique values, displays unique values, counts missing values, and identifies data types.
+- For datetime variables, it counts the number of unique values, identifies the earliest and latest dates, counts missing values, and identifies data types.
+
+### Signs of High Risk
+
+- Masses of missing values in the descriptive statistics results could hint at high risk or failure, indicating potential data collection, integrity, and quality issues.
+- Detection of inappropriate distributions for numerical variables, like having negative values for variables that are always supposed to be positive.
+- Identifying inappropriate data types, like a continuous variable being encoded as a categorical type.
+
+### Strengths
+
+- Provides a comprehensive overview of the dataset.
+- Gives a snapshot into the essence of the numerical, categorical, and datetime fields.
+- Identifies potential data quality issues such as missing values or inconsistencies crucial for building credible machine learning models.
+- The metadata, including the data type and missing value information, are vital for anyone including data scientists dealing with the dataset before the modeling process.
+
+### Limitations
+
+- It does not perform any deeper statistical analysis or tests on the data.
+- It does not handle issues such as outliers, or relationships between variables.
+- It offers no insights into potential correlations or possible interactions between variables.
+- It does not investigate the potential impact of missing values on the performance of the machine learning models.
+- It does not explore potential transformation requirements that may be necessary to enhance the performance of the chosen algorithm.
diff --git a/docs/validmind/tests/data_validation/TabularNumericalHistograms.qmd b/docs/validmind/tests/data_validation/TabularNumericalHistograms.qmd
new file mode 100644
index 000000000..b8124e9ca
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TabularNumericalHistograms.qmd
@@ -0,0 +1,56 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TabularNumericalHistograms"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TabularNumericalHistograms
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'visualization')
+
+@tasks('classification', 'regression')
+
+defTabularNumericalHistograms(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Generates histograms for each numerical feature in a dataset to provide visual insights into data distribution and detect potential issues.
+
+### Purpose
+
+The purpose of this test is to provide visual analysis of numerical data through the generation of histograms for each numerical feature in the dataset. Histograms aid in the exploratory analysis of data, offering insight into the distribution of the data, skewness, presence of outliers, and central tendencies. It helps in understanding if the inputs to the model are normally distributed, which is a common assumption in many machine learning algorithms.
+
+### Test Mechanism
+
+This test scans the provided dataset and extracts all the numerical columns. For each numerical column, it constructs a histogram using plotly, with 50 bins. The deployment of histograms offers a robust visual aid, ensuring unruffled identification and understanding of numerical data distribution patterns.
+
+### Signs of High Risk
+
+- A high degree of skewness
+- Unexpected data distributions
+- Existence of extreme outliers in the histograms
+
+These may indicate issues with the data that the model is receiving. If data for a numerical feature is expected to follow a certain distribution (like a normal distribution) but does not, it could lead to sub-par performance by the model. As such these instances should be treated as high-risk indicators.
+
+### Strengths
+
+- Provides a simple, easy-to-interpret visualization of how data for each numerical attribute is distributed.
+- Helps detect skewed values and outliers that could potentially harm the AI model's performance.
+- Can be applied to large datasets and multiple numerical variables conveniently.
+
+### Limitations
+
+- Only works with numerical data, thus ignoring non-numerical or categorical data.
+- Does not analyze relationships between different features, only the individual feature distributions.
+- Is a univariate analysis and may miss patterns or anomalies that only appear when considering multiple variables together.
+- Does not provide any insight into how these features affect the output of the model; it is purely an input analysis tool.
diff --git a/docs/validmind/tests/data_validation/TargetRateBarPlots.qmd b/docs/validmind/tests/data_validation/TargetRateBarPlots.qmd
new file mode 100644
index 000000000..d055f8bd2
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TargetRateBarPlots.qmd
@@ -0,0 +1,49 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TargetRateBarPlots"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TargetRateBarPlots
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'visualization', 'categorical_data')
+
+@tasks('classification')
+
+defTargetRateBarPlots(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Generates bar plots visualizing the default rates of categorical features for a classification machine learning model.
+
+### Purpose
+
+This test, implemented as a metric, is designed to provide an intuitive, graphical summary of the decision-making patterns exhibited by a categorical classification machine learning model. The model's performance is evaluated using bar plots depicting the ratio of target rates—meaning the proportion of positive classes—for different categorical inputs. This allows for an easy, at-a-glance understanding of the model's accuracy.
+
+### Test Mechanism
+
+The test involves creating a pair of bar plots for each categorical feature in the dataset. The first plot depicts the frequency of each category in the dataset, with each category visually distinguished by its unique color. The second plot shows the mean target rate of each category (sourced from the "default_column"). Plotly, a Python library, is used to generate these plots, with distinct plots created for each feature. If no specific columns are selected, the test will generate plots for each categorical column in the dataset.
+
+### Signs of High Risk
+
+- Inconsistent or non-binary values in the "default_column" could complicate or render impossible the calculation of average target rates.
+- Particularly low or high target rates for a specific category might suggest that the model is misclassifying instances of that category.
+
+### Strengths
+
+- This test offers a visually interpretable breakdown of the model's decisions, providing an easy way to spot irregularities, inconsistencies, or patterns.
+- Its flexibility allows for the inspection of one or multiple columns, as needed.
+
+### Limitations
+
+- The readability of the bar plots drops as the number of distinct categories increases in the dataset, which can make them harder to understand and less useful.
diff --git a/docs/validmind/tests/data_validation/TimeSeriesDescription.qmd b/docs/validmind/tests/data_validation/TimeSeriesDescription.qmd
new file mode 100644
index 000000000..d0e8baaed
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TimeSeriesDescription.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TimeSeriesDescription"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TimeSeriesDescription
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'analysis')
+
+@tasks('regression')
+
+defTimeSeriesDescription(dataset):
+
+:::
+
+
+
+Generates a detailed analysis for the provided time series dataset, summarizing key statistics to identify trends, patterns, and data quality issues.
+
+### Purpose
+
+The TimeSeriesDescription function aims to analyze an individual time series by providing a summary of key statistics. This helps in understanding trends, patterns, and data quality issues within the time series.
+
+### Test Mechanism
+
+The function extracts the time series data and provides a summary of key statistics. The dataset is expected to have a datetime index. The function checks this and raises an error if the index is not in datetime format. For each variable (column) in the dataset, appropriate statistics including start date, end date, frequency, number of missing values, count, min, and max values are calculated.
+
+### Signs of High Risk
+
+- If the index of the dataset is not in datetime format, it could lead to errors in time-series analysis.
+- Inconsistent or missing data within the dataset might affect the analysis of trends and patterns.
+
+### Strengths
+
+- Provides a comprehensive summary of key statistics for each variable, helping to identify data quality issues such as missing values.
+- Helps in understanding the distribution and range of the data by including min and max values.
+
+### Limitations
+
+- Assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access the pandas DataFrame.
+- Only analyzes datasets with a datetime index and will raise an error for other types of indices.
+- Does not handle large datasets efficiently; performance may degrade with very large datasets.
diff --git a/docs/validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.qmd b/docs/validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.qmd
new file mode 100644
index 000000000..75dddb44e
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TimeSeriesDescriptiveStatistics"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TimeSeriesDescriptiveStatistics
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'analysis')
+
+@tasks('regression')
+
+defTimeSeriesDescriptiveStatistics(dataset):
+
+:::
+
+
+
+Evaluates the descriptive statistics of a time series dataset to identify trends, patterns, and data quality issues.
+
+### Purpose
+
+The purpose of the TimeSeriesDescriptiveStatistics function is to analyze an individual time series by providing a summary of key descriptive statistics. This analysis helps in understanding trends, patterns, and data quality issues within the time series dataset.
+
+### Test Mechanism
+
+The function extracts the time series data and provides a summary of key descriptive statistics. The dataset is expected to have a datetime index, and the function will check this and raise an error if the index is not in a datetime format. For each variable (column) in the dataset, appropriate statistics, including start date, end date, min, mean, max, skewness, kurtosis, and count, are calculated.
+
+### Signs of High Risk
+
+- If the index of the dataset is not in datetime format, it could lead to errors in time-series analysis.
+- Inconsistent or missing data within the dataset might affect the analysis of trends and patterns.
+
+### Strengths
+
+- Provides a comprehensive summary of key descriptive statistics for each variable.
+- Helps identify data quality issues and understand the distribution of the data.
+
+### Limitations
+
+- Assumes the dataset is provided as a DataFrameDataset object with a .df attribute to access the pandas DataFrame.
+- Only analyzes datasets with a datetime index and will raise an error for other types of indices.
+- Does not handle large datasets efficiently, and performance may degrade with very large datasets.
diff --git a/docs/validmind/tests/data_validation/TimeSeriesFrequency.qmd b/docs/validmind/tests/data_validation/TimeSeriesFrequency.qmd
new file mode 100644
index 000000000..e5cd2a3e6
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TimeSeriesFrequency.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TimeSeriesFrequency"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TimeSeriesFrequency
+
+
+
+::: {.signature}
+
+@tags('time_series_data')
+
+@tasks('regression')
+
+defTimeSeriesFrequency(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Evaluates consistency of time series data frequency and generates a frequency plot.
+
+### Purpose
+
+The purpose of the TimeSeriesFrequency test is to evaluate the consistency in the frequency of data points in a time-series dataset. This test inspects the intervals or duration between each data point to determine if a fixed pattern (such as daily, weekly, or monthly) exists. The identification of such patterns is crucial to time-series analysis as any irregularities could lead to erroneous results and hinder the model's capacity for identifying trends and patterns.
+
+### Test Mechanism
+
+Initially, the test checks if the dataframe index is in datetime format. Subsequently, it utilizes pandas' `infer_freq` method to identify the frequency of each data series within the dataframe. The `infer_freq` method attempts to establish the frequency of a time series and returns both the frequency string and a dictionary relating these strings to their respective labels. The test compares the frequencies of all datasets. If they share a common frequency, the test passes, but it fails if they do not. Additionally, Plotly is used to create a frequency plot, offering a visual depiction of the time differences between consecutive entries in the dataframe index.
+
+### Signs of High Risk
+
+- The test fails, indicating multiple unique frequencies within the dataset. This failure could suggest irregular intervals between observations, potentially interrupting pattern recognition or trend analysis.
+- The presence of missing or null frequencies could be an indication of inconsistencies in data or gaps within the data collection process.
+
+### Strengths
+
+- This test uses a systematic approach to checking the consistency of data frequency within a time-series dataset.
+- It increases the model's reliability by asserting the consistency of observations over time, an essential factor in time-series analysis.
+- The test generates a visual plot, providing an intuitive representation of the dataset's frequency distribution, which caters to visual learners and aids in interpretation and explanation.
+
+### Limitations
+
+- This test is only applicable to time-series datasets and hence not suitable for other types of datasets.
+- The `infer_freq` method might not always correctly infer frequency when faced with missing or irregular data points.
+- Depending on context or the model under development, mixed frequencies might sometimes be acceptable, but this test considers them a failing condition.
diff --git a/docs/validmind/tests/data_validation/TimeSeriesHistogram.qmd b/docs/validmind/tests/data_validation/TimeSeriesHistogram.qmd
new file mode 100644
index 000000000..a59e10f23
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TimeSeriesHistogram.qmd
@@ -0,0 +1,55 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TimeSeriesHistogram"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TimeSeriesHistogram
+
+
+
+::: {.signature}
+
+@tags('data_validation', 'visualization', 'time_series_data')
+
+@tasks('regression', 'time_series_forecasting')
+
+defTimeSeriesHistogram(dataset,nbins=30):
+
+:::
+
+
+
+Visualizes distribution of time-series data using histograms and Kernel Density Estimation (KDE) lines.
+
+### Purpose
+
+The TimeSeriesHistogram test aims to perform a histogram analysis on time-series data to assess the distribution of values within a dataset over time. This test is useful for regression tasks and can be applied to various types of data, such as internet traffic, stock prices, and weather data, providing insights into the probability distribution, skewness, and kurtosis of the dataset.
+
+### Test Mechanism
+
+This test operates on a specific column within the dataset that must have a datetime type index. For each column in the dataset, a histogram is created using Plotly's histplot function. If the dataset includes more than one time-series, a distinct histogram is plotted for each series. Additionally, a Kernel Density Estimate (KDE) line is drawn for each histogram, visualizing the data's underlying probability distribution. The x and y-axis labels are hidden to focus solely on the data distribution.
+
+### Signs of High Risk
+
+- The dataset lacks a column with a datetime type index.
+- The specified columns do not exist within the dataset.
+- High skewness or kurtosis in the data distribution, indicating potential bias.
+- Presence of significant outliers in the data distribution.
+
+### Strengths
+
+- Serves as a visual diagnostic tool for understanding data behavior and distribution trends.
+- Effective for analyzing both single and multiple time-series data.
+- KDE line provides a smooth estimate of the overall trend in data distribution.
+
+### Limitations
+
+- Provides a high-level view without specific numeric measures such as skewness or kurtosis.
+- The histogram loses some detail due to binning of data values.
+- Cannot handle non-numeric data columns.
+- Histogram shape may be sensitive to the number of bins used.
diff --git a/docs/validmind/tests/data_validation/TimeSeriesLinePlot.qmd b/docs/validmind/tests/data_validation/TimeSeriesLinePlot.qmd
new file mode 100644
index 000000000..c467e9467
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TimeSeriesLinePlot.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TimeSeriesLinePlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TimeSeriesLinePlot
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'visualization')
+
+@tasks('regression')
+
+defTimeSeriesLinePlot(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Generates and analyses time-series data through line plots revealing trends, patterns, anomalies over time.
+
+### Purpose
+
+The TimeSeriesLinePlot metric is designed to generate and analyze time series data through the creation of line plots. This assists in the initial inspection of the data by providing a visual representation of patterns, trends, seasonality, irregularity, and anomalies that may be present in the dataset over a period of time.
+
+### Test Mechanism
+
+The mechanism for this Python class involves extracting the column names from the provided dataset and subsequently generating line plots for each column using the Plotly Python library. For every column in the dataset, a time-series line plot is created where the values are plotted against the dataset's datetime index. It is important to note that indexes that are not of datetime type will result in a ValueError.
+
+### Signs of High Risk
+
+- Presence of time-series data that does not have datetime indices.
+- Provided columns do not exist in the provided dataset.
+- The detection of anomalous patterns or irregularities in the time-series plots, indicating potential high model instability or probable predictive error.
+
+### Strengths
+
+- The visual representation of complex time series data, which simplifies understanding and helps in recognizing temporal trends, patterns, and anomalies.
+- The adaptability of the metric, which allows it to effectively work with multiple time series within the same dataset.
+- Enables the identification of anomalies and irregular patterns through visual inspection, assisting in spotting potential data or model performance problems.
+
+### Limitations
+
+- The effectiveness of the metric is heavily reliant on the quality and patterns of the provided time series data.
+- Exclusively a visual tool, it lacks the capability to provide quantitative measurements, making it less effective for comparing and ranking multiple models or when specific numerical diagnostics are needed.
+- The metric necessitates that the time-specific data has been transformed into a datetime index, with the data formatted correctly.
+- The metric has an inherent limitation in that it cannot extract deeper statistical insights from the time series data, which can limit its efficacy with complex data structures and phenomena.
diff --git a/docs/validmind/tests/data_validation/TimeSeriesMissingValues.qmd b/docs/validmind/tests/data_validation/TimeSeriesMissingValues.qmd
new file mode 100644
index 000000000..396864c8e
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TimeSeriesMissingValues.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TimeSeriesMissingValues"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TimeSeriesMissingValues
+
+
+
+::: {.signature}
+
+@tags('time_series_data')
+
+@tasks('regression')
+
+defTimeSeriesMissingValues(dataset:validmind.vm_models.VMDataset,min_threshold:int=1):
+
+:::
+
+
+
+Validates time-series data quality by confirming the count of missing values is below a certain threshold.
+
+### Purpose
+
+This test is designed to validate the quality of a historical time-series dataset by verifying that the number of missing values is below a specified threshold. As time-series models greatly depend on the continuity and temporality of data points, missing values could compromise the model's performance. Consequently, this test aims to ensure data quality and readiness for the machine learning model, safeguarding its predictive capacity.
+
+### Test Mechanism
+
+The test method commences by validating if the dataset has a datetime index; if not, an error is raised. It establishes a lower limit threshold for missing values and performs a missing values check on each column of the dataset. An object for the test result is created stating whether the number of missing values is within the specified threshold. Additionally, the test calculates the percentage of missing values alongside the raw count.
+
+### Signs of High Risk
+
+- The number of missing values in any column of the dataset surpasses the threshold, marking a failure and a high-risk scenario. The reasons could range from incomplete data collection, faulty sensors to data preprocessing errors.
+
+### Strengths
+
+- Effectively identifies missing values which could adversely affect the model’s performance.
+- Applicable and customizable through the threshold parameter across different data sets.
+- Goes beyond raw numbers by calculating the percentage of missing values, offering a more relative understanding of data scarcity.
+
+### Limitations
+
+- Although it identifies missing values, the test does not provide solutions to handle them.
+- The test demands that the dataset should have a datetime index, hence limiting its use only to time series analysis.
+- The test's sensitivity to the 'min_threshold' parameter may raise false alarms if set too strictly or may overlook problematic data if set too loosely.
+- Solely focuses on the 'missingness' of the data and might fall short in addressing other aspects of data quality.
diff --git a/docs/validmind/tests/data_validation/TimeSeriesOutliers.qmd b/docs/validmind/tests/data_validation/TimeSeriesOutliers.qmd
new file mode 100644
index 000000000..75c54077f
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TimeSeriesOutliers.qmd
@@ -0,0 +1,57 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TimeSeriesOutliers"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TimeSeriesOutliers
+
+
+
+::: {.signature}
+
+@tags('time_series_data')
+
+@tasks('regression')
+
+defTimeSeriesOutliers(dataset:validmind.vm_models.VMDataset,zscore_threshold:int=3):
+
+:::
+
+
+
+Identifies and visualizes outliers in time-series data using the z-score method.
+
+### Purpose
+
+This test is designed to identify outliers in time-series data using the z-score method. It's vital for ensuring data quality before modeling, as outliers can skew predictive models and significantly impact their overall performance.
+
+### Test Mechanism
+
+The test processes a given dataset which must have datetime indexing, checks if a 'zscore_threshold' parameter has been supplied, and identifies columns with numeric data types. After finding numeric columns, the implementer then applies the z-score method to each numeric column, identifying outliers based on the threshold provided. Each outlier is listed together with their variable name, z-score, timestamp, and relative threshold in a dictionary and converted to a DataFrame for convenient output. Additionally, it produces visual plots for each time series illustrating outliers in the context of the broader dataset. The 'zscore_threshold' parameter sets the limit beyond which a data point will be labeled as an outlier. The default threshold is set at 3, indicating that any data point that falls 3 standard deviations away from the mean will be marked as an outlier.
+
+### Signs of High Risk
+
+- Many or substantial outliers are present within the dataset, indicating significant anomalies.
+- Data points with z-scores higher than the set threshold.
+- Potential impact on the performance of machine learning models if outliers are not properly addressed.
+
+### Strengths
+
+- The z-score method is a popular and robust method for identifying outliers in a dataset.
+- Simplifies time series maintenance by requiring a datetime index.
+- Identifies outliers for each numeric feature individually.
+- Provides an elaborate report showing variables, dates, z-scores, and pass/fail tests.
+- Offers visual inspection for detected outliers through plots.
+
+### Limitations
+
+- The test only identifies outliers in numeric columns, not in categorical variables.
+- The utility and accuracy of z-scores can be limited if the data doesn't follow a normal distribution.
+- The method relies on a subjective z-score threshold for deciding what constitutes an outlier, which might not always be suitable depending on the dataset and use case.
+- It does not address possible ways to handle identified outliers in the data.
+- The requirement for a datetime index could limit its application.
diff --git a/docs/validmind/tests/data_validation/TooManyZeroValues.qmd b/docs/validmind/tests/data_validation/TooManyZeroValues.qmd
new file mode 100644
index 000000000..c01535e6a
--- /dev/null
+++ b/docs/validmind/tests/data_validation/TooManyZeroValues.qmd
@@ -0,0 +1,55 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TooManyZeroValues"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TooManyZeroValues
+
+
+
+::: {.signature}
+
+@tags('tabular_data')
+
+@tasks('regression', 'classification')
+
+defTooManyZeroValues(dataset:validmind.vm_models.VMDataset,max_percent_threshold:float=0.03):
+
+:::
+
+
+
+Identifies numerical columns in a dataset that contain an excessive number of zero values, defined by a threshold percentage.
+
+### Purpose
+
+The 'TooManyZeroValues' test is utilized to identify numerical columns in the dataset that may present a quantity of zero values considered excessive. The aim is to detect situations where these may implicate data sparsity or a lack of variation, limiting their effectiveness within a machine learning model. The definition of 'too many' is quantified as a percentage of total values, with a default set to 3%.
+
+### Test Mechanism
+
+This test is conducted by looping through each column in the dataset and categorizing those that pertain to numerical data. On identifying a numerical column, the function computes the total quantity of zero values and their ratio to the total row count. Should the proportion exceed a pre-set threshold parameter, set by default at 0.03 or 3%, the column is considered to have failed the test. The results for each column are summarized and reported, indicating the count and percentage of zero values for each numerical column, alongside a status indicating whether the column has passed or failed the test.
+
+### Signs of High Risk
+
+- Numerical columns showing a high ratio of zero values when compared to the total count of rows (exceeding the predetermined threshold).
+- Columns characterized by zero values across the board suggest a complete lack of data variation, signifying high risk.
+
+### Strengths
+
+- Assists in highlighting columns featuring an excess of zero values that could otherwise go unnoticed within a large dataset.
+- Provides the flexibility to alter the threshold that determines when the quantity of zero values becomes 'too many', thus catering to specific needs of a particular analysis or model.
+- Offers feedback in the form of both counts and percentages of zero values, which allows a closer inspection of the distribution and proportion of zeros within a column.
+- Targets specifically numerical data, thereby avoiding inappropriate application to non-numerical columns and mitigating the risk of false test failures.
+
+### Limitations
+
+- Is exclusively designed to check for zero values and doesn’t assess the potential impact of other values that could affect the dataset, such as extremely high or low figures, missing values, or outliers.
+- Lacks the ability to detect a repetitive pattern of zeros, which could be significant in time-series or longitudinal data.
+- Zero values can actually be meaningful in some contexts; therefore, tagging them as 'too many' could potentially misinterpret the data to some extent.
+- This test does not take into consideration the context of the dataset, and fails to recognize that within certain columns, a high number of zero values could be quite normal and not necessarily an indicator of poor data quality.
+- Cannot evaluate non-numerical or categorical columns, which might bring with them different types of concerns or issues.
diff --git a/docs/validmind/tests/data_validation/UniqueRows.qmd b/docs/validmind/tests/data_validation/UniqueRows.qmd
new file mode 100644
index 000000000..1bb43a4a8
--- /dev/null
+++ b/docs/validmind/tests/data_validation/UniqueRows.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).UniqueRows"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## UniqueRows
+
+
+
+::: {.signature}
+
+@tags('tabular_data')
+
+@tasks('regression', 'classification')
+
+defUniqueRows(dataset:validmind.vm_models.VMDataset,min_percent_threshold:float=1):
+
+:::
+
+
+
+Verifies the diversity of the dataset by ensuring that the count of unique rows exceeds a prescribed threshold.
+
+### Purpose
+
+The UniqueRows test is designed to gauge the quality of the data supplied to the machine learning model by verifying that the count of distinct rows in the dataset exceeds a specific threshold, thereby ensuring a varied collection of data. Diversity in data is essential for training an unbiased and robust model that excels when faced with novel data.
+
+### Test Mechanism
+
+The testing process starts with calculating the total number of rows in the dataset. Subsequently, the count of unique rows is determined for each column in the dataset. If the percentage of unique rows (calculated as the ratio of unique rows to the overall row count) is less than the prescribed minimum percentage threshold given as a function parameter, the test passes. The results are cached and a final pass or fail verdict is given based on whether all columns have successfully passed the test.
+
+### Signs of High Risk
+
+- A lack of diversity in data columns, demonstrated by a count of unique rows that falls short of the preset minimum percentage threshold, is indicative of high risk.
+- This lack of variety in the data signals potential issues with data quality, possibly leading to overfitting in the model and issues with generalization, thus posing a significant risk.
+
+### Strengths
+
+- The UniqueRows test is efficient in evaluating the data's diversity across each information column in the dataset.
+- This test provides a quick, systematic method to assess data quality based on uniqueness, which can be pivotal in developing effective and unbiased machine learning models.
+
+### Limitations
+
+- A limitation of the UniqueRows test is its assumption that the data's quality is directly proportionate to its uniqueness, which may not always hold true. There might be contexts where certain non-unique rows are essential and should not be overlooked.
+- The test does not consider the relative 'importance' of each column in predicting the output, treating all columns equally.
+- This test may not be suitable or useful for categorical variables, where the count of unique categories is inherently limited.
diff --git a/docs/validmind/tests/data_validation/WOEBinPlots.qmd b/docs/validmind/tests/data_validation/WOEBinPlots.qmd
new file mode 100644
index 000000000..f11d7ec8f
--- /dev/null
+++ b/docs/validmind/tests/data_validation/WOEBinPlots.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).WOEBinPlots"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## WOEBinPlots
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'visualization', 'categorical_data')
+
+@tasks('classification')
+
+defWOEBinPlots(dataset:validmind.vm_models.VMDataset,breaks_adj:list=None,fig_height:int=600,fig_width:int=500):
+
+:::
+
+
+
+Generates visualizations of Weight of Evidence (WoE) and Information Value (IV) for understanding predictive power of categorical variables in a data set.
+
+### Purpose
+
+This test is designed to visualize the Weight of Evidence (WoE) and Information Value (IV) for categorical variables in a provided dataset. By showcasing the data distribution across different categories of each feature, it aids in understanding each variable's predictive power in the context of a classification-based machine learning model. Commonly used in credit scoring models, WoE and IV are robust statistical methods for evaluating a variable's predictive power.
+
+### Test Mechanism
+
+The test implementation follows defined steps. Initially, it selects non-numeric columns from the dataset and changes them to string type, paving the way for accurate binning. It then performs an automated WoE binning operation on these selected features, effectively categorizing the potential values of a variable into distinct bins. After the binning process, the function generates two separate visualizations (a scatter chart for WoE values and a bar chart for IV) for each variable. These visual presentations are formed according to the spread of each metric across various categories of each feature.
+
+### Signs of High Risk
+
+- Errors occurring during the binning process.
+- Challenges in converting non-numeric columns into string data type.
+- Misbalance in the distribution of WoE and IV, with certain bins overtaking others conspicuously. This could denote that the model is disproportionately dependent on certain variables or categories for predictions, an indication of potential risks to its robustness and generalizability.
+
+### Strengths
+
+- Provides a detailed visual representation of the relationship between feature categories and the target variable. This grants an intuitive understanding of each feature's contribution to the model.
+- Allows for easy identification of features with high impact, facilitating feature selection and enhancing comprehension of the model's decision logic.
+- WoE conversions are monotonic, upholding the rank ordering of the original data points, which simplifies analysis.
+
+### Limitations
+
+- The method is largely reliant on the binning process, and an inappropriate binning threshold or bin number choice might result in a misrepresentation of the variable's distribution.
+- While excellent for categorical data, the encoding of continuous variables into categorical can sometimes lead to information loss.
+- Extreme or outlier values can dramatically affect the computation of WoE and IV, skewing results.
+- The method requires a sufficient number of events per bin to generate a reliable information value and weight of evidence.
diff --git a/docs/validmind/tests/data_validation/WOEBinTable.qmd b/docs/validmind/tests/data_validation/WOEBinTable.qmd
new file mode 100644
index 000000000..70b1292d0
--- /dev/null
+++ b/docs/validmind/tests/data_validation/WOEBinTable.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).WOEBinTable"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## WOEBinTable
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'categorical_data')
+
+@tasks('classification')
+
+defWOEBinTable(dataset:validmind.vm_models.VMDataset,breaks_adj:list=None):
+
+:::
+
+
+
+Assesses the Weight of Evidence (WoE) and Information Value (IV) of each feature to evaluate its predictive power in a binary classification model.
+
+### Purpose
+
+The Weight of Evidence (WoE) and Information Value (IV) test is designed to evaluate the predictive power of each feature in a machine learning model. This test generates binned groups of values from each feature, computes the WoE and IV for each bin, and provides insights into the relationship between each feature and the target variable, illustrating their contribution to the model's predictive capabilities.
+
+### Test Mechanism
+
+The test uses the `scorecardpy.woebin` method to perform automatic binning of the dataset based on WoE. The method accepts a list of break points for binning numeric variables through the parameter `breaks_adj`. If no breaks are provided, it uses default binning. The bins are then used to calculate the WoE and IV values, effectively creating a dataframe that includes the bin boundaries, WoE, and IV values for each feature. A target variable is required in the dataset to perform this analysis.
+
+### Signs of High Risk
+
+- High IV values, indicating variables with excessive predictive power which might lead to overfitting.
+- Errors during the binning process, potentially due to inappropriate data types or poorly defined bins.
+
+### Strengths
+
+- Highly effective for feature selection in binary classification problems, as it quantifies the predictive information within each feature concerning the binary outcome.
+- The WoE transformation creates a monotonic relationship between the target and independent variables.
+
+### Limitations
+
+- Primarily designed for binary classification tasks, making it less applicable or reliable for multi-class classification or regression tasks.
+- Potential difficulties if the dataset has many features, non-binnable features, or non-numeric features.
+- The metric does not help in distinguishing whether the observed predictive factor is due to data randomness or a true phenomenon.
diff --git a/docs/validmind/tests/data_validation/ZivotAndrewsArch.qmd b/docs/validmind/tests/data_validation/ZivotAndrewsArch.qmd
new file mode 100644
index 000000000..11030f277
--- /dev/null
+++ b/docs/validmind/tests/data_validation/ZivotAndrewsArch.qmd
@@ -0,0 +1,50 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ZivotAndrewsArch"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ZivotAndrewsArch
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'stationarity', 'unit_root_test')
+
+@tasks('regression')
+
+defZivotAndrewsArch(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Evaluates the order of integration and stationarity of time series data using the Zivot-Andrews unit root test.
+
+### Purpose
+
+The Zivot-Andrews Arch metric is used to evaluate the order of integration for time series data in a machine learning model. It's designed to test for stationarity, a crucial aspect of time series analysis, where data points are independent of time. Stationarity means that the statistical properties such as mean, variance, and autocorrelation are constant over time.
+
+### Test Mechanism
+
+The Zivot-Andrews unit root test is performed on each feature in the dataset using the `ZivotAndrews` function from the `arch.unitroot` module. This function returns several metrics for each feature, including the statistical value, p-value (probability value), the number of lags used, and the number of observations. The p-value is used to decide on the null hypothesis (the time series has a unit root and is non-stationary) based on a chosen level of significance.
+
+### Signs of High Risk
+
+- A high p-value suggests high risk, indicating insufficient evidence to reject the null hypothesis, implying that the time series has a unit root and is non-stationary.
+- Non-stationary time series data can lead to misleading statistics and unreliable machine learning models.
+
+### Strengths
+
+- Dynamically tests for stationarity against structural breaks in time series data, offering robust evaluation of stationarity in features.
+- Especially beneficial with financial, economic, or other time-series data where data observations lack a consistent pattern and structural breaks may occur.
+
+### Limitations
+
+- Assumes data is derived from a single-equation, autoregressive model, making it less appropriate for multivariate time series data or data not aligning with this model.
+- May not account for unexpected shocks or changes in the series trend, both of which can significantly impact data stationarity.
diff --git a/docs/validmind/tests/data_validation/nlp.qmd b/docs/validmind/tests/data_validation/nlp.qmd
new file mode 100644
index 000000000..4d990c58d
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp.qmd
@@ -0,0 +1,18 @@
+---
+title: "[validmind](/validmind/validmind.qmd).nlp"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+- [CommonWords](nlp/CommonWords.qmd)
+- [Hashtags](nlp/Hashtags.qmd)
+- [LanguageDetection](nlp/LanguageDetection.qmd)
+- [Mentions](nlp/Mentions.qmd)
+- [PolarityAndSubjectivity](nlp/PolarityAndSubjectivity.qmd)
+- [Punctuations](nlp/Punctuations.qmd)
+- [Sentiment](nlp/Sentiment.qmd)
+- [StopWords](nlp/StopWords.qmd)
+- [TextDescription](nlp/TextDescription.qmd)
+- [Toxicity](nlp/Toxicity.qmd)
diff --git a/docs/validmind/tests/data_validation/nlp/CommonWords.qmd b/docs/validmind/tests/data_validation/nlp/CommonWords.qmd
new file mode 100644
index 000000000..a2e036f51
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp/CommonWords.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).CommonWords"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## CommonWords
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization', 'frequency_analysis')
+
+@tasks('text_classification', 'text_summarization')
+
+defCommonWords(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Assesses the most frequent non-stopwords in a text column for identifying prevalent language patterns.
+
+### Purpose
+
+The CommonWords metric is used to identify and visualize the most prevalent words within a specified text column of a dataset. This provides insights into the prevalent language patterns and vocabulary, especially useful in Natural Language Processing (NLP) tasks such as text classification and text summarization.
+
+### Test Mechanism
+
+The test methodology involves splitting the specified text column's entries into words, collating them into a corpus, and then counting the frequency of each word using the Counter. The forty most frequently occurring non-stopwords are then visualized in an interactive bar chart using Plotly, where the x-axis represents the words, and the y-axis indicates their frequency of occurrence.
+
+### Signs of High Risk
+
+- A lack of distinct words within the list, or the most common words being stopwords.
+- Frequent occurrence of irrelevant or inappropriate words could point out a poorly curated or noisy dataset.
+- An error returned due to the absence of a valid Dataset object, indicating high risk as the metric cannot be effectively implemented without it.
+
+### Strengths
+
+- The metric provides clear insights into the language features – specifically word frequency – of unstructured text data.
+- It can reveal prominent vocabulary and language patterns, which prove vital for feature extraction in NLP tasks.
+- The interactive visualization helps in quickly capturing the patterns and understanding the data intuitively.
+
+### Limitations
+
+- The test disregards semantic or context-related information as it solely focuses on word frequency.
+- It intentionally ignores stopwords, which might carry necessary significance in certain scenarios.
+- The applicability is limited to English-language text data as English stopwords are used for filtering, hence cannot account for data in other languages.
+- The metric requires a valid Dataset object, indicating a dependency condition that limits its broader applicability.
diff --git a/docs/validmind/tests/data_validation/nlp/Hashtags.qmd b/docs/validmind/tests/data_validation/nlp/Hashtags.qmd
new file mode 100644
index 000000000..ca9beeea3
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp/Hashtags.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Hashtags"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Hashtags
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization', 'frequency_analysis')
+
+@tasks('text_classification', 'text_summarization')
+
+defHashtags(dataset:validmind.vm_models.VMDataset,top_hashtags:int=25):
+
+:::
+
+
+
+Assesses hashtag frequency in a text column, highlighting usage trends and potential dataset bias or spam.
+
+### Purpose
+
+The Hashtags test is designed to measure the frequency of hashtags used within a given text column in a dataset. It is particularly useful for natural language processing tasks such as text classification and text summarization. The goal is to identify common trends and patterns in the use of hashtags, which can serve as critical indicators or features within a machine learning model.
+
+### Test Mechanism
+
+The test implements a regular expression (regex) to extract all hashtags from the specified text column. For each hashtag found, it makes a tally of its occurrences. It then outputs a list of the top N hashtags (default is 25, but customizable), sorted by their counts in descending order. The results are also visualized in a bar plot, with frequency counts on the y-axis and the corresponding hashtags on the x-axis.
+
+### Signs of High Risk
+
+- A low diversity in the usage of hashtags, as indicated by a few hashtags being used disproportionately more than others.
+- Repeated usage of one or few hashtags can be indicative of spam or a biased dataset.
+- If there are no or extremely few hashtags found in the dataset, it perhaps signifies that the text data does not contain structured social media data.
+
+### Strengths
+
+- Provides a concise visual representation of the frequency of hashtags, which can be critical for understanding trends about a particular topic in text data.
+- Instrumental in tasks specifically related to social media text analytics, such as opinion analysis and trend discovery.
+- Adaptable, allowing the flexibility to determine the number of top hashtags to be analyzed.
+
+### Limitations
+
+- Assumes the presence of hashtags and therefore may not be applicable for text datasets that do not contain hashtags (e.g., formal documents, scientific literature).
+- Language-specific limitations of hashtag formulations are not taken into account.
+- Does not account for typographical errors, variations, or synonyms in hashtags.
+- Does not provide context or sentiment associated with the hashtags, so the information provided may have limited utility on its own.
diff --git a/docs/validmind/tests/data_validation/nlp/LanguageDetection.qmd b/docs/validmind/tests/data_validation/nlp/LanguageDetection.qmd
new file mode 100644
index 000000000..33023a95d
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp/LanguageDetection.qmd
@@ -0,0 +1,59 @@
+---
+title: "[validmind](/validmind/validmind.qmd).LanguageDetection"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## LanguageDetection
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defLanguageDetection(dataset):
+
+:::
+
+
+
+Assesses the diversity of languages in a textual dataset by detecting and visualizing the distribution of languages.
+
+### Purpose
+
+The Language Detection test aims to identify and visualize the distribution of languages present within a textual dataset. This test helps in understanding the diversity of languages in the data, which is crucial for developing and validating multilingual models.
+
+### Test Mechanism
+
+This test operates by:
+
+- Checking if the dataset has a specified text column.
+- Using a language detection library to determine the language of each text entry in the dataset.
+- Generating a histogram plot of the language distribution, with language codes on the x-axis and their frequencies on the y-axis.
+
+If the text column is not specified, a ValueError is raised to ensure proper dataset configuration.
+
+### Signs of High Risk
+
+- A high proportion of entries returning "Unknown" language codes.
+- Detection of unexpectedly diverse or incorrect language codes, indicating potential data quality issues.
+- Significant imbalance in language distribution, which might indicate potential biases in the dataset.
+
+### Strengths
+
+- Provides a visual representation of language diversity within the dataset.
+- Helps identify data quality issues related to incorrect or unknown language detection.
+- Useful for ensuring that multilingual models have adequate and appropriate representation from various languages.
+
+### Limitations
+
+- Dependency on the accuracy of the language detection library, which may not be perfect.
+- Languages with similar structures or limited text length may be incorrectly classified.
+- The test returns "Unknown" for entries where language detection fails, which might mask underlying issues with certain languages or text formats.
diff --git a/docs/validmind/tests/data_validation/nlp/Mentions.qmd b/docs/validmind/tests/data_validation/nlp/Mentions.qmd
new file mode 100644
index 000000000..6e4f4069a
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp/Mentions.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Mentions"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Mentions
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization', 'frequency_analysis')
+
+@tasks('text_classification', 'text_summarization')
+
+defMentions(dataset:validmind.vm_models.VMDataset,top_mentions:int=25):
+
+:::
+
+
+
+Calculates and visualizes frequencies of '@' prefixed mentions in a text-based dataset for NLP model analysis.
+
+### Purpose
+
+The "Mentions" test is designed to gauge the quality of data in a Natural Language Processing (NLP) or text-focused Machine Learning model. The primary objective is to identify and calculate the frequency of 'mentions' within a chosen text column of a dataset. A 'mention' in this context refers to individual text elements that are prefixed by '@'. The output of this test reveals the most frequently mentioned entities or usernames, which can be integral for applications such as social media analyses or customer sentiment analyses.
+
+### Test Mechanism
+
+The test first verifies the existence of a text column in the provided dataset. It then employs a regular expression pattern to extract mentions from the text. Subsequently, the frequency of each unique mention is calculated. The test selects the most frequent mentions based on default or user-defined parameters, the default being the top 25, for representation. This process of thresholding forms the core of the test. A treemap plot visualizes the test results, where the size of each rectangle corresponds to the frequency of a particular mention.
+
+### Signs of High Risk
+
+- The lack of a valid text column in the dataset, which would result in the failure of the test execution.
+- The absence of any mentions within the text data, indicating that there might not be any text associated with '@'. This situation could point toward sparse or poor-quality data, thereby hampering the model's generalization or learning capabilities.
+
+### Strengths
+
+- The test is specifically optimized for text-based datasets which gives it distinct power in the context of NLP.
+- It enables quick identification and visually appealing representation of the predominant elements or mentions.
+- It can provide crucial insights about the most frequently mentioned entities or usernames.
+
+### Limitations
+
+- The test only recognizes mentions that are prefixed by '@', hence useful textual aspects not preceded by '@' might be ignored.
+- This test isn't suited for datasets devoid of textual data.
+- It does not provide insights on less frequently occurring data or outliers, which means potentially significant patterns could be overlooked.
diff --git a/docs/validmind/tests/data_validation/nlp/PolarityAndSubjectivity.qmd b/docs/validmind/tests/data_validation/nlp/PolarityAndSubjectivity.qmd
new file mode 100644
index 000000000..fb166026a
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp/PolarityAndSubjectivity.qmd
@@ -0,0 +1,57 @@
+---
+title: "[validmind](/validmind/validmind.qmd).PolarityAndSubjectivity"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## PolarityAndSubjectivity
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'data_validation')
+
+@tasks('nlp')
+
+defPolarityAndSubjectivity(dataset,threshold_subjectivity=0.5,threshold_polarity=0):
+
+:::
+
+
+
+Analyzes the polarity and subjectivity of text data within a given dataset to visualize the sentiment distribution.
+
+### Purpose
+
+The Polarity and Subjectivity test is designed to evaluate the sentiment expressed in textual data. By analyzing these aspects, it helps to identify the emotional tone and subjectivity of the dataset, which could be crucial in understanding customer feedback, social media sentiments, or other text-related data.
+
+### Test Mechanism
+
+This test uses TextBlob to compute the polarity and subjectivity scores of textual data in a given dataset. The mechanism includes:
+
+- Iterating through each text entry in the specified column of the dataset.
+- Applying the TextBlob library to compute the polarity (ranging from -1 for negative sentiment to +1 for positive sentiment) and subjectivity (ranging from 0 for objective to 1 for subjective) for each entry.
+- Creating a scatter plot using Plotly to visualize the relationship between polarity and subjectivity.
+
+### Signs of High Risk
+
+- High concentration of negative polarity values indicating prevalent negative sentiments.
+- High subjectivity scores suggesting the text data is largely opinion-based rather than factual.
+- Disproportionate clusters of extreme scores (e.g., many points near -1 or +1 polarity).
+
+### Strengths
+
+- Quantifies sentiment and subjectivity which can provide actionable insights.
+- Visualizes sentiment distribution, aiding in easy interpretation.
+- Utilizes well-established TextBlob library for sentiment analysis.
+
+### Limitations
+
+- Polarity and subjectivity calculations may oversimplify nuanced text sentiments.
+- Reliance on TextBlob which may not be accurate for all domains or contexts.
+- Visualization could become cluttered with very large datasets, making interpretation difficult.
diff --git a/docs/validmind/tests/data_validation/nlp/Punctuations.qmd b/docs/validmind/tests/data_validation/nlp/Punctuations.qmd
new file mode 100644
index 000000000..4befae9d5
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp/Punctuations.qmd
@@ -0,0 +1,56 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Punctuations"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Metrics functions for any Pandas-compatible datasets
+
+
+
+## Punctuations
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization', 'frequency_analysis')
+
+@tasks('text_classification', 'text_summarization', 'nlp')
+
+defPunctuations(dataset,count_mode='token'):
+
+:::
+
+
+
+Analyzes and visualizes the frequency distribution of punctuation usage in a given text dataset.
+
+### Purpose
+
+The Punctuations Metric's primary purpose is to analyze the frequency of punctuation usage within a given text dataset. This is often used in Natural Language Processing tasks, such as text classification and text summarization.
+
+### Test Mechanism
+
+The test begins by verifying that the input "dataset" is of the type VMDataset. The count_mode parameter must be either "token" (counts punctuation marks as individual tokens) or "word" (counts punctuation marks within words). Following that, a corpus is created from the dataset by splitting its text on spaces. Each unique punctuation character in the text corpus is then tallied. The frequency distribution of each punctuation symbol is visualized as a bar graph, with these results being stored as Figures and associated with the main Punctuations object.
+
+### Signs of High Risk
+
+- Excessive or unusual frequency of specific punctuation marks, potentially denoting dubious quality, data corruption, or skewed data.
+
+### Strengths
+
+- Provides valuable insights into the distribution of punctuation usage in a text dataset.
+- Important in validating the quality, consistency, and nature of the data.
+- Can provide hints about the style or tonality of the text corpus, such as informal and emotional context indicated by frequent exclamation marks.
+
+### Limitations
+
+- Focuses solely on punctuation usage, potentially missing other important textual characteristics.
+- General cultural or tonality assumptions based on punctuation distribution can be misguiding, as these vary across different languages and contexts.
+- Less effective with languages that use non-standard or different punctuation.
+- Visualization may lack interpretability when there are many unique punctuation marks in the dataset.
diff --git a/docs/validmind/tests/data_validation/nlp/Sentiment.qmd b/docs/validmind/tests/data_validation/nlp/Sentiment.qmd
new file mode 100644
index 000000000..d14251e32
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp/Sentiment.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Sentiment"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Sentiment
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'data_validation')
+
+@tasks('nlp')
+
+defSentiment(dataset):
+
+:::
+
+
+
+Analyzes the sentiment of text data within a dataset using the VADER sentiment analysis tool.
+
+### Purpose
+
+The Sentiment test evaluates the overall sentiment of text data within a dataset. By analyzing sentiment scores, it aims to ensure that the model is interpreting text data accurately and is not biased towards a particular sentiment.
+
+### Test Mechanism
+
+This test uses the VADER (Valence Aware Dictionary and sEntiment Reasoner) SentimentIntensityAnalyzer. It processes each text entry in a specified column of the dataset to calculate the compound sentiment score, which represents the overall sentiment polarity. The distribution of these sentiment scores is then visualized using a KDE (Kernel Density Estimation) plot, highlighting any skewness or concentration in sentiment.
+
+### Signs of High Risk
+
+- Extreme polarity in sentiment scores, indicating potential bias.
+- Unusual concentration of sentiment scores in a specific range.
+- Significant deviation from expected sentiment distribution for the given text data.
+
+### Strengths
+
+- Provides a clear visual representation of sentiment distribution.
+- Uses a well-established sentiment analysis tool (VADER).
+- Can handle a wide range of text data, making it flexible for various applications.
+
+### Limitations
+
+- May not capture nuanced or context-specific sentiments.
+- Relies heavily on the accuracy of the VADER sentiment analysis tool.
+- Visualization alone may not provide comprehensive insights into underlying causes of sentiment distribution.
diff --git a/docs/validmind/tests/data_validation/nlp/StopWords.qmd b/docs/validmind/tests/data_validation/nlp/StopWords.qmd
new file mode 100644
index 000000000..8aa52c38d
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp/StopWords.qmd
@@ -0,0 +1,58 @@
+---
+title: "[validmind](/validmind/validmind.qmd).StopWords"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Threshold based tests
+
+
+
+## StopWords
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'frequency_analysis', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defStopWords(dataset:validmind.vm_models.VMDataset,min_percent_threshold:float=0.5,num_words:int=25):
+
+:::
+
+
+
+Evaluates and visualizes the frequency of English stop words in a text dataset against a defined threshold.
+
+### Purpose
+
+The StopWords threshold test is a tool designed for assessing the quality of text data in an ML model. It focuses on the identification and analysis of "stop words" in a given dataset. Stop words are frequent, common, yet semantically insignificant words (for example: "the", "and", "is") in a language. This test evaluates the proportion of stop words to the total word count in the dataset, in essence, scrutinizing the frequency of stop word usage. The core objective is to highlight the prevalent stop words based on their usage frequency, which can be instrumental in cleaning the data from noise and improving ML model performance.
+
+### Test Mechanism
+
+The StopWords test initiates on receiving an input of a 'VMDataset' object. Absence of such an object will trigger an error. The methodology involves inspection of the text column of the VMDataset to create a 'corpus' (a collection of written texts). Leveraging the Natural Language Toolkit's (NLTK) stop word repository, the test screens the corpus for any stop words and documents their frequency. It further calculates the percentage usage of each stop word compared to the total word count in the corpus. This percentage is evaluated against a predefined 'min_percent_threshold'. If this threshold is breached, the test returns a failed output. Top prevailing stop words along with their usage percentages are returned, facilitated by a bar chart visualization of these stop words and their frequency.
+
+### Signs of High Risk
+
+- A percentage of any stop words exceeding the predefined 'min_percent_threshold'.
+- High frequency of stop words in the dataset which may adversely affect the application's analytical performance due to noise creation.
+
+### Strengths
+
+- The ability to scrutinize and quantify the usage of stop words.
+- Provides insights into potential noise in the text data due to stop words.
+- Directly aids in enhancing model training efficiency.
+- Includes a bar chart visualization feature to easily interpret and action upon the stop words frequency information.
+
+### Limitations
+
+- The test only supports English stop words, making it less effective with datasets of other languages.
+- The 'min_percent_threshold' parameter may require fine-tuning for different datasets, impacting the overall effectiveness of the test.
+- Contextual use of the stop words within the dataset is not considered, potentially overlooking their significance in certain contexts.
+- The test focuses specifically on the frequency of stop words, not providing direct measures of model performance or predictive accuracy.
diff --git a/docs/validmind/tests/data_validation/nlp/TextDescription.qmd b/docs/validmind/tests/data_validation/nlp/TextDescription.qmd
new file mode 100644
index 000000000..463d89065
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp/TextDescription.qmd
@@ -0,0 +1,73 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TextDescription"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## create_metrics_df
+
+
+
+::: {.signature}
+
+defcreate_metrics_df(df,text_column,unwanted_tokens,lang):
+
+:::
+
+
+
+## TextDescription
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defTextDescription(dataset:validmind.vm_models.VMDataset,unwanted_tokens:set={'s', "s'", 'mr', 'ms', 'mrs', 'dr', "'s", ' ', "''", 'dollar', 'us', '\`\`'},lang:str='english'):
+
+:::
+
+
+
+Conducts comprehensive textual analysis on a dataset using NLTK to evaluate various parameters and generate visualizations.
+
+### Purpose
+
+The TextDescription test aims to conduct a thorough textual analysis of a dataset using the NLTK (Natural Language Toolkit) library. It evaluates various metrics such as total words, total sentences, average sentence length, total paragraphs, total unique words, most common words, total punctuations, and lexical diversity. The goal is to understand the nature of the text and anticipate challenges machine learning models might face in text processing, language understanding, or summarization tasks.
+
+### Test Mechanism
+
+The test works by:
+
+- Parsing the dataset and tokenizing the text into words, sentences, and paragraphs using NLTK.
+- Removing stopwords and unwanted tokens.
+- Calculating parameters like total words, total sentences, average sentence length, total paragraphs, total unique words, total punctuations, and lexical diversity.
+- Generating scatter plots to visualize correlations between various metrics (e.g., Total Words vs Total Sentences).
+
+### Signs of High Risk
+
+- Anomalies or increased complexity in lexical diversity.
+- Longer sentences and paragraphs.
+- High uniqueness of words.
+- Large number of unwanted tokens.
+- Missing or erroneous visualizations.
+
+### Strengths
+
+- Essential for pre-processing text data in machine learning models.
+- Provides a comprehensive breakdown of text data, aiding in understanding its complexity.
+- Generates visualizations to help comprehend text structure and complexity.
+
+### Limitations
+
+- Highly dependent on the NLTK library, limiting the test to supported languages.
+- Limited customization for removing undesirable tokens and stop words.
+- Does not consider semantic or grammatical complexities.
+- Assumes well-structured documents, which may result in inaccuracies with poorly formatted text.
diff --git a/docs/validmind/tests/data_validation/nlp/Toxicity.qmd b/docs/validmind/tests/data_validation/nlp/Toxicity.qmd
new file mode 100644
index 000000000..459834d9b
--- /dev/null
+++ b/docs/validmind/tests/data_validation/nlp/Toxicity.qmd
@@ -0,0 +1,58 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Toxicity"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Toxicity
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'data_validation')
+
+@tasks('nlp')
+
+defToxicity(dataset):
+
+:::
+
+
+
+Assesses the toxicity of text data within a dataset to visualize the distribution of toxicity scores.
+
+### Purpose
+
+The Toxicity test aims to evaluate the level of toxic content present in a text dataset by leveraging a pre-trained toxicity model. It helps in identifying potentially harmful or offensive language that may negatively impact users or stakeholders.
+
+### Test Mechanism
+
+This test uses a pre-trained toxicity evaluation model and applies it to each text entry in the specified column of a dataset’s dataframe. The procedure involves:
+
+- Loading a pre-trained toxicity model.
+- Extracting the text from the specified column in the dataset.
+- Computing toxicity scores for each text entry.
+- Generating a KDE (Kernel Density Estimate) plot to visualize the distribution of these toxicity scores.
+
+### Signs of High Risk
+
+- High concentration of high toxicity scores in the KDE plot.
+- A significant proportion of text entries with toxicity scores above a predefined threshold.
+- Wide distribution of toxicity scores, indicating inconsistency in content quality.
+
+### Strengths
+
+- Provides a visual representation of toxicity distribution, making it easier to identify outliers.
+- Uses a robust pre-trained model for toxicity evaluation.
+- Can process large text datasets efficiently.
+
+### Limitations
+
+- Depends on the accuracy and bias of the pre-trained toxicity model.
+- Does not provide context-specific insights, which may be necessary for nuanced understanding.
+- May not capture all forms of subtle or indirect toxic language.
diff --git a/docs/validmind/tests/model_validation.qmd b/docs/validmind/tests/model_validation.qmd
new file mode 100644
index 000000000..d78bd7592
--- /dev/null
+++ b/docs/validmind/tests/model_validation.qmd
@@ -0,0 +1,26 @@
+---
+title: "[validmind](/validmind/validmind.qmd).model_validation"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+- [BertScore](model_validation/BertScore.qmd)
+- [BleuScore](model_validation/BleuScore.qmd)
+- [ClusterSizeDistribution](model_validation/ClusterSizeDistribution.qmd)
+- [ContextualRecall](model_validation/ContextualRecall.qmd)
+- [FeaturesAUC](model_validation/FeaturesAUC.qmd)
+- [MeteorScore](model_validation/MeteorScore.qmd)
+- [ModelMetadata](model_validation/ModelMetadata.qmd)
+- [ModelPredictionResiduals](model_validation/ModelPredictionResiduals.qmd)
+- [RegardScore](model_validation/RegardScore.qmd)
+- [RegressionResidualsPlot](model_validation/RegressionResidualsPlot.qmd)
+- [RougeScore](model_validation/RougeScore.qmd)
+- [sklearn](model_validation/sklearn.qmd)
+- [statsmodels](model_validation/statsmodels.qmd)
+- [TimeSeriesPredictionsPlot](model_validation/TimeSeriesPredictionsPlot.qmd)
+- [TimeSeriesPredictionWithCI](model_validation/TimeSeriesPredictionWithCI.qmd)
+- [TimeSeriesR2SquareBySegments](model_validation/TimeSeriesR2SquareBySegments.qmd)
+- [TokenDisparity](model_validation/TokenDisparity.qmd)
+- [ToxicityScore](model_validation/ToxicityScore.qmd)
diff --git a/docs/validmind/tests/model_validation/BertScore.qmd b/docs/validmind/tests/model_validation/BertScore.qmd
new file mode 100644
index 000000000..89e519ca4
--- /dev/null
+++ b/docs/validmind/tests/model_validation/BertScore.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).BertScore"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## BertScore
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defBertScore(dataset,model,evaluation_model='distilbert-base-uncased'):
+
+:::
+
+
+
+Assesses the quality of machine-generated text using BERTScore metrics and visualizes results through histograms and bar charts, alongside compiling a comprehensive table of descriptive statistics.
+
+### Purpose
+
+This function is designed to assess the quality of text generated by machine learning models using BERTScore metrics. BERTScore evaluates text generation models' performance by calculating precision, recall, and F1 score based on BERT contextual embeddings.
+
+### Test Mechanism
+
+The function starts by extracting the true and predicted values from the provided dataset and model. It then initializes the BERTScore evaluator. For each pair of true and predicted texts, the function calculates the BERTScore metrics and compiles them into a dataframe. Histograms and bar charts are generated for each BERTScore metric (Precision, Recall, and F1 Score) to visualize their distribution. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for each metric, providing a comprehensive summary of the model's performance. The test uses the `evaluation_model` param to specify the huggingface model to use for evaluation. `microsoft/deberta-xlarge-mnli` is the best-performing model but is very large and may be slow without a GPU. `microsoft/deberta-large-mnli` is a smaller model that is faster to run and `distilbert-base-uncased` is much lighter and can run on a CPU but is less accurate.
+
+### Signs of High Risk
+
+- Consistently low scores across BERTScore metrics could indicate poor quality in the generated text, suggesting that the model fails to capture the essential content of the reference texts.
+- Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
+- Low recall scores may indicate that important information from the reference text is being omitted.
+- An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the model's ability to balance informativeness and conciseness.
+
+### Strengths
+
+- Provides a multifaceted evaluation of text quality through different BERTScore metrics, offering a detailed view of model performance.
+- Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the scores.
+- Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
+
+### Limitations
+
+- BERTScore relies on the contextual embeddings from BERT models, which may not fully capture all nuances of text similarity.
+- The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
+- While useful for comparison, BERTScore metrics alone do not provide a complete assessment of a model's performance and should be supplemented with other metrics and qualitative analysis.
diff --git a/docs/validmind/tests/model_validation/BleuScore.qmd b/docs/validmind/tests/model_validation/BleuScore.qmd
new file mode 100644
index 000000000..e43893810
--- /dev/null
+++ b/docs/validmind/tests/model_validation/BleuScore.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).BleuScore"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## BleuScore
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defBleuScore(dataset,model):
+
+:::
+
+
+
+Evaluates the quality of machine-generated text using BLEU metrics and visualizes the results through histograms and bar charts, alongside compiling a comprehensive table of descriptive statistics for BLEU scores.
+
+### Purpose
+
+This function is designed to assess the quality of text generated by machine learning models using the BLEU metric. BLEU, which stands for Bilingual Evaluation Understudy, is a metric used to evaluate the overlap of n-grams between the machine-generated text and reference texts. This evaluation is crucial for tasks such as text summarization, machine translation, and text generation, where the goal is to produce text that accurately reflects the content and meaning of human-crafted references.
+
+### Test Mechanism
+
+The function starts by extracting the true and predicted values from the provided dataset and model. It then initializes the BLEU evaluator. For each pair of true and predicted texts, the function calculates the BLEU scores and compiles them into a dataframe. Histograms and bar charts are generated for the BLEU scores to visualize their distribution. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for the BLEU scores, providing a comprehensive summary of the model's performance.
+
+### Signs of High Risk
+
+- Consistently low BLEU scores could indicate poor quality in the generated text, suggesting that the model fails to capture the essential content of the reference texts.
+- Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
+- Low recall scores may indicate that important information from the reference text is being omitted.
+- An imbalanced performance between precision and recall, reflected by a low BLEU score, could signal issues in the model's ability to balance informativeness and conciseness.
+
+### Strengths
+
+- Provides a straightforward and widely-used evaluation of text quality through BLEU scores.
+- Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the scores.
+- Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
+
+### Limitations
+
+- BLEU metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or grammatical quality of the text.
+- The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
+- While useful for comparison, BLEU scores alone do not provide a complete assessment of a model's performance and should be supplemented with other metrics and qualitative analysis.
diff --git a/docs/validmind/tests/model_validation/ClusterSizeDistribution.qmd b/docs/validmind/tests/model_validation/ClusterSizeDistribution.qmd
new file mode 100644
index 000000000..e1a8052f7
--- /dev/null
+++ b/docs/validmind/tests/model_validation/ClusterSizeDistribution.qmd
@@ -0,0 +1,59 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ClusterSizeDistribution"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ClusterSizeDistribution
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance')
+
+@tasks('clustering')
+
+defClusterSizeDistribution(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel):
+
+:::
+
+
+
+Assesses the performance of clustering models by comparing the distribution of cluster sizes in model predictions with the actual data.
+
+### Purpose
+
+The Cluster Size Distribution test aims to assess the performance of clustering models by comparing the distribution of cluster sizes in the model's predictions with the actual data. This comparison helps determine if the clustering model's output aligns well with the true cluster distribution, providing insights into the model's accuracy and performance.
+
+### Test Mechanism
+
+The test mechanism involves the following steps:
+
+- Run the clustering model on the provided dataset to obtain predictions.
+- Convert both the actual and predicted outputs into pandas dataframes.
+- Use pandas built-in functions to derive the cluster size distributions from these dataframes.
+- Construct two histograms: one for the actual cluster size distribution and one for the predicted distribution.
+- Plot the histograms side-by-side for visual comparison.
+
+### Signs of High Risk
+
+- Discrepancies between the actual cluster size distribution and the predicted cluster size distribution.
+- Irregular distribution of data across clusters in the predicted outcomes.
+- High number of outlier clusters suggesting the model struggles to correctly group data.
+
+### Strengths
+
+- Provides a visual and intuitive way to compare the clustering model's performance against actual data.
+- Effectively reveals where the model may be over- or underestimating cluster sizes.
+- Versatile as it works well with any clustering model.
+
+### Limitations
+
+- Assumes that the actual cluster distribution is optimal, which may not always be the case.
+- Relies heavily on visual comparison, which could be subjective and may not offer a precise numerical measure of performance.
+- May not fully capture other important aspects of clustering, such as cluster density, distances between clusters, and the shape of clusters.
diff --git a/docs/validmind/tests/model_validation/ContextualRecall.qmd b/docs/validmind/tests/model_validation/ContextualRecall.qmd
new file mode 100644
index 000000000..07106e637
--- /dev/null
+++ b/docs/validmind/tests/model_validation/ContextualRecall.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ContextualRecall"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ContextualRecall
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defContextualRecall(dataset,model):
+
+:::
+
+
+
+Evaluates a Natural Language Generation model's ability to generate contextually relevant and factually correct text, visualizing the results through histograms and bar charts, alongside compiling a comprehensive table of descriptive statistics for contextual recall scores.
+
+### Purpose
+
+The Contextual Recall metric is used to evaluate the ability of a natural language generation (NLG) model to generate text that appropriately reflects the given context or prompt. It measures the model's capability to remember and reproduce the main context in its resulting output. This metric is critical in natural language processing tasks, as the coherency and contextuality of the generated text are essential.
+
+### Test Mechanism
+
+The function starts by extracting the true and predicted values from the provided dataset and model. It then tokenizes the reference and candidate texts into discernible words or tokens using NLTK. The token overlap between the reference and candidate texts is identified, and the Contextual Recall score is computed by dividing the number of overlapping tokens by the total number of tokens in the reference text. Scores are calculated for each test dataset instance, resulting in an array of scores. These scores are visualized using a histogram and a bar chart to show score variations across different rows. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for the contextual recall scores, providing a comprehensive summary of the model's performance.
+
+### Signs of High Risk
+
+- Low contextual recall scores could indicate that the model is not effectively reflecting the original context in its output, leading to incoherent or contextually misaligned text.
+- A consistent trend of low recall scores could suggest underperformance of the model.
+
+### Strengths
+
+- Provides a quantifiable measure of a model's adherence to the context and factual elements of the generated narrative.
+- Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of contextual recall scores.
+- Descriptive statistics offer a concise summary of the model's performance in generating contextually relevant texts.
+
+### Limitations
+
+- The focus on word overlap could result in high scores for texts that use many common words, even when these texts lack coherence or meaningful context.
+- This metric does not consider the order of words, which could lead to overestimated scores for scrambled outputs.
+- Models that effectively use infrequent words might be undervalued, as these words might not overlap as often.
diff --git a/docs/validmind/tests/model_validation/FeaturesAUC.qmd b/docs/validmind/tests/model_validation/FeaturesAUC.qmd
new file mode 100644
index 000000000..56c68b268
--- /dev/null
+++ b/docs/validmind/tests/model_validation/FeaturesAUC.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).FeaturesAUC"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## FeaturesAUC
+
+
+
+::: {.signature}
+
+@tags('feature_importance', 'AUC', 'visualization')
+
+@tasks('classification')
+
+defFeaturesAUC(dataset:validmind.vm_models.VMDataset,fontsize:int=12,figure_height:int=500):
+
+:::
+
+
+
+Evaluates the discriminatory power of each individual feature within a binary classification model by calculating the Area Under the Curve (AUC) for each feature separately.
+
+### Purpose
+
+The central objective of this metric is to quantify how well each feature on its own can differentiate between the two classes in a binary classification problem. It serves as a univariate analysis tool that can help in pre-modeling feature selection or post-modeling interpretation.
+
+### Test Mechanism
+
+For each feature, the metric treats the feature values as raw scores to compute the AUC against the actual binary outcomes. It provides an AUC value for each feature, offering a simple yet powerful indication of each feature's univariate classification strength.
+
+### Signs of High Risk
+
+- A feature with a low AUC score may not be contributing significantly to the differentiation between the two classes, which could be a concern if it is expected to be predictive.
+- Conversely, a surprisingly high AUC for a feature not believed to be informative may suggest data leakage or other issues with the data.
+
+### Strengths
+
+- By isolating each feature, it highlights the individual contribution of features to the classification task without the influence of other variables.
+- Useful for both initial feature evaluation and for providing insights into the model's reliance on individual features after model training.
+
+### Limitations
+
+- Does not reflect the combined effects of features or any interaction between them, which can be critical in certain models.
+- The AUC values are calculated without considering the model's use of the features, which could lead to different interpretations of feature importance when considering the model holistically.
+- This metric is applicable only to binary classification tasks and cannot be directly extended to multiclass classification or regression without modifications.
diff --git a/docs/validmind/tests/model_validation/MeteorScore.qmd b/docs/validmind/tests/model_validation/MeteorScore.qmd
new file mode 100644
index 000000000..05b669cac
--- /dev/null
+++ b/docs/validmind/tests/model_validation/MeteorScore.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).MeteorScore"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## MeteorScore
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defMeteorScore(dataset,model):
+
+:::
+
+
+
+Assesses the quality of machine-generated translations by comparing them to human-produced references using the METEOR score, which evaluates precision, recall, and word order.
+
+### Purpose
+
+The METEOR (Metric for Evaluation of Translation with Explicit ORdering) score is designed to evaluate the quality of machine translations by comparing them against reference translations. It emphasizes both the accuracy and fluency of translations, incorporating precision, recall, and word order into its assessment.
+
+### Test Mechanism
+
+The function starts by extracting the true and predicted values from the provided dataset and model. The METEOR score is computed for each pair of machine-generated translation (prediction) and its corresponding human-produced reference. This is done by considering unigram matches between the translations, including matches based on surface forms, stemmed forms, and synonyms. The score is a combination of unigram precision and recall, adjusted for word order through a fragmentation penalty. Scores are compiled into a dataframe, and histograms and bar charts are generated to visualize the distribution of METEOR scores. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for the METEOR scores, providing a comprehensive summary of the model's performance.
+
+### Signs of High Risk
+
+- Lower METEOR scores can indicate a lack of alignment between the machine-generated translations and their human-produced references, highlighting potential deficiencies in both the accuracy and fluency of translations.
+- Significant discrepancies in word order or an excessive fragmentation penalty could signal issues with how the translation model processes and reconstructs sentence structures, potentially compromising the natural flow of translated text.
+- Persistent underperformance across a variety of text types or linguistic contexts might suggest a broader inability of the model to adapt to the nuances of different languages or dialects, pointing towards gaps in its training or inherent limitations.
+
+### Strengths
+
+- Incorporates a balanced consideration of precision and recall, weighted towards recall to reflect the importance of content coverage in translations.
+- Directly accounts for word order, offering a nuanced evaluation of translation fluency beyond simple lexical matching.
+- Adapts to various forms of lexical similarity, including synonyms and stemmed forms, allowing for flexible matching.
+
+### Limitations
+
+- While comprehensive, the complexity of METEOR's calculation can make it computationally intensive, especially for large datasets.
+- The use of external resources for synonym and stemming matching may introduce variability based on the resources' quality and relevance to the specific translation task.
diff --git a/docs/validmind/tests/model_validation/ModelMetadata.qmd b/docs/validmind/tests/model_validation/ModelMetadata.qmd
new file mode 100644
index 000000000..3de4bf844
--- /dev/null
+++ b/docs/validmind/tests/model_validation/ModelMetadata.qmd
@@ -0,0 +1,48 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ModelMetadata"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ModelMetadata
+
+
+
+::: {.signature}
+
+@tags('model_training', 'metadata')
+
+@tasks('regression', 'time_series_forecasting')
+
+defModelMetadata(model):
+
+:::
+
+
+
+Compare metadata of different models and generate a summary table with the results.
+
+**Purpose**: The purpose of this function is to compare the metadata of different models, including information about their architecture, framework, framework version, and programming language.
+
+**Test Mechanism**: The function retrieves the metadata for each model using `get_model_info`, renames columns according to a predefined set of labels, and compiles this information into a summary table.
+
+**Signs of High Risk**:
+
+- Inconsistent or missing metadata across models can indicate potential issues in model documentation or management.
+- Significant differences in framework versions or programming languages might pose challenges in model integration and deployment.
+
+**Strengths**:
+
+- Provides a clear comparison of essential model metadata.
+- Standardizes metadata labels for easier interpretation and comparison.
+- Helps identify potential compatibility or consistency issues across models.
+
+**Limitations**:
+
+- Assumes that the `get_model_info` function returns all necessary metadata fields.
+- Relies on the correctness and completeness of the metadata provided by each model.
+- Does not include detailed parameter information, focusing instead on high-level metadata.
diff --git a/docs/validmind/tests/model_validation/ModelPredictionResiduals.qmd b/docs/validmind/tests/model_validation/ModelPredictionResiduals.qmd
new file mode 100644
index 000000000..fb0c40959
--- /dev/null
+++ b/docs/validmind/tests/model_validation/ModelPredictionResiduals.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ModelPredictionResiduals"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ModelPredictionResiduals
+
+
+
+::: {.signature}
+
+@tags('regression')
+
+@tasks('residual_analysis', 'visualization')
+
+defModelPredictionResiduals(dataset,model,nbins=100,p_value_threshold=0.05,start_date=None,end_date=None):
+
+:::
+
+
+
+Assesses normality and behavior of residuals in regression models through visualization and statistical tests.
+
+### Purpose
+
+The Model Prediction Residuals test aims to visualize the residuals of model predictions and assess their normality using the Kolmogorov-Smirnov (KS) test. It helps to identify potential issues related to model assumptions and effectiveness.
+
+### Test Mechanism
+
+The function calculates residuals and generates two figures: one for the time series of residuals and one for the histogram of residuals. It also calculates the KS test for normality and summarizes the results in a table.
+
+### Signs of High Risk
+
+- Residuals are not normally distributed, indicating potential issues with model assumptions.
+- High skewness or kurtosis in the residuals, which may suggest model misspecification.
+
+### Strengths
+
+- Provides clear visualizations of residuals over time and their distribution.
+- Includes statistical tests to assess the normality of residuals.
+- Helps in identifying potential model misspecifications and assumption violations.
+
+### Limitations
+
+- Assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access the pandas DataFrame.
+- Only generates plots for datasets with a datetime index, resulting in errors for other types of indices.
diff --git a/docs/validmind/tests/model_validation/RegardScore.qmd b/docs/validmind/tests/model_validation/RegardScore.qmd
new file mode 100644
index 000000000..510341da7
--- /dev/null
+++ b/docs/validmind/tests/model_validation/RegardScore.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegardScore"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegardScore
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defRegardScore(dataset,model):
+
+:::
+
+
+
+Assesses the sentiment and potential biases in text generated by NLP models by computing and visualizing regard scores.
+
+### Purpose
+
+The `RegardScore` test aims to evaluate the levels of regard (positive, negative, neutral, or other) in texts generated by NLP models. It helps in understanding the sentiment and bias present in the generated content.
+
+### Test Mechanism
+
+This test extracts the true and predicted values from the provided dataset and model. It then computes the regard scores for each text instance using a preloaded `regard` evaluation tool. The scores are compiled into dataframes, and visualizations such as histograms and bar charts are generated to display the distribution of regard scores. Additionally, descriptive statistics (mean, median, standard deviation, minimum, and maximum) are calculated for the regard scores, providing a comprehensive overview of the model's performance.
+
+### Signs of High Risk
+
+- Noticeable skewness in the histogram, especially when comparing the predicted regard scores with the target regard scores, can indicate biases or inconsistencies in the model.
+- Lack of neutral scores in the model's predictions, despite a balanced distribution in the target data, might signal an issue.
+
+### Strengths
+
+- Provides a clear evaluation of regard levels in generated texts, aiding in ensuring content appropriateness.
+- Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of regard scores.
+- Descriptive statistics offer a concise summary of the model's performance in generating texts with balanced sentiments.
+
+### Limitations
+
+- The accuracy of the regard scores is contingent upon the underlying `regard` tool.
+- The scores provide a broad overview but do not specify which portions or tokens of the text are responsible for high regard.
+- Supplementary, in-depth analysis might be needed for granular insights.
diff --git a/docs/validmind/tests/model_validation/RegressionResidualsPlot.qmd b/docs/validmind/tests/model_validation/RegressionResidualsPlot.qmd
new file mode 100644
index 000000000..0c330d523
--- /dev/null
+++ b/docs/validmind/tests/model_validation/RegressionResidualsPlot.qmd
@@ -0,0 +1,56 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionResidualsPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionResidualsPlot
+
+
+
+::: {.signature}
+
+@tags('model_performance', 'visualization')
+
+@tasks('regression')
+
+defRegressionResidualsPlot(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset,bin_size:float=0.1):
+
+:::
+
+
+
+Evaluates regression model performance using residual distribution and actual vs. predicted plots.
+
+### Purpose
+
+The `RegressionResidualsPlot` metric aims to evaluate the performance of regression models. By generating and analyzing two plots – a distribution of residuals and a scatter plot of actual versus predicted values – this tool helps to visually appraise how well the model predicts and the nature of errors it makes.
+
+### Test Mechanism
+
+The process begins by extracting the true output values (`y_true`) and the model's predicted values (`y_pred`). Residuals are computed by subtracting predicted from true values. These residuals are then visualized using a histogram to display their distribution. Additionally, a scatter plot is derived to compare true values against predicted values, together with a "Perfect Fit" line, which represents an ideal match (predicted values equal actual values), facilitating the assessment of the model's predictive accuracy.
+
+### Signs of High Risk
+
+- Residuals showing a non-normal distribution, especially those with frequent extreme values.
+- Significant deviations of predicted values from actual values in the scatter plot.
+- Sparse density of data points near the "Perfect Fit" line in the scatter plot, indicating poor prediction accuracy.
+- Visible patterns or trends in the residuals plot, suggesting the model's failure to capture the underlying data structure adequately.
+
+### Strengths
+
+- Provides a direct, visually intuitive assessment of a regression model’s accuracy and handling of data.
+- Visual plots can highlight issues of underfitting or overfitting.
+- Can reveal systematic deviations or trends that purely numerical metrics might miss.
+- Applicable across various regression model types.
+
+### Limitations
+
+- Relies on visual interpretation, which can be subjective and less precise than numerical evaluations.
+- May be difficult to interpret in cases with multi-dimensional outputs due to the plots’ two-dimensional nature.
+- Overlapping data points in the residuals plot can complicate interpretation efforts.
+- Does not summarize model performance into a single quantifiable metric, which might be needed for comparative or summary analyses.
diff --git a/docs/validmind/tests/model_validation/RougeScore.qmd b/docs/validmind/tests/model_validation/RougeScore.qmd
new file mode 100644
index 000000000..afd4d8271
--- /dev/null
+++ b/docs/validmind/tests/model_validation/RougeScore.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RougeScore"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RougeScore
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defRougeScore(dataset,model,metric='rouge-1'):
+
+:::
+
+
+
+Assesses the quality of machine-generated text using ROUGE metrics and visualizes the results to provide comprehensive performance insights.
+
+### Purpose
+
+The ROUGE Score test is designed to evaluate the quality of text generated by machine learning models using various ROUGE metrics. ROUGE, which stands for Recall-Oriented Understudy for Gisting Evaluation, measures the overlap of n-grams, word sequences, and word pairs between machine-generated text and reference texts. This evaluation is crucial for tasks like text summarization, machine translation, and text generation, where the goal is to produce text that accurately reflects the content and meaning of human-crafted references.
+
+### Test Mechanism
+
+The test extracts the true and predicted values from the provided dataset and model. It initializes the ROUGE evaluator with the specified metric (e.g., ROUGE-1). For each pair of true and predicted texts, it calculates the ROUGE scores and compiles them into a dataframe. Histograms and bar charts are generated for each ROUGE metric (Precision, Recall, and F1 Score) to visualize their distribution. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for each metric, providing a comprehensive summary of the model's performance.
+
+### Signs of High Risk
+
+- Consistently low scores across ROUGE metrics could indicate poor quality in the generated text, suggesting that the model fails to capture the essential content of the reference texts.
+- Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
+- Low recall scores may indicate that important information from the reference text is being omitted.
+- An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the model's ability to balance informativeness and conciseness.
+
+### Strengths
+
+- Provides a multifaceted evaluation of text quality through different ROUGE metrics, offering a detailed view of model performance.
+- Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the scores.
+- Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
+
+### Limitations
+
+- ROUGE metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or grammatical quality of the text.
+- The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
+- While useful for comparison, ROUGE scores alone do not provide a complete assessment of a model's performance and should be supplemented with other metrics and qualitative analysis.
diff --git a/docs/validmind/tests/model_validation/TimeSeriesPredictionWithCI.qmd b/docs/validmind/tests/model_validation/TimeSeriesPredictionWithCI.qmd
new file mode 100644
index 000000000..ae74963b1
--- /dev/null
+++ b/docs/validmind/tests/model_validation/TimeSeriesPredictionWithCI.qmd
@@ -0,0 +1,58 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TimeSeriesPredictionWithCI"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TimeSeriesPredictionWithCI
+
+
+
+::: {.signature}
+
+@tags('model_predictions', 'visualization')
+
+@tasks('regression', 'time_series_forecasting')
+
+defTimeSeriesPredictionWithCI(dataset,model,confidence=0.95):
+
+:::
+
+
+
+Assesses predictive accuracy and uncertainty in time series models, highlighting breaches beyond confidence intervals.
+
+### Purpose
+
+The purpose of the Time Series Prediction with Confidence Intervals (CI) test is to visualize the actual versus predicted values for time series data, including confidence intervals, and to compute and report the number of breaches beyond these intervals. This helps in evaluating the reliability and accuracy of the model's predictions.
+
+### Test Mechanism
+
+The function performs the following steps:
+
+- Calculates the standard deviation of prediction errors.
+- Determines the confidence intervals using a specified confidence level, typically 95%.
+- Counts the number of actual values that fall outside the confidence intervals, referred to as breaches.
+- Generates a plot visualizing the actual values, predicted values, and confidence intervals.
+- Returns a DataFrame summarizing the breach information, including the total breaches, upper breaches, and lower breaches.
+
+### Signs of High Risk
+
+- A high number of breaches indicates that the model's predictions are not reliable within the specified confidence level.
+- Significant deviations between actual and predicted values may highlight model inadequacies or issues with data quality.
+
+### Strengths
+
+- Provides a visual representation of prediction accuracy and the uncertainty around predictions.
+- Includes a statistical measure of prediction reliability through confidence intervals.
+- Computes and reports breaches, offering a quantitative assessment of prediction performance.
+
+### Limitations
+
+- Assumes that the dataset is provided as a DataFrameDataset object with a datetime index.
+- Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+- The calculation of confidence intervals assumes normally distributed errors, which may not hold for all datasets.
diff --git a/docs/validmind/tests/model_validation/TimeSeriesPredictionsPlot.qmd b/docs/validmind/tests/model_validation/TimeSeriesPredictionsPlot.qmd
new file mode 100644
index 000000000..3e67ebb14
--- /dev/null
+++ b/docs/validmind/tests/model_validation/TimeSeriesPredictionsPlot.qmd
@@ -0,0 +1,48 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TimeSeriesPredictionsPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TimeSeriesPredictionsPlot
+
+
+
+::: {.signature}
+
+@tags('model_predictions', 'visualization')
+
+@tasks('regression', 'time_series_forecasting')
+
+defTimeSeriesPredictionsPlot(dataset,model):
+
+:::
+
+
+
+Plot actual vs predicted values for time series data and generate a visual comparison for the model.
+
+### Purpose
+
+The purpose of this function is to visualize the actual versus predicted values for time series data for a single model.
+
+### Test Mechanism
+
+The function plots the actual values from the dataset and overlays the predicted values from the model using Plotly for interactive visualization.
+
+- Large discrepancies between actual and predicted values indicate poor model performance.
+- Systematic deviations in predicted values can highlight model bias or issues with data patterns.
+
+### Strengths
+
+- Provides a clear visual comparison of model predictions against actual values.
+- Uses Plotly for interactive and visually appealing plots.
+
+### Limitations
+
+- Assumes that the dataset is provided as a DataFrameDataset object with a datetime index.
+- Requires that `dataset.y_pred(model)` returns the predicted values for the model.
diff --git a/docs/validmind/tests/model_validation/TimeSeriesR2SquareBySegments.qmd b/docs/validmind/tests/model_validation/TimeSeriesR2SquareBySegments.qmd
new file mode 100644
index 000000000..60b00b3f2
--- /dev/null
+++ b/docs/validmind/tests/model_validation/TimeSeriesR2SquareBySegments.qmd
@@ -0,0 +1,56 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TimeSeriesR2SquareBySegments"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TimeSeriesR2SquareBySegments
+
+
+
+::: {.signature}
+
+@tags('model_performance', 'sklearn')
+
+@tasks('regression', 'time_series_forecasting')
+
+defTimeSeriesR2SquareBySegments(dataset,model,segments=None):
+
+:::
+
+
+
+Evaluates the R-Squared values of regression models over specified time segments in time series data to assess segment-wise model performance.
+
+### Purpose
+
+The TimeSeriesR2SquareBySegments test aims to evaluate the R-Squared values for several regression models across different segments of time series data. This helps in determining how well the models explain the variability in the data within each specific time segment.
+
+### Test Mechanism
+
+- Provides a visual representation of model performance across different time segments.
+- Allows for identification of segments where the model performs poorly.
+- Calculating the R-Squared values for each segment.
+- Generating a bar chart to visually represent the R-Squared values across different models and segments.
+
+### Signs of High Risk
+
+- Significantly low R-Squared values for certain time segments, indicating poor model performance in those periods.
+- Large variability in R-Squared values across different segments for the same model, suggesting inconsistent performance.
+
+### Strengths
+
+- Provides a visual representation of how well models perform over different time periods.
+- Helps identify time segments where models may need improvement or retraining.
+- Facilitates comparison between multiple models in a straightforward manner.
+
+### Limitations
+
+- Assumes datasets are provided as DataFrameDataset objects with the attributes `y`, `y_pred`, and `feature_columns`.
+- Requires that `dataset.y_pred(model)` returns predicted values for the model.
+- Assumes that both `y_true` and `y_pred` are pandas Series with datetime indices, which may not always be the case.
+- May not account for more nuanced temporal dependencies within the segments.
diff --git a/docs/validmind/tests/model_validation/TokenDisparity.qmd b/docs/validmind/tests/model_validation/TokenDisparity.qmd
new file mode 100644
index 000000000..236ebc7ff
--- /dev/null
+++ b/docs/validmind/tests/model_validation/TokenDisparity.qmd
@@ -0,0 +1,50 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TokenDisparity"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TokenDisparity
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defTokenDisparity(dataset,model):
+
+:::
+
+
+
+Evaluates the token disparity between reference and generated texts, visualizing the results through histograms and bar charts, alongside compiling a comprehensive table of descriptive statistics for token counts.
+
+### Purpose
+
+The Token Disparity test aims to assess the difference in the number of tokens between reference texts and texts generated by the model. Understanding token disparity is essential for evaluating how well the generated content matches the expected length and richness of the reference texts.
+
+### Test Mechanism
+
+The test extracts true and predicted values from the dataset and model. It computes the number of tokens in each reference and generated text. The results are visualized using histograms and bar charts to display the distribution of token counts. Additionally, a table of descriptive statistics, including the mean, median, standard deviation, minimum, and maximum token counts, is compiled to provide a detailed summary of token usage.
+
+### Signs of High Risk
+
+- Significant disparity in token counts between reference and generated texts could indicate issues with text generation quality, such as verbosity or lack of detail.
+- Consistently low token counts in generated texts compared to references might suggest that the model is producing incomplete or overly concise outputs.
+
+### Strengths
+
+- Provides a simple yet effective evaluation of text length and token usage.
+- Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of token counts.
+- Descriptive statistics offer a concise summary of the model's performance in generating texts of appropriate length.
+
+### Limitations
+
+- Token counts alone do not provide a complete assessment of text quality and should be supplemented with other metrics and qualitative analysis.
diff --git a/docs/validmind/tests/model_validation/ToxicityScore.qmd b/docs/validmind/tests/model_validation/ToxicityScore.qmd
new file mode 100644
index 000000000..f9b99c051
--- /dev/null
+++ b/docs/validmind/tests/model_validation/ToxicityScore.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ToxicityScore"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ToxicityScore
+
+
+
+::: {.signature}
+
+@tags('nlp', 'text_data', 'visualization')
+
+@tasks('text_classification', 'text_summarization')
+
+defToxicityScore(dataset,model):
+
+:::
+
+
+
+Assesses the toxicity levels of texts generated by NLP models to identify and mitigate harmful or offensive content.
+
+### Purpose
+
+The ToxicityScore metric is designed to evaluate the toxicity levels of texts generated by models. This is crucial for identifying and mitigating harmful or offensive content in machine-generated texts.
+
+### Test Mechanism
+
+The function starts by extracting the input, true, and predicted values from the provided dataset and model. The toxicity score is computed for each text using a preloaded `toxicity` evaluation tool. The scores are compiled into dataframes, and histograms and bar charts are generated to visualize the distribution of toxicity scores. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for the toxicity scores, providing a comprehensive summary of the model's performance.
+
+### Signs of High Risk
+
+- Drastic spikes in toxicity scores indicate potentially toxic content within the associated text segment.
+- Persistent high toxicity scores across multiple texts may suggest systemic issues in the model's text generation process.
+
+### Strengths
+
+- Provides a clear evaluation of toxicity levels in generated texts, helping to ensure content safety and appropriateness.
+- Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of toxicity scores.
+- Descriptive statistics offer a concise summary of the model's performance in generating non-toxic texts.
+
+### Limitations
+
+- The accuracy of the toxicity scores is contingent upon the underlying `toxicity` tool.
+- The scores provide a broad overview but do not specify which portions or tokens of the text are responsible for high toxicity.
+- Supplementary, in-depth analysis might be needed for granular insights.
diff --git a/docs/validmind/tests/model_validation/sklearn.qmd b/docs/validmind/tests/model_validation/sklearn.qmd
new file mode 100644
index 000000000..9ec181ffb
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn.qmd
@@ -0,0 +1,44 @@
+---
+title: "[validmind](/validmind/validmind.qmd).sklearn"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+- [AdjustedMutualInformation](sklearn/AdjustedMutualInformation.qmd)
+- [AdjustedRandIndex](sklearn/AdjustedRandIndex.qmd)
+- [CalibrationCurve](sklearn/CalibrationCurve.qmd)
+- [ClassifierPerformance](sklearn/ClassifierPerformance.qmd)
+- [ClassifierThresholdOptimization](sklearn/ClassifierThresholdOptimization.qmd)
+- [ClusterCosineSimilarity](sklearn/ClusterCosineSimilarity.qmd)
+- [ClusterPerformanceMetrics](sklearn/ClusterPerformanceMetrics.qmd)
+- [CompletenessScore](sklearn/CompletenessScore.qmd)
+- [ConfusionMatrix](sklearn/ConfusionMatrix.qmd)
+- [FeatureImportance](sklearn/FeatureImportance.qmd)
+- [FowlkesMallowsScore](sklearn/FowlkesMallowsScore.qmd)
+- [HomogeneityScore](sklearn/HomogeneityScore.qmd)
+- [HyperParametersTuning](sklearn/HyperParametersTuning.qmd)
+- [KMeansClustersOptimization](sklearn/KMeansClustersOptimization.qmd)
+- [MinimumAccuracy](sklearn/MinimumAccuracy.qmd)
+- [MinimumF1Score](sklearn/MinimumF1Score.qmd)
+- [MinimumROCAUCScore](sklearn/MinimumROCAUCScore.qmd)
+- [ModelParameters](sklearn/ModelParameters.qmd)
+- [ModelsPerformanceComparison](sklearn/ModelsPerformanceComparison.qmd)
+- [OverfitDiagnosis](sklearn/OverfitDiagnosis.qmd)
+- [PermutationFeatureImportance](sklearn/PermutationFeatureImportance.qmd)
+- [PopulationStabilityIndex](sklearn/PopulationStabilityIndex.qmd)
+- [PrecisionRecallCurve](sklearn/PrecisionRecallCurve.qmd)
+- [RegressionErrors](sklearn/RegressionErrors.qmd)
+- [RegressionErrorsComparison](sklearn/RegressionErrorsComparison.qmd)
+- [RegressionPerformance](sklearn/RegressionPerformance.qmd)
+- [RegressionR2Square](sklearn/RegressionR2Square.qmd)
+- [RegressionR2SquareComparison](sklearn/RegressionR2SquareComparison.qmd)
+- [RobustnessDiagnosis](sklearn/RobustnessDiagnosis.qmd)
+- [ROCCurve](sklearn/ROCCurve.qmd)
+- [ScoreProbabilityAlignment](sklearn/ScoreProbabilityAlignment.qmd)
+- [SHAPGlobalImportance](sklearn/SHAPGlobalImportance.qmd)
+- [SilhouettePlot](sklearn/SilhouettePlot.qmd)
+- [TrainingTestDegradation](sklearn/TrainingTestDegradation.qmd)
+- [VMeasure](sklearn/VMeasure.qmd)
+- [WeakspotsDiagnosis](sklearn/WeakspotsDiagnosis.qmd)
diff --git a/docs/validmind/tests/model_validation/sklearn/AdjustedMutualInformation.qmd b/docs/validmind/tests/model_validation/sklearn/AdjustedMutualInformation.qmd
new file mode 100644
index 000000000..4ee45faa2
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/AdjustedMutualInformation.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).AdjustedMutualInformation"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## AdjustedMutualInformation
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance', 'clustering')
+
+@tasks('clustering')
+
+defAdjustedMutualInformation(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Evaluates clustering model performance by measuring mutual information between true and predicted labels, adjusting for chance.
+
+### Purpose
+
+The purpose of this metric (Adjusted Mutual Information) is to evaluate the performance of a machine learning model, more specifically, a clustering model. It measures the mutual information between the true labels and the ones predicted by the model, adjusting for chance.
+
+### Test Mechanism
+
+The Adjusted Mutual Information (AMI) uses sklearn's `adjusted_mutual_info_score` function. This function calculates the mutual information between the true labels and the ones predicted while correcting for the chance correlation expected due to random label assignments. This test requires the model, the training dataset, and the test dataset as inputs.
+
+### Signs of High Risk
+
+- Low Adjusted Mutual Information Score: This score ranges between 0 and 1. A low score (closer to 0) can indicate poor model performance as the predicted labels do not align well with the true labels.
+- In case of high-dimensional data, if the algorithm shows high scores, this could also be a potential risk as AMI may not perform reliably.
+
+### Strengths
+
+- The AMI metric takes into account the randomness of the predicted labels, which makes it more robust than the simple Mutual Information.
+- The scale of AMI is not dependent on the sizes of the clustering, allowing for comparability between different datasets or models.
+- Good for comparing the output of clustering algorithms where the number of clusters is not known a priori.
+
+### Limitations
+
+- Adjusted Mutual Information does not take into account the continuous nature of some data. As a result, it may not be the best choice for regression or other continuous types of tasks.
+- AMI has the drawback of being biased towards clusterings with a higher number of clusters.
+- In comparison to other metrics, AMI can be slower to compute.
+- The interpretability of the score can be complex as it depends on the understanding of information theory concepts.
diff --git a/docs/validmind/tests/model_validation/sklearn/AdjustedRandIndex.qmd b/docs/validmind/tests/model_validation/sklearn/AdjustedRandIndex.qmd
new file mode 100644
index 000000000..8785c2860
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/AdjustedRandIndex.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).AdjustedRandIndex"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## AdjustedRandIndex
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance', 'clustering')
+
+@tasks('clustering')
+
+defAdjustedRandIndex(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Measures the similarity between two data clusters using the Adjusted Rand Index (ARI) metric in clustering machine learning models.
+
+### Purpose
+
+The Adjusted Rand Index (ARI) metric is intended to measure the similarity between two data clusters. This metric is specifically used for clustering machine learning models to quantify how well the model is clustering and producing data groups. It involves comparing the model's produced clusters against the actual (true) clusters found in the dataset.
+
+### Test Mechanism
+
+The Adjusted Rand Index (ARI) is calculated using the `adjusted_rand_score` method from the `sklearn.metrics` module in Python. The test requires inputs including the model itself and the model's training and test datasets. The model's computed clusters and the true clusters are compared, and the similarities are measured to compute the ARI.
+
+### Signs of High Risk
+
+- If the ARI is close to zero, it signifies that the model's cluster assignments are random and do not match the actual dataset clusters, indicating a high risk.
+- An ARI of less than zero indicates that the model's clustering performance is worse than random.
+
+### Strengths
+
+- ARI is normalized and provides a consistent metric between -1 and +1, irrespective of raw cluster sizes or dataset size variations.
+- It does not require a ground truth for computation, making it ideal for unsupervised learning model evaluations.
+- It penalizes for false positives and false negatives, providing a robust measure of clustering quality.
+
+### Limitations
+
+- In real-world situations, true clustering is often unknown, which can hinder the practical application of the ARI.
+- The ARI requires all individual data instances to be independent, which may not always hold true.
+- It may be difficult to interpret the implications of an ARI score without context or a benchmark, as it is heavily dependent on the characteristics of the dataset used.
diff --git a/docs/validmind/tests/model_validation/sklearn/CalibrationCurve.qmd b/docs/validmind/tests/model_validation/sklearn/CalibrationCurve.qmd
new file mode 100644
index 000000000..aa7ec86fc
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/CalibrationCurve.qmd
@@ -0,0 +1,73 @@
+---
+title: "[validmind](/validmind/validmind.qmd).CalibrationCurve"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## CalibrationCurve
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance', 'classification')
+
+@tasks('classification')
+
+defCalibrationCurve(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset,n_bins:int=10):
+
+:::
+
+
+
+Evaluates the calibration of probability estimates by comparing predicted probabilities against observed frequencies.
+
+### Purpose
+
+The Calibration Curve test assesses how well a model's predicted probabilities align with actual observed frequencies. This is crucial for applications requiring accurate probability estimates, such as risk assessment, decision-making systems, and cost-sensitive applications where probability calibration directly impacts business decisions.
+
+### Test Mechanism
+
+The test uses sklearn's calibration_curve function to:
+
+1. Sort predictions into bins based on predicted probabilities
+1. Calculate the mean predicted probability in each bin
+1. Compare against the observed frequency of positive cases
+1. Plot the results against the perfect calibration line (y=x) The resulting curve shows how well the predicted probabilities match empirical probabilities.
+
+### Signs of High Risk
+
+- Significant deviation from the perfect calibration line
+- Systematic overconfidence (predictions too close to 0 or 1)
+- Systematic underconfidence (predictions clustered around 0.5)
+- Empty or sparse bins indicating poor probability coverage
+- Sharp discontinuities in the calibration curve
+- Different calibration patterns across different probability ranges
+- Consistent over/under estimation in critical probability regions
+- Large confidence intervals in certain probability ranges
+
+### Strengths
+
+- Visual and intuitive interpretation of probability quality
+- Identifies systematic biases in probability estimates
+- Supports probability threshold selection
+- Helps understand model confidence patterns
+- Applicable across different classification models
+- Enables comparison between different models
+- Guides potential need for recalibration
+- Critical for risk-sensitive applications
+
+### Limitations
+
+- Sensitive to the number of bins chosen
+- Requires sufficient samples in each bin for reliable estimates
+- May mask local calibration issues within bins
+- Does not account for feature-dependent calibration issues
+- Limited to binary classification problems
+- Cannot detect all forms of miscalibration
+- Assumes bin boundaries are appropriate for the problem
+- May be affected by class imbalance
diff --git a/docs/validmind/tests/model_validation/sklearn/ClassifierPerformance.qmd b/docs/validmind/tests/model_validation/sklearn/ClassifierPerformance.qmd
new file mode 100644
index 000000000..5a652ec0d
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/ClassifierPerformance.qmd
@@ -0,0 +1,65 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ClassifierPerformance"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ClassifierPerformance
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'model_performance')
+
+@tasks('classification', 'text_classification')
+
+defClassifierPerformance(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel,average:str='macro'):
+
+:::
+
+
+
+Evaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy, and ROC AUC scores.
+
+### Purpose
+
+The Classifier Performance test is designed to evaluate the performance of Machine Learning classification models. It accomplishes this by computing precision, recall, F1-Score, and accuracy, as well as the ROC AUC (Receiver operating characteristic - Area under the curve) scores, thereby providing a comprehensive analytic view of the models' performance. The test is adaptable, handling binary and multiclass models equally effectively.
+
+### Test Mechanism
+
+The test produces a report that includes precision, recall, F1-Score, and accuracy, by leveraging the `classification_report` from scikit-learn's metrics module. For multiclass models, macro and weighted averages for these scores are also calculated. Additionally, the ROC AUC scores are calculated and included in the report using the `multiclass_roc_auc_score` function. The outcome of the test (report format) differs based on whether the model is binary or multiclass.
+
+### Signs of High Risk
+
+- Low values for precision, recall, F1-Score, accuracy, and ROC AUC, indicating poor performance.
+- Imbalance in precision and recall scores.
+- A low ROC AUC score, especially scores close to 0.5 or lower, suggesting a failing model.
+
+### Strengths
+
+- Versatile, capable of assessing both binary and multiclass models.
+- Utilizes a variety of commonly employed performance metrics, offering a comprehensive view of model performance.
+- The use of ROC-AUC as a metric is beneficial for evaluating unbalanced datasets.
+
+### Limitations
+
+- Assumes correctly identified labels for binary classification models.
+- Specifically designed for classification models and not suitable for regression models.
+- May provide limited insights if the test dataset does not represent real-world scenarios adequately.
+
+
+
+## multiclass_roc_auc_score
+
+
+
+::: {.signature}
+
+defmulticlass_roc_auc_score(y_test,y_pred,average='macro'):
+
+:::
diff --git a/docs/validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.qmd b/docs/validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.qmd
new file mode 100644
index 000000000..fac8d4406
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.qmd
@@ -0,0 +1,114 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ClassifierThresholdOptimization"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ClassifierThresholdOptimization
+
+
+
+::: {.signature}
+
+@tags('model_validation', 'threshold_optimization', 'classification_metrics')
+
+@tasks('classification')
+
+defClassifierThresholdOptimization(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel,methods:Optional\[List\[str\]\]=None,target_recall:Optional\[float\]=None) → Dict\[str, Union\[pd.DataFrame, go.Figure\]\]:
+
+:::
+
+
+
+Analyzes and visualizes different threshold optimization methods for binary classification models.
+
+### Purpose
+
+The Classifier Threshold Optimization test identifies optimal decision thresholds using various methods to balance different performance metrics. This helps adapt the model's decision boundary to specific business requirements, such as minimizing false positives in fraud detection or achieving target recall in medical diagnosis.
+
+### Test Mechanism
+
+The test implements multiple threshold optimization methods:
+
+1. Youden's J statistic (maximizing sensitivity + specificity - 1)
+1. F1-score optimization (balancing precision and recall)
+1. Precision-Recall equality point
+1. Target recall achievement
+1. Naive (0.5) threshold For each method, it computes ROC and PR curves, identifies optimal points, and provides comprehensive performance metrics at each threshold.
+
+### Signs of High Risk
+
+- Large discrepancies between different optimization methods
+- Optimal thresholds far from the default 0.5
+- Poor performance metrics across all thresholds
+- Significant gap between achieved and target recall
+- Unstable thresholds across different methods
+- Extreme trade-offs between precision and recall
+- Threshold optimization showing minimal impact
+- Business metrics not improving with optimization
+
+### Strengths
+
+- Multiple optimization strategies for different needs
+- Visual and numerical results for comparison
+- Support for business-driven optimization (target recall)
+- Comprehensive performance metrics at each threshold
+- Integration with ROC and PR curves
+- Handles class imbalance through various metrics
+- Enables informed threshold selection
+- Supports cost-sensitive decision making
+
+### Limitations
+
+- Assumes cost of false positives/negatives are known
+- May need adjustment for highly imbalanced datasets
+- Threshold might not be stable across different samples
+- Cannot handle multi-class problems directly
+- Optimization methods may conflict with business needs
+- Requires sufficient validation data
+- May not capture temporal changes in optimal threshold
+- Single threshold may not be optimal for all subgroups
+
+**Arguments**
+
+- `dataset`: VMDataset containing features and target
+- `model`: VMModel containing predictions
+- `methods`: List of methods to compare (default: \['youden', 'f1', 'precision_recall'\])
+- `target_recall`: Target recall value if using 'target_recall' method
+
+**Returns**
+
+- Dictionary containing:
+- table: DataFrame comparing different threshold optimization methods (using weighted averages for precision, recall, and f1)
+- figure: Plotly figure showing ROC and PR curves with optimal thresholds
+
+
+
+## find_optimal_threshold
+
+
+
+::: {.signature}
+
+deffind_optimal_threshold(y_true:np.ndarray,y_prob:np.ndarray,method:str='youden',target_recall:Optional\[float\]=None) → Dict\[str, Union\[str, float\]\]:
+
+:::
+
+
+
+Find the optimal classification threshold using various methods.
+
+**Arguments**
+
+- `y_true`: True binary labels
+- `y_prob`: Predicted probabilities
+- `method`: Method to use for finding optimal threshold
+- `target_recall`: Required if method='target_recall'
+
+**Returns**
+
+- Dictionary containing threshold and metrics
diff --git a/docs/validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.qmd b/docs/validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.qmd
new file mode 100644
index 000000000..79071e15f
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ClusterCosineSimilarity"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ClusterCosineSimilarity
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance', 'clustering')
+
+@tasks('clustering')
+
+defClusterCosineSimilarity(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Measures the intra-cluster similarity of a clustering model using cosine similarity.
+
+### Purpose
+
+The purpose of this metric is to measure how similar the data points within each cluster of a clustering model are. This is done using cosine similarity, which compares the multi-dimensional direction (but not magnitude) of data vectors. From a Model Risk Management perspective, this metric is used to quantitatively validate that clusters formed by a model have high intra-cluster similarity.
+
+### Test Mechanism
+
+This test works by first extracting the true and predicted clusters of the model's training data. Then, it computes the centroid (average data point) of each cluster. Next, it calculates the cosine similarity between each data point within a cluster and its respective centroid. Finally, it outputs the mean cosine similarity of each cluster, highlighting how similar, on average, data points in a cluster are to the cluster's centroid.
+
+### Signs of High Risk
+
+- Low mean cosine similarity for one or more clusters: If the mean cosine similarity is low, the data points within the respective cluster have high variance in their directions. This can be indicative of poor clustering, suggesting that the model might not be suitably separating the data into distinct patterns.
+- High disparity between mean cosine similarity values across clusters: If there's a significant difference in mean cosine similarity across different clusters, this could indicate imbalance in how the model forms clusters.
+
+### Strengths
+
+- Cosine similarity operates in a multi-dimensional space, making it effective for measuring similarity in high dimensional datasets, typical for many machine learning problems.
+- It provides an agnostic view of the cluster performance by only considering the direction (and not the magnitude) of each vector.
+- This metric is not dependent on the scale of the variables, making it equally effective on different scales.
+
+### Limitations
+
+- Cosine similarity does not consider magnitudes (i.e. lengths) of vectors, only their direction. This means it may overlook instances where clusters have been adequately separated in terms of magnitude.
+- This method summarily assumes that centroids represent the average behavior of data points in each cluster. This might not always be true, especially in clusters with high amounts of variance or non-spherical shapes.
+- It primarily works with continuous variables and is not suitable for binary or categorical variables.
+- Lastly, although rare, perfect perpendicular vectors (cosine similarity = 0) could be within the same cluster, which may give an inaccurate representation of a 'bad' cluster due to low cosine similarity score.
diff --git a/docs/validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.qmd b/docs/validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.qmd
new file mode 100644
index 000000000..9557693de
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.qmd
@@ -0,0 +1,58 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ClusterPerformanceMetrics"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ClusterPerformanceMetrics
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance', 'clustering')
+
+@tasks('clustering')
+
+defClusterPerformanceMetrics(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Evaluates the performance of clustering machine learning models using multiple established metrics.
+
+### Purpose
+
+The `ClusterPerformanceMetrics` test is used to assess the performance and validity of clustering machine learning models. It evaluates homogeneity, completeness, V measure score, the Adjusted Rand Index, the Adjusted Mutual Information, and the Fowlkes-Mallows score of the model. These metrics provide a holistic understanding of the model's ability to accurately form clusters of the given dataset.
+
+### Test Mechanism
+
+The `ClusterPerformanceMetrics` test runs a clustering ML model over a given dataset and then calculates six metrics using the Scikit-learn metrics computation functions: Homogeneity Score, Completeness Score, V Measure, Adjusted Rand Index (ARI), Adjusted Mutual Information (AMI), and Fowlkes-Mallows Score. It then returns the result as a summary, presenting the metric values for both training and testing datasets.
+
+### Signs of High Risk
+
+- Low Homogeneity Score: Indicates that the clusters formed contain a variety of classes, resulting in less pure clusters.
+- Low Completeness Score: Suggests that class instances are scattered across multiple clusters rather than being gathered in a single cluster.
+- Low V Measure: Reports a low overall clustering performance.
+- ARI close to 0 or Negative: Implies that clustering results are random or disagree with the true labels.
+- AMI close to 0: Means that clustering labels are random compared with the true labels.
+- Low Fowlkes-Mallows score: Signifies less precise and poor clustering performance in terms of precision and recall.
+
+### Strengths
+
+- Provides a comprehensive view of clustering model performance by examining multiple clustering metrics.
+- Uses established and widely accepted metrics from scikit-learn, providing reliability in the results.
+- Able to provide performance metrics for both training and testing datasets.
+- Clearly defined and human-readable descriptions of each score make it easy to understand what each score represents.
+
+### Limitations
+
+- Only applies to clustering models; not suitable for other types of machine learning models.
+- Does not test for overfitting or underfitting in the clustering model.
+- All the scores rely on ground truth labels, the absence or inaccuracy of which can lead to misleading results.
+- Does not consider aspects like computational efficiency of the model or its capability to handle high dimensional data.
diff --git a/docs/validmind/tests/model_validation/sklearn/CompletenessScore.qmd b/docs/validmind/tests/model_validation/sklearn/CompletenessScore.qmd
new file mode 100644
index 000000000..1b6e3aa20
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/CompletenessScore.qmd
@@ -0,0 +1,50 @@
+---
+title: "[validmind](/validmind/validmind.qmd).CompletenessScore"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## CompletenessScore
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance', 'clustering')
+
+@tasks('clustering')
+
+defCompletenessScore(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Evaluates a clustering model's capacity to categorize instances from a single class into the same cluster.
+
+### Purpose
+
+The Completeness Score metric is used to assess the performance of clustering models. It measures the extent to which all the data points that are members of a given class are elements of the same cluster. The aim is to determine the capability of the model to categorize all instances from a single class into the same cluster.
+
+### Test Mechanism
+
+This test takes three inputs, a model and its associated training and testing datasets. It invokes the `completeness_score` function from the sklearn library on the labels predicted by the model. High scores indicate that data points from the same class generally appear in the same cluster, while low scores suggest the opposite.
+
+### Signs of High Risk
+
+- Low completeness score: This suggests that the model struggles to group instances from the same class into one cluster, indicating poor clustering performance.
+
+### Strengths
+
+- The Completeness Score provides an effective method for assessing the performance of a clustering model, specifically its ability to group class instances together.
+- This test metric conveniently relies on the capabilities provided by the sklearn library, ensuring consistent and reliable test results.
+
+### Limitations
+
+- This metric only evaluates a specific aspect of clustering, meaning it may not provide a holistic or complete view of the model's performance.
+- It cannot assess the effectiveness of the model in differentiating between separate classes, as it is solely focused on how well data points from the same class are grouped.
+- The Completeness Score only applies to clustering models; it cannot be used for other types of machine learning models.
diff --git a/docs/validmind/tests/model_validation/sklearn/ConfusionMatrix.qmd b/docs/validmind/tests/model_validation/sklearn/ConfusionMatrix.qmd
new file mode 100644
index 000000000..a05a0207d
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/ConfusionMatrix.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ConfusionMatrix"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ConfusionMatrix
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization')
+
+@tasks('classification', 'text_classification')
+
+defConfusionMatrix(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel,threshold:float=0.5):
+
+:::
+
+
+
+Evaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix heatmap.
+
+### Purpose
+
+The Confusion Matrix tester is designed to assess the performance of a classification Machine Learning model. This performance is evaluated based on how well the model is able to correctly classify True Positives, True Negatives, False Positives, and False Negatives - fundamental aspects of model accuracy.
+
+### Test Mechanism
+
+The mechanism used involves taking the predicted results (`y_test_predict`) from the classification model and comparing them against the actual values (`y_test_true`). A confusion matrix is built using the unique labels extracted from `y_test_true`, employing scikit-learn's metrics. The matrix is then visually rendered with the help of Plotly's `create_annotated_heatmap` function. A heatmap is created which provides a two-dimensional graphical representation of the model's performance, showcasing distributions of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN).
+
+### Signs of High Risk
+
+- High numbers of False Positives (FP) and False Negatives (FN), depicting that the model is not effectively classifying the values.
+- Low numbers of True Positives (TP) and True Negatives (TN), implying that the model is struggling with correctly identifying class labels.
+
+### Strengths
+
+- It provides a simplified yet comprehensive visual snapshot of the classification model's predictive performance.
+- It distinctly brings out True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), thus making it easier to focus on potential areas of improvement.
+- The matrix is beneficial in dealing with multi-class classification problems as it can provide a simple view of complex model performances.
+- It aids in understanding the different types of errors that the model could potentially make, as it provides in-depth insights into Type-I and Type-II errors.
+
+### Limitations
+
+- In cases of unbalanced classes, the effectiveness of the confusion matrix might be lessened. It may wrongly interpret the accuracy of a model that is essentially just predicting the majority class.
+- It does not provide a single unified statistic that could evaluate the overall performance of the model. Different aspects of the model's performance are evaluated separately instead.
+- It mainly serves as a descriptive tool and does not offer the capability for statistical hypothesis testing.
+- Risks of misinterpretation exist because the matrix doesn't directly provide precision, recall, or F1-score data. These metrics have to be computed separately.
diff --git a/docs/validmind/tests/model_validation/sklearn/FeatureImportance.qmd b/docs/validmind/tests/model_validation/sklearn/FeatureImportance.qmd
new file mode 100644
index 000000000..2e5be43a5
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/FeatureImportance.qmd
@@ -0,0 +1,56 @@
+---
+title: "[validmind](/validmind/validmind.qmd).FeatureImportance"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## FeatureImportance
+
+
+
+::: {.signature}
+
+@tags('model_explainability', 'sklearn')
+
+@tasks('regression', 'time_series_forecasting')
+
+defFeatureImportance(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel,num_features:int=3):
+
+:::
+
+
+
+Compute feature importance scores for a given model and generate a summary table with the top important features.
+
+### Purpose
+
+The Feature Importance Comparison test is designed to compare the feature importance scores for different models when applied to various datasets. By doing so, it aims to identify the most impactful features and assess the consistency of feature importance across models.
+
+### Test Mechanism
+
+This test works by iterating through each dataset-model pair and calculating permutation feature importance (PFI) scores. It then generates a summary table containing the top `num_features` important features for each model. The process involves:
+
+- Extracting features and target data from each dataset.
+- Computing PFI scores using `sklearn.inspection.permutation_importance`.
+- Sorting and selecting the top features based on their importance scores.
+- Compiling these features into a summary table for comparison.
+
+### Signs of High Risk
+
+- Key features expected to be important are ranked low, indicating potential issues with model training or data quality.
+- High variance in feature importance scores across different models, suggesting instability in feature selection.
+
+### Strengths
+
+- Provides a clear comparison of the most important features for each model.
+- Uses permutation importance, which is a model-agnostic method and can be applied to any estimator.
+
+### Limitations
+
+- Assumes that the dataset is provided as a DataFrameDataset object with `x_df` and `y_df` methods to access feature and target data.
+- Requires that `model.model` is compatible with `sklearn.inspection.permutation_importance`.
+- The function's output is dependent on the number of features specified by `num_features`, which defaults to 3 but can be adjusted.
diff --git a/docs/validmind/tests/model_validation/sklearn/FowlkesMallowsScore.qmd b/docs/validmind/tests/model_validation/sklearn/FowlkesMallowsScore.qmd
new file mode 100644
index 000000000..1b0658b57
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/FowlkesMallowsScore.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).FowlkesMallowsScore"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## FowlkesMallowsScore
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance')
+
+@tasks('clustering')
+
+defFowlkesMallowsScore(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel):
+
+:::
+
+
+
+Evaluates the similarity between predicted and actual cluster assignments in a model using the Fowlkes-Mallows score.
+
+### Purpose
+
+The FowlkesMallowsScore is a performance metric used to validate clustering algorithms within machine learning models. The score intends to evaluate the matching grade between two clusters. It measures the similarity between the predicted and actual cluster assignments, thus gauging the accuracy of the model's clustering capability.
+
+### Test Mechanism
+
+The FowlkesMallowsScore method applies the `fowlkes_mallows_score` function from the `sklearn` library to evaluate the model's accuracy in clustering different types of data. The test fetches the datasets from the model's training and testing datasets as inputs then compares the resulting clusters against the previously known clusters to obtain a score. A high score indicates a better clustering performance by the model.
+
+### Signs of High Risk
+
+- A low Fowlkes-Mallows score (near zero): This indicates that the model's clustering capability is poor and the algorithm isn't properly grouping data.
+- Inconsistently low scores across different datasets: This may indicate that the model's clustering performance is not robust and the model may fail when applied to unseen data.
+
+### Strengths
+
+- The Fowlkes-Mallows score is a simple and effective method for evaluating the performance of clustering algorithms.
+- This metric takes into account both precision and recall in its calculation, therefore providing a balanced and comprehensive measure of model performance.
+- The Fowlkes-Mallows score is non-biased meaning it treats False Positives and False Negatives equally.
+
+### Limitations
+
+- As a pairwise-based method, this score can be computationally intensive for large datasets and can become unfeasible as the size of the dataset increases.
+- The Fowlkes-Mallows score works best with balanced distribution of samples across clusters. If this condition is not met, the score can be skewed.
+- It does not handle mismatching numbers of clusters between the true and predicted labels. As such, it may return misleading results if the predicted labels suggest a different number of clusters than what is in the true labels.
diff --git a/docs/validmind/tests/model_validation/sklearn/HomogeneityScore.qmd b/docs/validmind/tests/model_validation/sklearn/HomogeneityScore.qmd
new file mode 100644
index 000000000..92fede7ad
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/HomogeneityScore.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).HomogeneityScore"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## HomogeneityScore
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance')
+
+@tasks('clustering')
+
+defHomogeneityScore(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel):
+
+:::
+
+
+
+Assesses clustering homogeneity by comparing true and predicted labels, scoring from 0 (heterogeneous) to 1 (homogeneous).
+
+### Purpose
+
+The Homogeneity Score encapsulated in this performance test is used to measure the homogeneity of the clusters formed by a machine learning model. In simple terms, a clustering result satisfies homogeneity if all of its clusters contain only points which are members of a single class.
+
+### Test Mechanism
+
+This test uses the `homogeneity_score` function from the `sklearn.metrics` library to compare the ground truth class labels of the training and testing sets with the labels predicted by the given model. The returned score is a metric of the clustering accuracy, and ranges from 0.0 to 1.0, with 1.0 denoting the highest possible degree of homogeneity.
+
+### Signs of High Risk
+
+- A score close to 0: This denotes that clusters are highly heterogenous and points within the same cluster might not belong to the same class.
+- A significantly lower score for testing data compared to the score for training data: This can indicate overfitting, where the model has learned to perfectly match the training data but fails to perform well on unseen data.
+
+### Strengths
+
+- It provides a simple quantitative measure of the degree to which clusters contain points from only one class.
+- Useful for validating clustering solutions where the ground truth — class membership of points — is known.
+- It's agnostic to the absolute labels, and cares only that the points within the same cluster have the same class label.
+
+### Limitations
+
+- The Homogeneity Score is not useful for clustering solutions where the ground truth labels are not known.
+- It doesn’t work well with differently sized clusters since it gives predominance to larger clusters.
+- The score does not address the actual number of clusters formed, or the evenness of cluster sizes. It only checks the homogeneity within the given clusters created by the model.
diff --git a/docs/validmind/tests/model_validation/sklearn/HyperParametersTuning.qmd b/docs/validmind/tests/model_validation/sklearn/HyperParametersTuning.qmd
new file mode 100644
index 000000000..87d7b8819
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/HyperParametersTuning.qmd
@@ -0,0 +1,84 @@
+---
+title: "[validmind](/validmind/validmind.qmd).HyperParametersTuning"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## custom_recall
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance')
+
+@tasks('classification', 'clustering')
+
+defcustom_recall(y_true,y_pred_proba,threshold=0.5):
+
+:::
+
+
+
+## HyperParametersTuning
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance')
+
+@tasks('clustering', 'classification')
+
+defHyperParametersTuning(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset,param_grid:dict,scoring:Union\[str, List, Dict\]=None,thresholds:Union\[float, List\[float\]\]=None,fit_params:dict=None):
+
+:::
+
+
+
+Performs exhaustive grid search over specified parameter ranges to find optimal model configurations across different metrics and decision thresholds.
+
+### Purpose
+
+The Hyperparameter Tuning test systematically explores the model's parameter space to identify optimal configurations. It supports multiple optimization metrics and decision thresholds, providing a comprehensive view of how different parameter combinations affect various aspects of model performance.
+
+### Test Mechanism
+
+The test uses scikit-learn's GridSearchCV to perform cross-validation for each parameter combination. For each specified threshold and optimization metric, it creates a scoring dictionary with threshold-adjusted metrics, performs grid search with cross-validation, records best parameters and corresponding scores, and combines results into a comparative table. This process is repeated for each optimization metric to provide a comprehensive view of model performance under different configurations.
+
+### Signs of High Risk
+
+- Large performance variations across different parameter combinations
+- Significant discrepancies between different optimization metrics
+- Best parameters at the edges of the parameter grid
+- Unstable performance across different thresholds
+- Overly complex model configurations (risk of overfitting)
+- Very different optimal parameters for different metrics
+- Cross-validation scores showing high variance
+- Extreme parameter values in best configurations
+
+### Strengths
+
+- Comprehensive exploration of parameter space
+- Supports multiple optimization metrics
+- Allows threshold optimization
+- Provides comparative view across different configurations
+- Uses cross-validation for robust evaluation
+- Helps understand trade-offs between different metrics
+- Enables systematic parameter selection
+- Supports both classification and clustering tasks
+
+### Limitations
+
+- Computationally expensive for large parameter grids
+- May not find global optimum (limited to grid points)
+- Cannot handle dependencies between parameters
+- Memory intensive for large datasets
+- Limited to scikit-learn compatible models
+- Cross-validation splits may not preserve time series structure
+- Grid search may miss optimal values between grid points
+- Resource intensive for high-dimensional parameter spaces
diff --git a/docs/validmind/tests/model_validation/sklearn/KMeansClustersOptimization.qmd b/docs/validmind/tests/model_validation/sklearn/KMeansClustersOptimization.qmd
new file mode 100644
index 000000000..7f9351fe1
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/KMeansClustersOptimization.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).KMeansClustersOptimization"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## KMeansClustersOptimization
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance', 'kmeans')
+
+@tasks('clustering')
+
+defKMeansClustersOptimization(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset,n_clusters:Union\[List\[int\], None\]=None):
+
+:::
+
+
+
+Optimizes the number of clusters in K-means models using Elbow and Silhouette methods.
+
+### Purpose
+
+This metric is used to optimize the number of clusters used in K-means clustering models. It intends to measure and evaluate the optimal number of clusters by leveraging two methodologies, namely the Elbow method and the Silhouette method. This is crucial as an inappropriate number of clusters can either overly simplify or overcomplicate the structure of the data, thereby undermining the effectiveness of the model.
+
+### Test Mechanism
+
+The test mechanism involves iterating over a predefined range of cluster numbers and applying both the Elbow method and the Silhouette method. The Elbow method computes the sum of the minimum euclidean distances between data points and their respective cluster centers (distortion). This value decreases as the number of clusters increases; the optimal number is typically at the 'elbow' point where the decrease in distortion becomes less pronounced. Meanwhile, the Silhouette method calculates the average silhouette score for each data point in the dataset, providing a measure of how similar each item is to its own cluster compared to other clusters. The optimal number of clusters under this method is the one that maximizes the average silhouette score. The results of both methods are plotted for visual inspection.
+
+### Signs of High Risk
+
+- A high distortion value or a low silhouette average score for the optimal number of clusters.
+- No clear 'elbow' point or plateau observed in the distortion plot, or a uniformly low silhouette average score across different numbers of clusters, suggesting the data is not amenable to clustering.
+- An optimal cluster number that is unreasonably high or low, suggestive of overfitting or underfitting, respectively.
+
+### Strengths
+
+- Provides both a visual and quantitative method to determine the optimal number of clusters.
+- Leverages two different methods (Elbow and Silhouette), thereby affording robustness and versatility in assessing the data's clusterability.
+- Facilitates improved model performance by allowing for an informed selection of the number of clusters.
+
+### Limitations
+
+- Assumes that a suitable number of clusters exists in the data, which may not always be true, especially for complex or noisy data.
+- Both methods may fail to provide definitive answers when the data lacks clear cluster structures.
+- Might not be straightforward to determine the 'elbow' point or maximize the silhouette average score, especially in larger and complicated datasets.
+- Assumes spherical clusters (due to using the Euclidean distance in the Elbow method), which might not align with the actual structure of the data.
diff --git a/docs/validmind/tests/model_validation/sklearn/MinimumAccuracy.qmd b/docs/validmind/tests/model_validation/sklearn/MinimumAccuracy.qmd
new file mode 100644
index 000000000..8cfaf5400
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/MinimumAccuracy.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).MinimumAccuracy"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## MinimumAccuracy
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'model_performance')
+
+@tasks('classification', 'text_classification')
+
+defMinimumAccuracy(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel,min_threshold:float=0.7):
+
+:::
+
+
+
+Checks if the model's prediction accuracy meets or surpasses a specified threshold.
+
+### Purpose
+
+The Minimum Accuracy test’s objective is to verify whether the model's prediction accuracy on a specific dataset meets or surpasses a predetermined minimum threshold. Accuracy, which is simply the ratio of correct predictions to total predictions, is a key metric for evaluating the model's performance. Considering binary as well as multiclass classifications, accurate labeling becomes indispensable.
+
+### Test Mechanism
+
+The test mechanism involves contrasting the model's accuracy score with a preset minimum threshold value, with the default being 0.7. The accuracy score is computed utilizing sklearn’s `accuracy_score` method, where the true labels `y_true` and predicted labels `class_pred` are compared. If the accuracy score is above the threshold, the test receives a passing mark. The test returns the result along with the accuracy score and threshold used for the test.
+
+### Signs of High Risk
+
+- Model fails to achieve or surpass the predefined score threshold.
+- Persistent scores below the threshold, indicating a high risk of inaccurate predictions.
+
+### Strengths
+
+- Simplicity, presenting a straightforward measure of holistic model performance across all classes.
+- Particularly advantageous when classes are balanced.
+- Versatile, as it can be implemented on both binary and multiclass classification tasks.
+
+### Limitations
+
+- Misleading accuracy scores when classes in the dataset are highly imbalanced.
+- Favoritism towards the majority class, giving an inaccurate perception of model performance.
+- Inability to measure the model's precision, recall, or capacity to manage false positives or false negatives.
+- Focused on overall correctness and may not be sufficient for all types of model analytics.
diff --git a/docs/validmind/tests/model_validation/sklearn/MinimumF1Score.qmd b/docs/validmind/tests/model_validation/sklearn/MinimumF1Score.qmd
new file mode 100644
index 000000000..78de3af74
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/MinimumF1Score.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).MinimumF1Score"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## MinimumF1Score
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'model_performance')
+
+@tasks('classification', 'text_classification')
+
+defMinimumF1Score(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel,min_threshold:float=0.5):
+
+:::
+
+
+
+Assesses if the model's F1 score on the validation set meets a predefined minimum threshold, ensuring balanced performance between precision and recall.
+
+### Purpose
+
+The main objective of this test is to ensure that the F1 score, a balanced measure of precision and recall, of the model meets or surpasses a predefined threshold on the validation dataset. The F1 score is highly useful for gauging model performance in classification tasks, especially in cases where the distribution of positive and negative classes is skewed.
+
+### Test Mechanism
+
+The F1 score for the validation dataset is computed through scikit-learn's metrics in Python. The scoring mechanism differs based on the classification problem: for multi-class problems, macro averaging is used, and for binary classification, the built-in `f1_score` calculation is used. The obtained F1 score is then assessed against the predefined minimum F1 score that is expected from the model.
+
+### Signs of High Risk
+
+- If a model returns an F1 score that is less than the established threshold, it is regarded as high risk.
+- A low F1 score might suggest that the model is not finding an optimal balance between precision and recall, failing to effectively identify positive classes while minimizing false positives.
+
+### Strengths
+
+- Provides a balanced measure of a model's performance by accounting for both false positives and false negatives.
+- Particularly advantageous in scenarios with imbalanced class distribution, where accuracy can be misleading.
+- Flexibility in setting the threshold value allows tailored minimum acceptable performance standards.
+
+### Limitations
+
+- May not be suitable for all types of models and machine learning tasks.
+- The F1 score assumes an equal cost for false positives and false negatives, which may not be true in some real-world scenarios.
+- Practitioners might need to rely on other metrics such as precision, recall, or the ROC-AUC score that align more closely with specific requirements.
diff --git a/docs/validmind/tests/model_validation/sklearn/MinimumROCAUCScore.qmd b/docs/validmind/tests/model_validation/sklearn/MinimumROCAUCScore.qmd
new file mode 100644
index 000000000..3698dd80d
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/MinimumROCAUCScore.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).MinimumROCAUCScore"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## MinimumROCAUCScore
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'model_performance')
+
+@tasks('classification', 'text_classification')
+
+defMinimumROCAUCScore(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel,min_threshold:float=0.5):
+
+:::
+
+
+
+Validates model by checking if the ROC AUC score meets or surpasses a specified threshold.
+
+### Purpose
+
+The Minimum ROC AUC Score test is used to determine the model's performance by ensuring that the Receiver Operating Characteristic Area Under the Curve (ROC AUC) score on the validation dataset meets or exceeds a predefined threshold. The ROC AUC score indicates how well the model can distinguish between different classes, making it a crucial measure in binary and multiclass classification tasks.
+
+### Test Mechanism
+
+This test implementation calculates the multiclass ROC AUC score on the true target values and the model's predictions. The test converts the multi-class target variables into binary format using `LabelBinarizer` before computing the score. If this ROC AUC score is higher than the predefined threshold (defaulted to 0.5), the test passes; otherwise, it fails. The results, including the ROC AUC score, the threshold, and whether the test passed or failed, are then stored in a `ThresholdTestResult` object.
+
+### Signs of High Risk
+
+- A high risk or failure in the model's performance as related to this metric would be represented by a low ROC AUC score, specifically any score lower than the predefined minimum threshold. This suggests that the model is struggling to distinguish between different classes effectively.
+
+### Strengths
+
+- The test considers both the true positive rate and false positive rate, providing a comprehensive performance measure.
+- ROC AUC score is threshold-independent meaning it measures the model's quality across various classification thresholds.
+- Works robustly with binary as well as multi-class classification problems.
+
+### Limitations
+
+- ROC AUC may not be useful if the class distribution is highly imbalanced; it could perform well in terms of AUC but still fail to predict the minority class.
+- The test does not provide insight into what specific aspects of the model are causing poor performance if the ROC AUC score is unsatisfactory.
+- The use of macro average for multiclass ROC AUC score implies equal weightage to each class, which might not be appropriate if the classes are imbalanced.
diff --git a/docs/validmind/tests/model_validation/sklearn/ModelParameters.qmd b/docs/validmind/tests/model_validation/sklearn/ModelParameters.qmd
new file mode 100644
index 000000000..91942186a
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/ModelParameters.qmd
@@ -0,0 +1,60 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ModelParameters"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ModelParameters
+
+
+
+::: {.signature}
+
+@tags('model_training', 'metadata')
+
+@tasks('classification', 'regression')
+
+defModelParameters(model,model_params=None):
+
+:::
+
+
+
+Extracts and displays model parameters in a structured format for transparency and reproducibility.
+
+### Purpose
+
+The Model Parameters test is designed to provide transparency into model configuration and ensure reproducibility of machine learning models. It accomplishes this by extracting and presenting all relevant parameters that define the model's behavior, making it easier to audit, validate, and reproduce model training.
+
+### Test Mechanism
+
+The test leverages scikit-learn's API convention of get_params() to extract model parameters. It produces a structured DataFrame containing parameter names and their corresponding values. For models that follow scikit-learn's API (including XGBoost, RandomForest, and other estimators), all parameters are automatically extracted and displayed.
+
+### Signs of High Risk
+
+- Missing crucial parameters that should be explicitly set
+- Extreme parameter values that could indicate overfitting (e.g., unlimited tree depth)
+- Inconsistent parameters across different versions of the same model type
+- Parameter combinations known to cause instability or poor performance
+- Default values used for critical parameters that should be tuned
+
+### Strengths
+
+- Universal compatibility with scikit-learn API-compliant models
+- Ensures transparency in model configuration
+- Facilitates model reproducibility and version control
+- Enables systematic parameter auditing
+- Supports both classification and regression models
+- Helps identify potential configuration issues
+
+### Limitations
+
+- Only works with models implementing scikit-learn's get_params() method
+- Cannot capture dynamic parameters set during model training
+- Does not validate parameter values for model-specific appropriateness
+- Parameter meanings and impacts may vary across different model types
+- Cannot detect indirect parameter interactions or their effects on model performance
diff --git a/docs/validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.qmd b/docs/validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.qmd
new file mode 100644
index 000000000..95ffbc9e5
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ModelsPerformanceComparison"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ModelsPerformanceComparison
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'model_comparison')
+
+@tasks('classification', 'text_classification')
+
+defModelsPerformanceComparison(dataset:validmind.vm_models.VMDataset,models:list\[validmind.vm_models.VMModel\]):
+
+:::
+
+
+
+Evaluates and compares the performance of multiple Machine Learning models using various metrics like accuracy, precision, recall, and F1 score.
+
+### Purpose
+
+The Models Performance Comparison test aims to evaluate and compare the performance of various Machine Learning models using test data. It employs multiple metrics such as accuracy, precision, recall, and the F1 score, among others, to assess model performance and assist in selecting the most effective model for the designated task.
+
+### Test Mechanism
+
+The test employs Scikit-learn’s performance metrics to evaluate each model's performance for both binary and multiclass classification tasks. To compare performances, the test runs each model against the test dataset, then produces a comprehensive classification report. This report includes metrics such as accuracy, precision, recall, and the F1 score. Based on whether the task at hand is binary or multiclass classification, it calculates metrics for all the classes and their weighted averages, macro averages, and per-class metrics. The test will be skipped if no models are supplied.
+
+### Signs of High Risk
+
+- Low scores in accuracy, precision, recall, and F1 metrics indicate a potentially high risk.
+- A low area under the Receiver Operating Characteristic (ROC) curve (roc_auc score) is another possible indicator of high risk.
+- If the metrics scores are significantly lower than alternative models, this might suggest a high risk of failure.
+
+### Strengths
+
+- Provides a simple way to compare the performance of multiple models, accommodating both binary and multiclass classification tasks.
+- Offers a holistic view of model performance through a comprehensive report of key performance metrics.
+- The inclusion of the ROC AUC score is advantageous, as this robust performance metric can effectively handle class imbalance issues.
+
+### Limitations
+
+- May not be suitable for more complex performance evaluations that consider factors such as prediction speed, computational cost, or business-specific constraints.
+- The test's reliability depends on the provided test dataset; hence, the selected models' performance could vary with unseen data or changes in the data distribution.
+- The ROC AUC score might not be as meaningful or easily interpretable for multilabel/multiclass tasks.
diff --git a/docs/validmind/tests/model_validation/sklearn/OverfitDiagnosis.qmd b/docs/validmind/tests/model_validation/sklearn/OverfitDiagnosis.qmd
new file mode 100644
index 000000000..659d21b8e
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/OverfitDiagnosis.qmd
@@ -0,0 +1,59 @@
+---
+title: "[validmind](/validmind/validmind.qmd).OverfitDiagnosis"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## OverfitDiagnosis
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'linear_regression', 'model_diagnosis')
+
+@tasks('classification', 'regression')
+
+defOverfitDiagnosis(model:validmind.vm_models.VMModel,datasets:List\[validmind.vm_models.VMDataset\],metric:str=None,cut_off_threshold:float=DEFAULT_THRESHOLD):
+
+:::
+
+
+
+Assesses potential overfitting in a model's predictions, identifying regions where performance between training and testing sets deviates significantly.
+
+### Purpose
+
+The Overfit Diagnosis test aims to identify areas in a model's predictions where there is a significant difference in performance between the training and testing sets. This test helps to pinpoint specific regions or feature segments where the model may be overfitting.
+
+### Test Mechanism
+
+This test compares the model's performance on training versus test data, grouped by feature columns. It calculates the difference between the training and test performance for each group and identifies regions where this difference exceeds a specified threshold:
+
+- The test works for both classification and regression models.
+- It defaults to using the AUC metric for classification models and the MSE metric for regression models.
+- The threshold for identifying overfitting regions is set to 0.04 by default.
+- The test calculates the performance metrics for each feature segment and plots regions where the performance gap exceeds the threshold.
+
+### Signs of High Risk
+
+- Significant gaps between training and test performance metrics for specific feature segments.
+- Multiple regions with performance gaps exceeding the defined threshold.
+- Higher than expected differences in predicted versus actual values in the test set compared to the training set.
+
+### Strengths
+
+- Identifies specific areas where overfitting occurs.
+- Supports multiple performance metrics, providing flexibility.
+- Applicable to both classification and regression models.
+- Visualization of overfitting segments aids in better understanding and debugging.
+
+### Limitations
+
+- The default threshold may not be suitable for all use cases and requires tuning.
+- May not capture more subtle forms of overfitting that do not exceed the threshold.
+- Assumes that the binning of features adequately represents the data segments.
diff --git a/docs/validmind/tests/model_validation/sklearn/PermutationFeatureImportance.qmd b/docs/validmind/tests/model_validation/sklearn/PermutationFeatureImportance.qmd
new file mode 100644
index 000000000..8e292cbb1
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/PermutationFeatureImportance.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).PermutationFeatureImportance"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## PermutationFeatureImportance
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization')
+
+@tasks('classification', 'text_classification')
+
+defPermutationFeatureImportance(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset,fontsize:Union\[int, None\]=None,figure_height:Union\[int, None\]=None):
+
+:::
+
+
+
+Assesses the significance of each feature in a model by evaluating the impact on model performance when feature values are randomly rearranged.
+
+### Purpose
+
+The Permutation Feature Importance (PFI) metric aims to assess the importance of each feature used by the Machine Learning model. The significance is measured by evaluating the decrease in the model's performance when the feature's values are randomly arranged.
+
+### Test Mechanism
+
+PFI is calculated via the `permutation_importance` method from the `sklearn.inspection` module. This method shuffles the columns of the feature dataset and measures the impact on the model's performance. A significant decrease in performance after permutating a feature's values deems the feature as important. On the other hand, if performance remains the same, the feature is likely not important. The output of the PFI metric is a figure illustrating the importance of each feature.
+
+### Signs of High Risk
+
+- The model heavily relies on a feature with highly variable or easily permutable values, indicating instability.
+- A feature deemed unimportant by the model but expected to have a significant effect on the outcome based on domain knowledge is not influencing the model's predictions.
+
+### Strengths
+
+- Provides insights into the importance of different features and may reveal underlying data structure.
+- Can indicate overfitting if a particular feature or set of features overly impacts the model's predictions.
+- Model-agnostic and can be used with any classifier that provides a measure of prediction accuracy before and after feature permutation.
+
+### Limitations
+
+- Does not imply causality; it only presents the amount of information that a feature provides for the prediction task.
+- Does not account for interactions between features. If features are correlated, the permutation importance may allocate importance to one and not the other.
+- Cannot interact with certain libraries like statsmodels, pytorch, catboost, etc., thus limiting its applicability.
diff --git a/docs/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.qmd b/docs/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.qmd
new file mode 100644
index 000000000..bf276645c
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.qmd
@@ -0,0 +1,70 @@
+---
+title: "[validmind](/validmind/validmind.qmd).PopulationStabilityIndex"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## calculate_psi
+
+
+
+::: {.signature}
+
+defcalculate_psi(score_initial,score_new,num_bins=10,mode='fixed'):
+
+:::
+
+
+
+Taken from: https://towardsdatascience.com/checking-model-stability-and-population-shift-with-psi-and-csi-6d12af008783
+
+
+
+## PopulationStabilityIndex
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'model_performance')
+
+@tasks('classification', 'text_classification')
+
+defPopulationStabilityIndex(datasets:List\[validmind.vm_models.VMDataset\],model:validmind.vm_models.VMModel,num_bins:int=10,mode:str='fixed'):
+
+:::
+
+
+
+Assesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across different datasets.
+
+### Purpose
+
+The Population Stability Index (PSI) serves as a quantitative assessment for evaluating the stability of a machine learning model's output distributions when comparing two different datasets. Typically, these would be a development and a validation dataset or two datasets collected at different periods. The PSI provides a measurable indication of any significant shift in the model's performance over time or noticeable changes in the characteristics of the population the model is making predictions for.
+
+### Test Mechanism
+
+The implementation of the PSI in this script involves calculating the PSI for each feature between the training and test datasets. Data from both datasets is sorted and placed into either a predetermined number of bins or quantiles. The boundaries for these bins are initially determined based on the distribution of the training data. The contents of each bin are calculated and their respective proportions determined. Subsequently, the PSI is derived for each bin through a logarithmic transformation of the ratio of the proportions of data for each feature in the training and test datasets. The PSI, along with the proportions of data in each bin for both datasets, are displayed in a summary table, a grouped bar chart, and a scatter plot.
+
+### Signs of High Risk
+
+- A high PSI value is a clear indicator of high risk. Such a value suggests a significant shift in the model predictions or severe changes in the characteristics of the underlying population.
+- This ultimately suggests that the model may not be performing as well as expected and that it may be less reliable for making future predictions.
+
+### Strengths
+
+- The PSI provides a quantitative measure of the stability of a model over time or across different samples, making it an invaluable tool for evaluating changes in a model's performance.
+- It allows for direct comparisons across different features based on the PSI value.
+- The calculation and interpretation of the PSI are straightforward, facilitating its use in model risk management.
+- The use of visual aids such as tables and charts further simplifies the comprehension and interpretation of the PSI.
+
+### Limitations
+
+- The PSI test does not account for the interdependence between features: features that are dependent on one another may show similar shifts in their distributions, which in turn may result in similar PSI values.
+- The PSI test does not inherently provide insights into why there are differences in distributions or why the PSI values may have changed.
+- The test may not handle features with significant outliers adequately.
+- Additionally, the PSI test is performed on model predictions, not on the underlying data distributions which can lead to misinterpretations. Any changes in PSI could be due to shifts in the model (model drift), changes in the relationships between features and the target variable (concept drift), or both. However, distinguishing between these causes is non-trivial.
diff --git a/docs/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.qmd b/docs/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.qmd
new file mode 100644
index 000000000..a7e10a3ba
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).PrecisionRecallCurve"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## PrecisionRecallCurve
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'model_performance', 'visualization')
+
+@tasks('classification', 'text_classification')
+
+defPrecisionRecallCurve(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Evaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve.
+
+### Purpose
+
+The Precision Recall Curve metric is intended to evaluate the trade-off between precision and recall in classification models, particularly binary classification models. It assesses the model's capacity to produce accurate results (high precision), as well as its ability to capture a majority of all positive instances (high recall).
+
+### Test Mechanism
+
+The test extracts ground truth labels and prediction probabilities from the model's test dataset. It applies the `precision_recall_curve` method from the sklearn metrics module to these extracted labels and predictions, which computes a precision-recall pair for each possible threshold. This calculation results in an array of precision and recall scores that can be plotted against each other to form the Precision-Recall Curve. This curve is then visually represented by using Plotly's scatter plot.
+
+### Signs of High Risk
+
+- A lower area under the Precision-Recall Curve signifies high risk.
+- This corresponds to a model yielding a high amount of false positives (low precision) and/or false negatives (low recall).
+- If the curve is closer to the bottom left of the plot, rather than being closer to the top right corner, it can be a sign of high risk.
+
+### Strengths
+
+- This metric aptly represents the balance between precision (minimizing false positives) and recall (minimizing false negatives), which is especially critical in scenarios where both values are significant.
+- Through the graphic representation, it enables an intuitive understanding of the model's performance across different threshold levels.
+
+### Limitations
+
+- This metric is only applicable to binary classification models - it raises errors for multiclass classification models or Foundation models.
+- It may not fully represent the overall accuracy of the model if the cost of false positives and false negatives are extremely different, or if the dataset is heavily imbalanced.
diff --git a/docs/validmind/tests/model_validation/sklearn/ROCCurve.qmd b/docs/validmind/tests/model_validation/sklearn/ROCCurve.qmd
new file mode 100644
index 000000000..92060aa54
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/ROCCurve.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ROCCurve"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ROCCurve
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization')
+
+@tasks('classification', 'text_classification')
+
+defROCCurve(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Evaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic (ROC) curve and calculating the Area Under Curve (AUC) score.
+
+### Purpose
+
+The Receiver Operating Characteristic (ROC) curve is designed to evaluate the performance of binary classification models. This curve illustrates the balance between the True Positive Rate (TPR) and False Positive Rate (FPR) across various threshold levels. In combination with the Area Under the Curve (AUC), the ROC curve aims to measure the model's discrimination ability between the two defined classes in a binary classification problem (e.g., default vs non-default). Ideally, a higher AUC score signifies superior model performance in accurately distinguishing between the positive and negative classes.
+
+### Test Mechanism
+
+First, this script selects the target model and datasets that require binary classification. It then calculates the predicted probabilities for the test set, and uses this data, along with the true outcomes, to generate and plot the ROC curve. Additionally, it includes a line signifying randomness (AUC of 0.5). The AUC score for the model's ROC curve is also computed, presenting a numerical estimation of the model's performance. If any Infinite values are detected in the ROC threshold, these are effectively eliminated. The resulting ROC curve, AUC score, and thresholds are consequently saved for future reference.
+
+### Signs of High Risk
+
+- A high risk is potentially linked to the model's performance if the AUC score drops below or nears 0.5.
+- Another warning sign would be the ROC curve lying closer to the line of randomness, indicating no discriminative ability.
+- For the model to be deemed competent at its classification tasks, it is crucial that the AUC score is significantly above 0.5.
+
+### Strengths
+
+- The ROC Curve offers an inclusive visual depiction of a model's discriminative power throughout all conceivable classification thresholds, unlike other metrics that solely disclose model performance at one fixed threshold.
+- Despite the proportions of the dataset, the AUC Score, which represents the entire ROC curve as a single data point, continues to be consistent, proving to be the ideal choice for such situations.
+
+### Limitations
+
+- The primary limitation is that this test is exclusively structured for binary classification tasks, thus limiting its application towards other model types.
+- Furthermore, its performance might be subpar with models that output probabilities highly skewed towards 0 or 1.
+- At the extreme, the ROC curve could reflect high performance even when the majority of classifications are incorrect, provided that the model's ranking format is retained. This phenomenon is commonly termed the "Class Imbalance Problem".
diff --git a/docs/validmind/tests/model_validation/sklearn/RegressionErrors.qmd b/docs/validmind/tests/model_validation/sklearn/RegressionErrors.qmd
new file mode 100644
index 000000000..9f3dd1a77
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/RegressionErrors.qmd
@@ -0,0 +1,65 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionErrors"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionErrors
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance')
+
+@tasks('regression', 'classification')
+
+defRegressionErrors(model,dataset):
+
+:::
+
+
+
+Assesses the performance and error distribution of a regression model using various error metrics.
+
+### Purpose
+
+The purpose of the Regression Errors test is to measure the performance of a regression model by calculating several error metrics. This evaluation helps determine the model's accuracy and potential issues like overfitting or bias by analyzing differences in error metrics between the training and testing datasets.
+
+### Test Mechanism
+
+The test computes the following error metrics:
+
+- **Mean Absolute Error (MAE)**: Average of the absolute differences between true values and predicted values.
+- **Mean Squared Error (MSE)**: Average of the squared differences between true values and predicted values.
+- **Root Mean Squared Error (RMSE)**: Square root of the mean squared error.
+- **Mean Absolute Percentage Error (MAPE)**: Average of the absolute differences between true values and predicted values, divided by the true values, and expressed as a percentage.
+- **Mean Bias Deviation (MBD)**: Average bias between true values and predicted values.
+
+These metrics are calculated separately for the training and testing datasets and compared to identify discrepancies.
+
+### Signs of High Risk
+
+- High values for MAE, MSE, RMSE, or MAPE indicating poor model performance.
+- Large differences in error metrics between the training and testing datasets, suggesting overfitting.
+- Significant deviation of MBD from zero, indicating systematic bias in model predictions.
+
+### Strengths
+
+- Provides a comprehensive overview of model performance through multiple error metrics.
+- Individual metrics offer specific insights, e.g., MAE for interpretability, MSE for emphasizing larger errors.
+- RMSE is useful for being in the same unit as the target variable.
+- MAPE allows the error to be expressed as a percentage.
+- MBD detects systematic bias in model predictions.
+
+### Limitations
+
+- MAE and MSE are sensitive to outliers.
+- RMSE heavily penalizes larger errors, which might not always be desirable.
+- MAPE can be misleading when actual values are near zero.
+- MBD may not be suitable if bias varies with the magnitude of actual values.
+- These metrics may not capture all nuances of model performance and should be interpreted with domain-specific context.
diff --git a/docs/validmind/tests/model_validation/sklearn/RegressionErrorsComparison.qmd b/docs/validmind/tests/model_validation/sklearn/RegressionErrorsComparison.qmd
new file mode 100644
index 000000000..75818f819
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/RegressionErrorsComparison.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionErrorsComparison"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionErrorsComparison
+
+
+
+::: {.signature}
+
+@tags('model_performance', 'sklearn')
+
+@tasks('regression', 'time_series_forecasting')
+
+defRegressionErrorsComparison(datasets,models):
+
+:::
+
+
+
+Assesses multiple regression error metrics to compare model performance across different datasets, emphasizing systematic overestimation or underestimation and large percentage errors.
+
+### Purpose
+
+The purpose of this test is to compare regression errors for different models applied to various datasets. It aims to examine model performance using multiple error metrics, thereby identifying areas where models may be underperforming or exhibiting bias.
+
+### Test Mechanism
+
+The function iterates through each dataset-model pair and calculates various error metrics, including Mean Absolute Error (MAE), Mean Squared Error (MSE), Mean Absolute Percentage Error (MAPE), and Mean Bias Deviation (MBD). The results are summarized in a table, which provides a comprehensive view of each model's performance on the datasets.
+
+### Signs of High Risk
+
+- High Mean Absolute Error (MAE) or Mean Squared Error (MSE), indicating poor model performance.
+- High Mean Absolute Percentage Error (MAPE), suggesting large percentage errors, especially problematic if the true values are small.
+- Mean Bias Deviation (MBD) significantly different from zero, indicating systematic overestimation or underestimation by the model.
+
+### Strengths
+
+- Provides multiple error metrics to assess model performance from different perspectives.
+- Includes a check to avoid division by zero when calculating MAPE.
+
+### Limitations
+
+- Assumes that the dataset is provided as a DataFrameDataset object with `y`, `y_pred`, and `feature_columns` attributes.
+- Relies on the `logger` from `validmind.logging` to warn about zero values in `y_true`, which should be correctly implemented and imported.
+- Requires that `dataset.y_pred(model)` returns the predicted values for the model.
diff --git a/docs/validmind/tests/model_validation/sklearn/RegressionPerformance.qmd b/docs/validmind/tests/model_validation/sklearn/RegressionPerformance.qmd
new file mode 100644
index 000000000..c2df7cf1f
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/RegressionPerformance.qmd
@@ -0,0 +1,50 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionPerformance"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionPerformance
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance')
+
+@tasks('regression')
+
+defRegressionPerformance(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Evaluates the performance of a regression model using five different metrics: MAE, MSE, RMSE, MAPE, and MBD.
+
+### Purpose
+
+The Regression Models Performance Comparison metric is used to measure the performance of regression models. It calculates multiple evaluation metrics, including Mean Absolute Error (MAE), Mean Squared Error (MSE), Root Mean Squared Error (RMSE), Mean Absolute Percentage Error (MAPE), and Mean Bias Deviation (MBD), thereby enabling a comprehensive view of model performance.
+
+### Test Mechanism
+
+The test uses the sklearn library to calculate the MAE, MSE, RMSE, MAPE, and MBD. These calculations encapsulate both the direction and the magnitude of error in predictions, thereby providing a multi-faceted view of model accuracy.
+
+### Signs of High Risk
+
+- High values of MAE, MSE, RMSE, and MAPE, which indicate a high error rate and imply a larger departure of the model's predictions from the true values.
+- A large value of MBD, which shows a consistent bias in the model’s predictions.
+
+### Strengths
+
+- The metric evaluates models on five different metrics offering a comprehensive analysis of model performance.
+- It is designed to handle regression tasks and can be seamlessly integrated with libraries like sklearn.
+
+### Limitations
+
+- The metric only evaluates regression models and does not evaluate classification models.
+- The test assumes that the models have been trained and tested appropriately prior to evaluation. It does not handle pre-processing, feature selection, or other stages in the model lifecycle.
diff --git a/docs/validmind/tests/model_validation/sklearn/RegressionR2Square.qmd b/docs/validmind/tests/model_validation/sklearn/RegressionR2Square.qmd
new file mode 100644
index 000000000..36dd19d9a
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/RegressionR2Square.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionR2Square"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionR2Square
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance')
+
+@tasks('regression')
+
+defRegressionR2Square(dataset,model):
+
+:::
+
+
+
+Assesses the overall goodness-of-fit of a regression model by evaluating R-squared (R2) and Adjusted R-squared (Adj R2) scores to determine the model's explanatory power over the dependent variable.
+
+### Purpose
+
+The purpose of the RegressionR2Square Metric test is to measure the overall goodness-of-fit of a regression model. Specifically, this Python-based test evaluates the R-squared (R2) and Adjusted R-squared (Adj R2) scores, which are statistical measures used to assess the strength of the relationship between the model's predictors and the response variable.
+
+### Test Mechanism
+
+The test deploys the `r2_score` method from the Scikit-learn metrics module to measure the R2 score on both training and test sets. This score reflects the proportion of the variance in the dependent variable that is predictable from the independent variables. The test also calculates the Adjusted R2 score, which accounts for the number of predictors in the model to penalize model complexity and reduce overfitting. The Adjusted R2 score will be smaller if unnecessary predictors are included in the model.
+
+### Signs of High Risk
+
+- Low R2 or Adjusted R2 scores, suggesting that the model does not explain much variation in the dependent variable.
+- Significant discrepancy between R2 scores on the training set and test set, indicating overfitting and poor generalization to unseen data.
+
+### Strengths
+
+- Widely-used measure in regression analysis, providing a sound general indication of model performance.
+- Easy to interpret and understand, as it represents the proportion of the dependent variable's variance explained by the independent variables.
+- Adjusted R2 score helps control overfitting by penalizing unnecessary predictors.
+
+### Limitations
+
+- Sensitive to the inclusion of unnecessary predictors even though Adjusted R2 penalizes complexity.
+- Less reliable in cases of non-linear relationships or when the underlying assumptions of linear regression are violated.
+- Does not provide insight on whether the correct regression model was used or if key assumptions have been met.
diff --git a/docs/validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.qmd b/docs/validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.qmd
new file mode 100644
index 000000000..6dd1ab46d
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.qmd
@@ -0,0 +1,57 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionR2SquareComparison"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionR2SquareComparison
+
+
+
+::: {.signature}
+
+@tags('model_performance', 'sklearn')
+
+@tasks('regression', 'time_series_forecasting')
+
+defRegressionR2SquareComparison(datasets,models):
+
+:::
+
+
+
+Compares R-Squared and Adjusted R-Squared values for different regression models across multiple datasets to assess model performance and relevance of features.
+
+### Purpose
+
+The Regression R2 Square Comparison test aims to compare the R-Squared and Adjusted R-Squared values for different regression models across various datasets. It helps in assessing how well each model explains the variability in the dataset, and whether the models include irrelevant features.
+
+### Test Mechanism
+
+This test operates by:
+
+- Iterating through each dataset-model pair.
+- Calculating the R-Squared values to measure how much of the variability in the dataset is explained by the model.
+- Calculating the Adjusted R-Squared values, which adjust the R-Squared based on the number of predictors in the model, making it more reliable when comparing models with different numbers of features.
+- Generating a summary table containing these values for each combination of dataset and model.
+
+### Signs of High Risk
+
+- If the R-Squared values are significantly low, it indicates the model isn't explaining much of the variability in the dataset.
+- A significant difference between R-Squared and Adjusted R-Squared values might indicate that the model includes irrelevant features.
+
+### Strengths
+
+- Provides a quantitative measure of model performance in terms of variance explained.
+- Adjusted R-Squared accounts for the number of predictors, making it a more reliable measure when comparing models with different numbers of features.
+- Useful for time-series forecasting and regression tasks.
+
+### Limitations
+
+- Assumes the dataset is provided as a DataFrameDataset object with `y`, `y_pred`, and `feature_columns` attributes.
+- Relies on `adj_r2_score` from the `statsmodels.statsutils` module, which needs to be correctly implemented and imported.
+- Requires that `dataset.y_pred(model)` returns the predicted values for the model.
diff --git a/docs/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.qmd b/docs/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.qmd
new file mode 100644
index 000000000..bc848d43d
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.qmd
@@ -0,0 +1,57 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RobustnessDiagnosis"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RobustnessDiagnosis
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_diagnosis', 'visualization')
+
+@tasks('classification', 'regression')
+
+defRobustnessDiagnosis(datasets:List\[validmind.vm_models.VMDataset\],model:validmind.vm_models.VMModel,metric:str=None,scaling_factor_std_dev_list:List\[float\]=DEFAULT_STD_DEV_LIST,performance_decay_threshold:float=DEFAULT_DECAY_THRESHOLD):
+
+:::
+
+
+
+Assesses the robustness of a machine learning model by evaluating performance decay under noisy conditions.
+
+### Purpose
+
+The Robustness Diagnosis test aims to evaluate the resilience of a machine learning model when subjected to perturbations or noise in its input data. This is essential for understanding the model's ability to handle real-world scenarios where data may be imperfect or corrupted.
+
+### Test Mechanism
+
+This test introduces Gaussian noise to the numeric input features of the datasets at varying scales of standard deviation. The performance of the model is then measured using a specified metric. The process includes:
+
+- Adding Gaussian noise to numerical input features based on scaling factors.
+- Evaluating the model's performance on the perturbed data using metrics like AUC for classification tasks and MSE for regression tasks.
+- Aggregating and plotting the results to visualize performance decay relative to perturbation size.
+
+### Signs of High Risk
+
+- A significant drop in performance metrics with minimal noise.
+- Performance decay values exceeding the specified threshold.
+- Consistent failure to meet performance standards across multiple perturbation scales.
+
+### Strengths
+
+- Provides insights into the model's robustness against noisy or corrupted data.
+- Utilizes a variety of performance metrics suitable for both classification and regression tasks.
+- Visualization helps in understanding the extent of performance degradation.
+
+### Limitations
+
+- Gaussian noise might not adequately represent all types of real-world data perturbations.
+- Performance thresholds are somewhat arbitrary and might need tuning.
+- The test may not account for more complex or unstructured noise patterns that could affect model robustness.
diff --git a/docs/validmind/tests/model_validation/sklearn/SHAPGlobalImportance.qmd b/docs/validmind/tests/model_validation/sklearn/SHAPGlobalImportance.qmd
new file mode 100644
index 000000000..70f4d97bb
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/SHAPGlobalImportance.qmd
@@ -0,0 +1,112 @@
+---
+title: "[validmind](/validmind/validmind.qmd).SHAPGlobalImportance"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## generate_shap_plot
+
+
+
+::: {.signature}
+
+defgenerate_shap_plot(type\_:str,shap_values:np.ndarray,x_test:Union\[np.ndarray, pd.DataFrame\]) → plt.Figure:
+
+:::
+
+
+
+Plots two types of SHAP global importance (SHAP).
+
+**Arguments**
+
+- `type_`: The type of SHAP plot to generate. Must be "mean" or "summary".
+- `shap_values`: The SHAP values to plot.
+- `x_test`: The test data used to generate the SHAP values.
+
+**Returns**
+
+- The generated plot.
+
+
+
+## select_shap_values
+
+
+
+::: {.signature}
+
+defselect_shap_values(shap_values:Union\[np.ndarray, List\[np.ndarray\]\],class_of_interest:Optional\[int\]=None) → np.ndarray:
+
+:::
+
+
+
+Selects SHAP values for binary or multiclass classification.
+
+For regression models, returns the SHAP values directly as there are no classes.
+
+**Arguments**
+
+- `shap_values`: The SHAP values returned by the SHAP explainer. For multiclass classification, this will be a list where each element corresponds to a class. For regression, this will be a single array of SHAP values.
+- `class_of_interest`: The class index for which to retrieve SHAP values. If None (default), the function will assume binary classification and use class 1 by default.
+
+**Returns**
+
+- The SHAP values for the specified class (classification) or for the regression output.
+
+**Raises**
+
+- `ValueError`: If class_of_interest is specified and is out of bounds for the number of classes.
+
+
+
+## SHAPGlobalImportance
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'feature_importance', 'visualization')
+
+@tasks('classification', 'text_classification')
+
+defSHAPGlobalImportance(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset,kernel_explainer_samples:int=10,tree_or_linear_explainer_samples:int=200,class_of_interest:Optional\[int\]=None) → Dict\[str, Union\[plt.Figure, Dict\[str, float\]\]\]:
+
+:::
+
+
+
+Evaluates and visualizes global feature importance using SHAP values for model explanation and risk identification.
+
+### Purpose
+
+The SHAP (SHapley Additive exPlanations) Global Importance metric aims to elucidate model outcomes by attributing them to the contributing features. It assigns a quantifiable global importance to each feature via their respective absolute Shapley values, thereby making it suitable for tasks like classification (both binary and multiclass). This metric forms an essential part of model risk management.
+
+### Test Mechanism
+
+The exam begins with the selection of a suitable explainer which aligns with the model's type. For tree-based models like XGBClassifier, RandomForestClassifier, CatBoostClassifier, TreeExplainer is used whereas for linear models like LogisticRegression, XGBRegressor, LinearRegression, it is the LinearExplainer. Once the explainer calculates the Shapley values, these values are visualized using two specific graphical representations:
+
+1. Mean Importance Plot: This graph portrays the significance of individual features based on their absolute Shapley values. It calculates the average of these absolute Shapley values across all instances to highlight the global importance of features.
+
+1. Summary Plot: This visual tool combines the feature importance with their effects. Every dot on this chart represents a Shapley value for a certain feature in a specific case. The vertical axis is denoted by the feature whereas the horizontal one corresponds to the Shapley value. A color gradient indicates the value of the feature, gradually changing from low to high. Features are systematically organized in accordance with their importance.
+
+### Signs of High Risk
+
+- Overemphasis on certain features in SHAP importance plots, thus hinting at the possibility of model overfitting
+- Anomalies such as unexpected or illogical features showing high importance, which might suggest that the model's decisions are rooted in incorrect or undesirable reasoning
+- A SHAP summary plot filled with high variability or scattered data points, indicating a cause for concern
+
+### Strengths
+
+- SHAP does more than just illustrating global feature significance, it offers a detailed perspective on how different features shape the model's decision-making logic for each instance.
+- It provides clear insights into model behavior.
+
+### Limitations
+
+- High-dimensional data can convolute interpretations.
+- Associating importance with tangible real-world impact still involves a certain degree of subjectivity.
diff --git a/docs/validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.qmd b/docs/validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.qmd
new file mode 100644
index 000000000..b3d5e9938
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.qmd
@@ -0,0 +1,73 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ScoreProbabilityAlignment"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ScoreProbabilityAlignment
+
+
+
+::: {.signature}
+
+@tags('visualization', 'credit_risk', 'calibration')
+
+@tasks('classification')
+
+defScoreProbabilityAlignment(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset,score_column:str='score',n_bins:int=10):
+
+:::
+
+
+
+Analyzes the alignment between credit scores and predicted probabilities.
+
+### Purpose
+
+The Score-Probability Alignment test evaluates how well credit scores align with predicted default probabilities. This helps validate score scaling, identify potential calibration issues, and ensure scores reflect risk appropriately.
+
+### Test Mechanism
+
+The test:
+
+1. Groups scores into bins
+1. Calculates average predicted probability per bin
+1. Tests monotonicity of relationship
+1. Analyzes probability distribution within score bands
+
+### Signs of High Risk
+
+- Non-monotonic relationship between scores and probabilities
+- Large probability variations within score bands
+- Unexpected probability jumps between adjacent bands
+- Poor alignment with expected odds-to-score relationship
+- Inconsistent probability patterns across score ranges
+- Clustering of probabilities at extreme values
+- Score bands with similar probability profiles
+- Unstable probability estimates in key decision bands
+
+### Strengths
+
+- Direct validation of score-to-probability relationship
+- Identifies potential calibration issues
+- Supports score band validation
+- Helps understand model behavior
+- Useful for policy setting
+- Visual and numerical results
+- Easy to interpret
+- Supports regulatory documentation
+
+### Limitations
+
+- Sensitive to bin selection
+- Requires sufficient data per bin
+- May mask within-bin variations
+- Point-in-time analysis only
+- Cannot detect all forms of miscalibration
+- Assumes scores should align with probabilities
+- May oversimplify complex relationships
+- Limited to binary outcomes
diff --git a/docs/validmind/tests/model_validation/sklearn/SilhouettePlot.qmd b/docs/validmind/tests/model_validation/sklearn/SilhouettePlot.qmd
new file mode 100644
index 000000000..ebeb63e69
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/SilhouettePlot.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).SilhouettePlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## SilhouettePlot
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance')
+
+@tasks('clustering')
+
+defSilhouettePlot(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Calculates and visualizes Silhouette Score, assessing the degree of data point suitability to its cluster in ML models.
+
+### Purpose
+
+This test calculates the Silhouette Score, which is a model performance metric used in clustering applications. Primarily, the Silhouette Score evaluates how similar a data point is to its own cluster compared to other clusters. The metric ranges between -1 and 1, where a high value indicates that the object is well matched to its own cluster and poorly matched to neighboring clusters. Thus, the goal is to achieve a high Silhouette Score, implying well-separated clusters.
+
+### Test Mechanism
+
+The test first extracts the true and predicted labels from the model's training data. The test runs the Silhouette Score function, which takes as input the training dataset features and the predicted labels, subsequently calculating the average score. This average Silhouette Score is printed for reference. The script then calculates the silhouette coefficients for each data point, helping to form the Silhouette Plot. Each cluster is represented in this plot, with color distinguishing between different clusters. A red dashed line indicates the average Silhouette Score. The Silhouette Scores are also collected into a structured table, facilitating model performance analysis and comparison.
+
+### Signs of High Risk
+
+- A low Silhouette Score, potentially indicating that the clusters are not well separated and that data points may not be fitting well to their respective clusters.
+- A Silhouette Plot displaying overlapping clusters or the absence of clear distinctions between clusters visually also suggests poor clustering performance.
+
+### Strengths
+
+- The Silhouette Score provides a clear and quantitative measure of how well data points have been grouped into clusters, offering insights into model performance.
+- The Silhouette Plot provides an intuitive, graphical representation of the clustering mechanism, aiding visual assessments of model performance.
+- It does not require ground truth labels, so it's useful when true cluster assignments are not known.
+
+### Limitations
+
+- The Silhouette Score may be susceptible to the influence of outliers, which could impact its accuracy and reliability.
+- It assumes the clusters are convex and isotropic, which might not be the case with complex datasets.
+- Due to the average nature of the Silhouette Score, the metric does not account for individual data point assignment nuances, so potentially relevant details may be omitted.
+- Computationally expensive for large datasets, as it requires pairwise distance computations.
diff --git a/docs/validmind/tests/model_validation/sklearn/TrainingTestDegradation.qmd b/docs/validmind/tests/model_validation/sklearn/TrainingTestDegradation.qmd
new file mode 100644
index 000000000..11c84b21b
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/TrainingTestDegradation.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).TrainingTestDegradation"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## TrainingTestDegradation
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'model_performance', 'visualization')
+
+@tasks('classification', 'text_classification')
+
+defTrainingTestDegradation(datasets:List\[validmind.vm_models.VMDataset\],model:validmind.vm_models.VMModel,max_threshold:float=0.1):
+
+:::
+
+
+
+Tests if model performance degradation between training and test datasets exceeds a predefined threshold.
+
+### Purpose
+
+The `TrainingTestDegradation` class serves as a test to verify that the degradation in performance between the training and test datasets does not exceed a predefined threshold. This test measures the model's ability to generalize from its training data to unseen test data, assessing key classification metrics such as accuracy, precision, recall, and f1 score to verify the model's robustness and reliability.
+
+### Test Mechanism
+
+The code applies several predefined metrics, including accuracy, precision, recall, and f1 scores, to the model's predictions for both the training and test datasets. It calculates the degradation as the difference between the training score and test score divided by the training score. The test is considered successful if the degradation for each metric is less than the preset maximum threshold of 10%. The results are summarized in a table showing each metric's train score, test score, degradation percentage, and pass/fail status.
+
+### Signs of High Risk
+
+- A degradation percentage that exceeds the maximum allowed threshold of 10% for any of the evaluated metrics.
+- A high difference or gap between the metric scores on the training and the test datasets.
+- The 'Pass/Fail' column displaying 'Fail' for any of the evaluated metrics.
+
+### Strengths
+
+- Provides a quantitative measure of the model's ability to generalize to unseen data, which is key for predicting its practical real-world performance.
+- By evaluating multiple metrics, it takes into account different facets of model performance and enables a more holistic evaluation.
+- The use of a variable predefined threshold allows the flexibility to adjust the acceptability criteria for different scenarios.
+
+### Limitations
+
+- The test compares raw performance on training and test data but does not factor in the nature of the data. Areas with less representation in the training set might still perform poorly on unseen data.
+- It requires good coverage and balance in the test and training datasets to produce reliable results, which may not always be available.
+- The test is currently only designed for classification tasks.
diff --git a/docs/validmind/tests/model_validation/sklearn/VMeasure.qmd b/docs/validmind/tests/model_validation/sklearn/VMeasure.qmd
new file mode 100644
index 000000000..d4ee8dbe4
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/VMeasure.qmd
@@ -0,0 +1,49 @@
+---
+title: "[validmind](/validmind/validmind.qmd).VMeasure"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## VMeasure
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'model_performance')
+
+@tasks('clustering')
+
+defVMeasure(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel):
+
+:::
+
+
+
+Evaluates homogeneity and completeness of a clustering model using the V Measure Score.
+
+### Purpose
+
+The purpose of this metric, V Measure Score (V Score), is to evaluate the performance of a clustering model. It measures the homogeneity and completeness of a set of cluster labels, where homogeneity refers to each cluster containing only members of a single class and completeness meaning all members of a given class are assigned to the same cluster.
+
+### Test Mechanism
+
+ClusterVMeasure is a class that inherits from another class, ClusterPerformance. It uses the `v_measure_score` function from the sklearn module's metrics package. The required inputs to perform this metric are the model, train dataset, and test dataset. The test is appropriate for models tasked with clustering.
+
+### Signs of High Risk
+
+- Low V Measure Score: A low V Measure Score indicates that the clustering model has poor homogeneity or completeness, or both. This might signal that the model is failing to correctly cluster the data.
+
+### Strengths
+
+- The V Measure Score is a harmonic mean between homogeneity and completeness. This ensures that both attributes are taken into account when evaluating the model, providing an overall measure of its cluster validity.
+- The metric does not require knowledge of the ground truth classes when measuring homogeneity and completeness, making it applicable in instances where such information is unavailable.
+
+### Limitations
+
+- The V Measure Score can be influenced by the number of clusters, which means that it might not always reflect the quality of the clustering. Partitioning the data into many small clusters could lead to high homogeneity but low completeness, leading to a low V Measure Score even if the clustering might be useful.
+- It assumes equal importance of homogeneity and completeness. In some applications, one may be more important than the other. The V Measure Score does not provide flexibility in assigning different weights to homogeneity and completeness.
diff --git a/docs/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.qmd b/docs/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.qmd
new file mode 100644
index 000000000..236cefe91
--- /dev/null
+++ b/docs/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).WeakspotsDiagnosis"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## WeakspotsDiagnosis
+
+
+
+::: {.signature}
+
+@tags('sklearn', 'binary_classification', 'multiclass_classification', 'model_diagnosis', 'visualization')
+
+@tasks('classification', 'text_classification')
+
+defWeakspotsDiagnosis(datasets:List\[validmind.vm_models.VMDataset\],model:validmind.vm_models.VMModel,features_columns:Union\[List\[str\], None\]=None,metrics:Union\[Dict\[str, Callable\], None\]=None,thresholds:Union\[Dict\[str, float\], None\]=None):
+
+:::
+
+
+
+Identifies and visualizes weak spots in a machine learning model's performance across various sections of the feature space.
+
+### Purpose
+
+The weak spots test is applied to evaluate the performance of a machine learning model within specific regions of its feature space. This test slices the feature space into various sections, evaluating the model's outputs within each section against specific performance metrics (e.g., accuracy, precision, recall, and F1 scores). The ultimate aim is to identify areas where the model's performance falls below the set thresholds, thereby exposing its possible weaknesses and limitations.
+
+### Test Mechanism
+
+The test mechanism adopts an approach of dividing the feature space of the training dataset into numerous bins. The model's performance metrics (accuracy, precision, recall, F1 scores) are then computed for each bin on both the training and test datasets. A "weak spot" is identified if any of the performance metrics fall below a predetermined threshold for a particular bin on the test dataset. The test results are visually plotted as bar charts for each performance metric, indicating the bins which fail to meet the established threshold.
+
+### Signs of High Risk
+
+- Any performance metric of the model dropping below the set thresholds.
+- Significant disparity in performance between the training and test datasets within a bin could be an indication of overfitting.
+- Regions or slices with consistently low performance metrics. Such instances could mean that the model struggles to handle specific types of input data adequately, resulting in potentially inaccurate predictions.
+
+### Strengths
+
+- The test helps pinpoint precise regions of the feature space where the model's performance is below par, allowing for more targeted improvements to the model.
+- The graphical presentation of the performance metrics offers an intuitive way to understand the model's performance across different feature areas.
+- The test exhibits flexibility, letting users set different thresholds for various performance metrics according to the specific requirements of the application.
+
+### Limitations
+
+- The binning system utilized for the feature space in the test could over-simplify the model's behavior within each bin. The granularity of this slicing depends on the chosen 'bins' parameter and can sometimes be arbitrary.
+- The effectiveness of this test largely hinges on the selection of thresholds for the performance metrics, which may not hold universally applicable and could be subjected to the specifications of a particular model and application.
+- The test is unable to handle datasets with a text column, limiting its application to numerical or categorical data types only.
+- Despite its usefulness in highlighting problematic regions, the test does not offer direct suggestions for model improvement.
diff --git a/docs/validmind/tests/model_validation/statsmodels.qmd b/docs/validmind/tests/model_validation/statsmodels.qmd
new file mode 100644
index 000000000..3259ec7c0
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels.qmd
@@ -0,0 +1,24 @@
+---
+title: "[validmind](/validmind/validmind.qmd).statsmodels"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+- [AutoARIMA](statsmodels/AutoARIMA.qmd)
+- [CumulativePredictionProbabilities](statsmodels/CumulativePredictionProbabilities.qmd)
+- [DurbinWatsonTest](statsmodels/DurbinWatsonTest.qmd)
+- [GINITable](statsmodels/GINITable.qmd)
+- [KolmogorovSmirnov](statsmodels/KolmogorovSmirnov.qmd)
+- [Lilliefors](statsmodels/Lilliefors.qmd)
+- [PredictionProbabilitiesHistogram](statsmodels/PredictionProbabilitiesHistogram.qmd)
+- [RegressionCoeffs](statsmodels/RegressionCoeffs.qmd)
+- [RegressionFeatureSignificance](statsmodels/RegressionFeatureSignificance.qmd)
+- [RegressionModelForecastPlot](statsmodels/RegressionModelForecastPlot.qmd)
+- [RegressionModelForecastPlotLevels](statsmodels/RegressionModelForecastPlotLevels.qmd)
+- [RegressionModelSensitivityPlot](statsmodels/RegressionModelSensitivityPlot.qmd)
+- [RegressionModelSummary](statsmodels/RegressionModelSummary.qmd)
+- [RegressionPermutationFeatureImportance](statsmodels/RegressionPermutationFeatureImportance.qmd)
+- [ScorecardHistogram](statsmodels/ScorecardHistogram.qmd)
+- [statsutils](statsmodels/statsutils.qmd)
diff --git a/docs/validmind/tests/model_validation/statsmodels/AutoARIMA.qmd b/docs/validmind/tests/model_validation/statsmodels/AutoARIMA.qmd
new file mode 100644
index 000000000..279ea069f
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/AutoARIMA.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).AutoARIMA"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## AutoARIMA
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'forecasting', 'model_selection', 'statsmodels')
+
+@tasks('regression')
+
+defAutoARIMA(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Evaluates ARIMA models for time-series forecasting, ranking them using Bayesian and Akaike Information Criteria.
+
+### Purpose
+
+The AutoARIMA validation test is designed to evaluate and rank AutoRegressive Integrated Moving Average (ARIMA) models. These models are primarily used for forecasting time-series data. The validation test automatically fits multiple ARIMA models, with varying parameters, to every variable within the given dataset. The models are then ranked based on their Bayesian Information Criterion (BIC) and Akaike Information Criterion (AIC) values, which provide a basis for the efficient model selection process.
+
+### Test Mechanism
+
+This metric proceeds by generating an array of feasible combinations of ARIMA model parameters which are within a prescribed limit. These limits include `max_p`, `max_d`, `max_q`; they represent the autoregressive, differencing, and moving average components respectively. Upon applying these sets of parameters, the validation test fits each ARIMA model to the time-series data provided. For each model, it subsequently proceeds to calculate and record both the BIC and AIC values, which serve as performance indicators for the model fit. Prior to this parameter fitting process, the Augmented Dickey-Fuller test for data stationarity is conducted on the data series. If a series is found to be non-stationary, a warning message is sent out, given that ARIMA models necessitate input series to be stationary.
+
+### Signs of High Risk
+
+- If the p-value of the Augmented Dickey-Fuller test for a variable exceeds 0.05, a warning is logged. This warning indicates that the series might not be stationary, leading to potentially inaccurate results.
+- Consistent failure in fitting ARIMA models (as made evident through logged errors) might disclose issues with either the data or model stability.
+
+### Strengths
+
+- The AutoARIMA validation test simplifies the often complex task of selecting the most suitable ARIMA model based on BIC and AIC criteria.
+- The mechanism incorporates a check for non-stationarity within the data, which is a critical prerequisite for ARIMA models.
+- The exhaustive search through all possible combinations of model parameters enhances the likelihood of identifying the best-fit model.
+
+### Limitations
+
+- This validation test can be computationally costly as it involves creating and fitting multiple ARIMA models for every variable.
+- Although the test checks for non-stationarity and logs warnings where present, it does not apply any transformations to the data to establish stationarity.
+- The selection of models leans solely on BIC and AIC criteria, which may not yield the best predictive model in all scenarios.
+- The test is only applicable to regression tasks involving time-series data, and may not work effectively for other types of machine learning tasks.
diff --git a/docs/validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.qmd b/docs/validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.qmd
new file mode 100644
index 000000000..ab8c922d2
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).CumulativePredictionProbabilities"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## CumulativePredictionProbabilities
+
+
+
+::: {.signature}
+
+@tags('visualization', 'credit_risk')
+
+@tasks('classification')
+
+defCumulativePredictionProbabilities(dataset,model,title='Cumulative Probabilities'):
+
+:::
+
+
+
+Visualizes cumulative probabilities of positive and negative classes for both training and testing in classification models.
+
+### Purpose
+
+This metric is utilized to evaluate the distribution of predicted probabilities for positive and negative classes in a classification model. It provides a visual assessment of the model's behavior by plotting the cumulative probabilities for positive and negative classes across both the training and test datasets.
+
+### Test Mechanism
+
+The classification model is evaluated by first computing the predicted probabilities for each instance in both the training and test datasets, which are then added as a new column in these sets. The cumulative probabilities for positive and negative classes are subsequently calculated and sorted in ascending order. Cumulative distributions of these probabilities are created for both positive and negative classes across both training and test datasets. These cumulative probabilities are represented visually in a plot, containing two subplots - one for the training data and the other for the test data, with lines representing cumulative distributions of positive and negative classes.
+
+### Signs of High Risk
+
+- Imbalanced distribution of probabilities for either positive or negative classes.
+- Notable discrepancies or significant differences between the cumulative probability distributions for the training data versus the test data.
+- Marked discrepancies or large differences between the cumulative probability distributions for positive and negative classes.
+
+### Strengths
+
+- Provides a visual illustration of data, which enhances the ease of understanding and interpreting the model's behavior.
+- Allows for the comparison of model's behavior across training and testing datasets, providing insights about how well the model is generalized.
+- Differentiates between positive and negative classes and their respective distribution patterns, aiding in problem diagnosis.
+
+### Limitations
+
+- Exclusive to classification tasks and specifically to classification models.
+- Graphical results necessitate human interpretation and may not be directly applicable for automated risk detection.
+- The method does not give a solitary quantifiable measure of model risk, instead, it offers a visual representation and broad distributional information.
+- If the training and test datasets are not representative of the overall data distribution, the metric could provide misleading results.
diff --git a/docs/validmind/tests/model_validation/statsmodels/DurbinWatsonTest.qmd b/docs/validmind/tests/model_validation/statsmodels/DurbinWatsonTest.qmd
new file mode 100644
index 000000000..9f5138dbf
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/DurbinWatsonTest.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).DurbinWatsonTest"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## DurbinWatsonTest
+
+
+
+::: {.signature}
+
+@tasks('regression')
+
+@tags('time_series_data', 'forecasting', 'statistical_test', 'statsmodels')
+
+defDurbinWatsonTest(dataset,model,threshold=\[1.5, 2.5\]):
+
+:::
+
+
+
+Assesses autocorrelation in time series data features using the Durbin-Watson statistic.
+
+### Purpose
+
+The Durbin-Watson Test metric detects autocorrelation in time series data (where a set of data values influences their predecessors). Autocorrelation is a crucial factor for regression tasks as these often assume the independence of residuals. A model with significant autocorrelation may give unreliable predictions.
+
+### Test Mechanism
+
+Utilizing the `durbin_watson` function in the `statsmodels` Python library, the Durbin-Watson (DW) Test metric generates a statistical value for each feature of the training dataset. The function is looped over all columns of the dataset, calculating and caching the DW value for each column for further analysis. A DW metric value nearing 2 indicates no autocorrelation. Conversely, values approaching 0 suggest positive autocorrelation, and those leaning towards 4 imply negative autocorrelation.
+
+### Signs of High Risk
+
+- If a feature's DW value significantly deviates from 2, it could signal a high risk due to potential autocorrelation issues in the dataset.
+- A value closer to 0 could imply positive autocorrelation, while a value nearer to 4 could point to negative autocorrelation, both leading to potentially unreliable prediction models.
+
+### Strengths
+
+- The metric specializes in identifying autocorrelation in prediction model residuals.
+- Autocorrelation detection assists in diagnosing violation of various modeling technique assumptions, particularly in regression analysis and time-series data modeling.
+
+### Limitations
+
+- The Durbin-Watson Test mainly detects linear autocorrelation and could overlook other types of relationships.
+- The metric is highly sensitive to data points order. Shuffling the order could lead to notably different results.
+- The test only checks for first-order autocorrelation (between a variable and its immediate predecessor) and fails to detect higher-order autocorrelation.
diff --git a/docs/validmind/tests/model_validation/statsmodels/GINITable.qmd b/docs/validmind/tests/model_validation/statsmodels/GINITable.qmd
new file mode 100644
index 000000000..fefcc4e35
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/GINITable.qmd
@@ -0,0 +1,55 @@
+---
+title: "[validmind](/validmind/validmind.qmd).GINITable"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## GINITable
+
+
+
+::: {.signature}
+
+@tags('model_performance')
+
+@tasks('classification')
+
+defGINITable(dataset,model):
+
+:::
+
+
+
+Evaluates classification model performance using AUC, GINI, and KS metrics for training and test datasets.
+
+### Purpose
+
+The 'GINITable' metric is designed to evaluate the performance of a classification model by emphasizing its discriminatory power. Specifically, it calculates and presents three important metrics - the Area under the ROC Curve (AUC), the GINI coefficient, and the Kolmogorov-Smirnov (KS) statistic - for both training and test datasets.
+
+### Test Mechanism
+
+Using a dictionary for storing performance metrics for both the training and test datasets, the 'GINITable' metric calculates each of these metrics sequentially. The Area under the ROC Curve (AUC) is calculated via the `roc_auc_score` function from the Scikit-Learn library. The GINI coefficient, a measure of statistical dispersion, is then computed by doubling the AUC and subtracting 1. Finally, the Kolmogorov-Smirnov (KS) statistic is calculated via the `roc_curve` function from Scikit-Learn, with the False Positive Rate (FPR) subtracted from the True Positive Rate (TPR) and the maximum value taken from the resulting data. These metrics are then stored in a pandas DataFrame for convenient visualization.
+
+### Signs of High Risk
+
+- Low values for performance metrics may suggest a reduction in model performance, particularly a low AUC which indicates poor classification performance, or a low GINI coefficient, which could suggest a decreased ability to discriminate different classes.
+- A high KS value may be an indicator of potential overfitting, as this generally signifies a substantial divergence between positive and negative distributions.
+- Significant discrepancies between the performance on the training dataset and the test dataset may present another signal of high risk.
+
+### Strengths
+
+- Offers three key performance metrics (AUC, GINI, and KS) in one test, providing a more comprehensive evaluation of the model.
+- Provides a direct comparison between the model's performance on training and testing datasets, which aids in identifying potential underfitting or overfitting.
+- The applied metrics are class-distribution invariant, thereby remaining effective for evaluating model performance even when dealing with imbalanced datasets.
+- Presents the metrics in a user-friendly table format for easy comprehension and analysis.
+
+### Limitations
+
+- The GINI coefficient and KS statistic are both dependent on the AUC value. Therefore, any errors in the calculation of the latter will adversely impact the former metrics too.
+- Mainly suited for binary classification models and may require modifications for effective application in multi-class scenarios.
+- The metrics used are threshold-dependent and may exhibit high variability based on the chosen cut-off points.
+- The test does not incorporate a method to efficiently handle missing or inefficiently processed data, which could lead to inaccuracies in the metrics if the data is not appropriately preprocessed.
diff --git a/docs/validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.qmd b/docs/validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.qmd
new file mode 100644
index 000000000..11f34d86b
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).KolmogorovSmirnov"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## KolmogorovSmirnov
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'data_distribution', 'statistical_test', 'statsmodels')
+
+@tasks('classification', 'regression')
+
+defKolmogorovSmirnov(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset,dist:str='norm'):
+
+:::
+
+
+
+Assesses whether each feature in the dataset aligns with a normal distribution using the Kolmogorov-Smirnov test.
+
+### Purpose
+
+The Kolmogorov-Smirnov (KS) test evaluates the distribution of features in a dataset to determine their alignment with a normal distribution. This is important because many statistical methods and machine learning models assume normality in the data distribution.
+
+### Test Mechanism
+
+This test calculates the KS statistic and corresponding p-value for each feature in the dataset. It does so by comparing the cumulative distribution function of the feature with an ideal normal distribution. The KS statistic and p-value for each feature are then stored in a dictionary. The p-value threshold to reject the normal distribution hypothesis is not preset, providing flexibility for different applications.
+
+### Signs of High Risk
+
+- Elevated KS statistic for a feature combined with a low p-value, indicating a significant divergence from a normal distribution.
+- Features with notable deviations that could create problems if the model assumes normality in data distribution.
+
+### Strengths
+
+- The KS test is sensitive to differences in the location and shape of empirical cumulative distribution functions.
+- It is non-parametric and adaptable to various datasets, as it does not assume any specific data distribution.
+- Provides detailed insights into the distribution of individual features.
+
+### Limitations
+
+- The test's sensitivity to disparities in the tails of data distribution might cause false alarms about non-normality.
+- Less effective for multivariate distributions, as it is designed for univariate distributions.
+- Does not identify specific types of non-normality, such as skewness or kurtosis, which could impact model fitting.
diff --git a/docs/validmind/tests/model_validation/statsmodels/Lilliefors.qmd b/docs/validmind/tests/model_validation/statsmodels/Lilliefors.qmd
new file mode 100644
index 000000000..616c53c52
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/Lilliefors.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Lilliefors"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Lilliefors
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'data_distribution', 'statistical_test', 'statsmodels')
+
+@tasks('classification', 'regression')
+
+defLilliefors(dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Assesses the normality of feature distributions in an ML model's training dataset using the Lilliefors test.
+
+### Purpose
+
+The purpose of this metric is to utilize the Lilliefors test, named in honor of the Swedish statistician Hubert Lilliefors, in order to assess whether the features of the machine learning model's training dataset conform to a normal distribution. This is done because the assumption of normal distribution plays a vital role in numerous statistical procedures as well as numerous machine learning models. Should the features fail to follow a normal distribution, some model types may not operate at optimal efficiency. This can potentially lead to inaccurate predictions.
+
+### Test Mechanism
+
+The application of this test happens across all feature columns within the training dataset. For each feature, the Lilliefors test returns a test statistic and p-value. The test statistic quantifies how far the feature's distribution is from an ideal normal distribution, whereas the p-value aids in determining the statistical relevance of this deviation. The final results are stored within a dictionary, the keys of which correspond to the name of the feature column, and the values being another dictionary which houses the test statistic and p-value.
+
+### Signs of High Risk
+
+- If the p-value corresponding to a specific feature sinks below a pre-established significance level, generally set at 0.05, then it can be deduced that the distribution of that feature significantly deviates from a normal distribution. This can present a high risk for models that assume normality, as these models may perform inaccurately or inefficiently in the presence of such a feature.
+
+### Strengths
+
+- One advantage of the Lilliefors test is its utility irrespective of whether the mean and variance of the normal distribution are known in advance. This makes it a more robust option in real-world situations where these values might not be known.
+- The test has the ability to screen every feature column, offering a holistic view of the dataset.
+
+### Limitations
+
+- Despite the practical applications of the Lilliefors test in validating normality, it does come with some limitations.
+- It is only capable of testing unidimensional data, thus rendering it ineffective for datasets with interactions between features or multi-dimensional phenomena.
+- The test might not be as sensitive as some other tests (like the Anderson-Darling test) in detecting deviations from a normal distribution.
+- Like any other statistical test, Lilliefors test may also produce false positives or negatives. Hence, banking solely on this test, without considering other characteristics of the data, may give rise to risks.
diff --git a/docs/validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.qmd b/docs/validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.qmd
new file mode 100644
index 000000000..cec69295a
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.qmd
@@ -0,0 +1,62 @@
+---
+title: "[validmind](/validmind/validmind.qmd).PredictionProbabilitiesHistogram"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## PredictionProbabilitiesHistogram
+
+
+
+::: {.signature}
+
+@tags('visualization', 'credit_risk')
+
+@tasks('classification')
+
+defPredictionProbabilitiesHistogram(dataset,model,title='Histogram of Predictive Probabilities'):
+
+:::
+
+
+
+Assesses the predictive probability distribution for binary classification to evaluate model performance and potential overfitting or bias.
+
+### Purpose
+
+The Prediction Probabilities Histogram test is designed to generate histograms displaying the Probability of Default (PD) predictions for both positive and negative classes in training and testing datasets. This helps in evaluating the performance of a classification model.
+
+### Test Mechanism
+
+The metric follows these steps to execute the test:
+
+- Extracts the target column from both the train and test datasets.
+- Uses the model's predict function to calculate probabilities.
+- Adds these probabilities as a new column to the training and testing dataframes.
+- Generates histograms for each class (0 or 1) within the training and testing datasets.
+- Sets different opacities for the histograms to enhance visualization.
+- Overlays the four histograms (two for training and two for testing) on two different subplot frames.
+- Returns a plotly graph object displaying the visualization.
+
+### Signs of High Risk
+
+- Significant discrepancies between the histograms of training and testing data.
+- Large disparities between the histograms for the positive and negative classes.
+- Potential overfitting or bias indicated by significant issues.
+- Unevenly distributed probabilities suggesting inaccurate model predictions.
+
+### Strengths
+
+- Offers a visual representation of the PD predictions made by the model, aiding in understanding its behavior.
+- Assesses both the training and testing datasets, adding depth to model validation.
+- Highlights disparities between classes, providing insights into class imbalance or data skewness.
+- Effectively visualizes risk spread, which is particularly beneficial for credit risk prediction.
+
+### Limitations
+
+- Specifically tailored for binary classification scenarios and not suited for multi-class classification tasks.
+- Provides a robust visual representation but lacks a quantifiable measure to assess model performance.
diff --git a/docs/validmind/tests/model_validation/statsmodels/RegressionCoeffs.qmd b/docs/validmind/tests/model_validation/statsmodels/RegressionCoeffs.qmd
new file mode 100644
index 000000000..999e7c2d1
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/RegressionCoeffs.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionCoeffs"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionCoeffs
+
+
+
+::: {.signature}
+
+@tags('tabular_data', 'visualization', 'model_training')
+
+@tasks('regression')
+
+defRegressionCoeffs(model):
+
+:::
+
+
+
+Assesses the significance and uncertainty of predictor variables in a regression model through visualization of coefficients and their 95% confidence intervals.
+
+### Purpose
+
+The `RegressionCoeffs` metric visualizes the estimated regression coefficients alongside their 95% confidence intervals, providing insights into the impact and significance of predictor variables on the response variable. This visualization helps to understand the variability and uncertainty in the model's estimates, aiding in the evaluation of the significance of each predictor.
+
+### Test Mechanism
+
+The function operates by extracting the estimated coefficients and their standard errors from the regression model. Using these, it calculates the confidence intervals at a 95% confidence level, which indicates the range within which the true coefficient value is expected to fall 95% of the time. The confidence intervals are computed using the Z-value associated with the 95% confidence level. The coefficients and their confidence intervals are then visualized in a bar plot. The x-axis represents the predictor variables, the y-axis represents the estimated coefficients, and the error bars depict the confidence intervals.
+
+### Signs of High Risk
+
+- The confidence interval for a coefficient contains the zero value, suggesting that the predictor may not significantly contribute to the model.
+- Multiple coefficients with confidence intervals that include zero, potentially indicating issues with model reliability.
+- Very wide confidence intervals, which may suggest high uncertainty in the coefficient estimates and potential model instability.
+
+### Strengths
+
+- Provides a clear visualization that allows for easy interpretation of the significance and impact of predictor variables.
+- Includes confidence intervals, which provide additional information about the uncertainty surrounding each coefficient estimate.
+
+### Limitations
+
+- The method assumes normality of residuals and independence of observations, assumptions that may not always hold true in practice.
+- It does not address issues related to multi-collinearity among predictor variables, which can affect the interpretation of coefficients.
+- This metric is limited to regression tasks using tabular data and is not applicable to other types of machine learning tasks or data structures.
diff --git a/docs/validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.qmd b/docs/validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.qmd
new file mode 100644
index 000000000..4a9c3653d
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionFeatureSignificance"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionFeatureSignificance
+
+
+
+::: {.signature}
+
+@tags('statistical_test', 'model_interpretation', 'visualization', 'feature_importance')
+
+@tasks('regression')
+
+defRegressionFeatureSignificance(model:validmind.vm_models.VMModel,fontsize:int=10,p_threshold:float=0.05):
+
+:::
+
+
+
+Assesses and visualizes the statistical significance of features in a regression model.
+
+### Purpose
+
+The Regression Feature Significance metric assesses the significance of each feature in a given set of regression model. It creates a visualization displaying p-values for every feature of the model, assisting model developers in understanding which features are most influential in their model.
+
+### Test Mechanism
+
+The test mechanism involves extracting the model's coefficients and p-values for each feature, and then plotting these values. The x-axis on the plot contains the p-values while the y-axis denotes the coefficients of each feature. A vertical red line is drawn at the threshold for p-value significance, which is 0.05 by default. Any features with p-values to the left of this line are considered statistically significant at the chosen level.
+
+### Signs of High Risk
+
+- Any feature with a high p-value (greater than the threshold) is considered a potential high risk, as it suggests the feature is not statistically significant and may not be reliably contributing to the model's predictions.
+- A high number of such features may indicate problems with the model validation, variable selection, and overall reliability of the model predictions.
+
+### Strengths
+
+- Helps identify the features that significantly contribute to a model's prediction, providing insights into the feature importance.
+- Provides tangible, easy-to-understand visualizations to interpret the feature significance.
+
+### Limitations
+
+- This metric assumes model features are independent, which may not always be the case. Multicollinearity (high correlation amongst predictors) can cause high variance and unreliable statistical tests of significance.
+- The p-value strategy for feature selection doesn't take into account the magnitude of the effect, focusing solely on whether the feature is likely non-zero.
+- This test is specific to regression models and wouldn't be suitable for other types of ML models.
+- P-value thresholds are somewhat arbitrary and do not always indicate practical significance, only statistical significance.
diff --git a/docs/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.qmd b/docs/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.qmd
new file mode 100644
index 000000000..ab3b8a992
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionModelForecastPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionModelForecastPlot
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'forecasting', 'visualization')
+
+@tasks('regression')
+
+defRegressionModelForecastPlot(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset,start_date:Union\[str, None\]=None,end_date:Union\[str, None\]=None):
+
+:::
+
+
+
+Generates plots to visually compare the forecasted outcomes of a regression model against actual observed values over a specified date range.
+
+### Purpose
+
+This metric is useful for time-series models or any model where the outcome changes over time, allowing direct comparison of predicted vs actual values. It can help identify overfitting or underfitting situations as well as general model performance.
+
+### Test Mechanism
+
+This test generates a plot with the x-axis representing the date ranging from the specified "start_date" to the "end_date", while the y-axis shows the value of the outcome variable. Two lines are plotted: one representing the forecasted values and the other representing the observed values. The "start_date" and "end_date" can be parameters of this test; if these parameters are not provided, they are set to the minimum and maximum date available in the dataset.
+
+### Signs of High Risk
+
+- High risk or failure signs could be deduced visually from the plots if the forecasted line significantly deviates from the observed line, indicating the model's predicted values are not matching actual outcomes.
+- A model that struggles to handle the edge conditions like maximum and minimum data points could also be considered a sign of risk.
+
+### Strengths
+
+- Visualization: The plot provides an intuitive and clear illustration of how well the forecast matches the actual values, making it straightforward even for non-technical stakeholders to interpret.
+- Flexibility: It allows comparison for multiple models and for specified time periods.
+- Model Evaluation: It can be useful in identifying overfitting or underfitting situations, as these will manifest as discrepancies between the forecasted and observed values.
+
+### Limitations
+
+- Interpretation Bias: Interpretation of the plot is subjective and can lead to different conclusions by different evaluators.
+- Lack of Precision: Visual representation might not provide precise values of the deviation.
+- Inapplicability: Limited to cases where the order of data points (time-series) matters, it might not be of much use in problems that are not related to time series prediction.
diff --git a/docs/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.qmd b/docs/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.qmd
new file mode 100644
index 000000000..b5a20af5c
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.qmd
@@ -0,0 +1,67 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionModelForecastPlotLevels"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## integrate_diff
+
+
+
+::: {.signature}
+
+defintegrate_diff(series_diff,start_value):
+
+:::
+
+
+
+## RegressionModelForecastPlotLevels
+
+
+
+::: {.signature}
+
+@tags('time_series_data', 'forecasting', 'visualization')
+
+@tasks('regression')
+
+defRegressionModelForecastPlotLevels(model:validmind.vm_models.VMModel,dataset:validmind.vm_models.VMDataset):
+
+:::
+
+
+
+Assesses the alignment between forecasted and observed values in regression models through visual plots
+
+### Purpose
+
+This test aims to visually assess the performance of a regression model by comparing its forecasted values against the actual observed values for both the raw and transformed (integrated) data. This helps determine the accuracy of the model and can help identify overfitting or underfitting. The integration is applied to highlight the trend rather than the absolute level.
+
+### Test Mechanism
+
+This test generates two plots:
+
+- Raw data vs forecast
+- Transformed data vs forecast
+
+The transformed data is created by performing a cumulative sum on the raw data.
+
+### Signs of High Risk
+
+- Significant deviation between forecasted and observed values.
+- Patterns suggesting overfitting or underfitting.
+- Large discrepancies in the plotted forecasts, indicating potential issues with model generalizability and precision.
+
+### Strengths
+
+- Provides an intuitive, visual way to assess multiple regression models, aiding in easier interpretation and evaluation of forecast accuracy.
+
+### Limitations
+
+- Relies heavily on visual interpretation, which may vary between individuals.
+- Does not provide a numerical metric to quantify forecast accuracy, relying solely on visual assessment.
diff --git a/docs/validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.qmd b/docs/validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.qmd
new file mode 100644
index 000000000..f46376133
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.qmd
@@ -0,0 +1,64 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionModelSensitivityPlot"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## integrate_diff
+
+
+
+::: {.signature}
+
+defintegrate_diff(series_diff,start_value):
+
+:::
+
+
+
+## RegressionModelSensitivityPlot
+
+
+
+::: {.signature}
+
+@tags('senstivity_analysis', 'visualization')
+
+@tasks('regression')
+
+defRegressionModelSensitivityPlot(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel,shocks:List\[float\]=\[0.1\],transformation:Union\[str, None\]=None):
+
+:::
+
+
+
+Assesses the sensitivity of a regression model to changes in independent variables by applying shocks and visualizing the impact.
+
+### Purpose
+
+The Regression Sensitivity Plot test is designed to perform sensitivity analysis on regression models. This test aims to measure the impact of slight changes (shocks) applied to individual variables on the system's outcome while keeping all other variables constant. By doing so, it analyzes the effects of each independent variable on the dependent variable within the regression model, helping identify significant risk factors that could substantially influence the model's output.
+
+### Test Mechanism
+
+This test operates by initially applying shocks of varying magnitudes, defined by specific parameters, to each of the model's features, one at a time. With all other variables held constant, a new prediction is made for each dataset subjected to shocks. Any changes in the model's predictions are directly attributed to the shocks applied. If the transformation parameter is set to "integrate," initial predictions and target values undergo transformation via an integration function before being plotted. Finally, a plot demonstrating observed values against predicted values for each model is generated, showcasing a distinct line graph illustrating predictions for each shock.
+
+### Signs of High Risk
+
+- Drastic alterations in model predictions due to minor shocks to an individual variable, indicating high sensitivity and potential over-dependence on that variable.
+- Unusually high or unpredictable shifts in response to shocks, suggesting potential model instability or overfitting.
+
+### Strengths
+
+- Helps identify variables that strongly influence model outcomes, aiding in understanding feature importance.
+- Generates visual plots, making results easily interpretable even to non-technical stakeholders.
+- Useful in identifying overfitting and detecting unstable models that react excessively to minor variable changes.
+
+### Limitations
+
+- Operates on the assumption that all other variables remain unchanged during the application of a shock, which may not reflect real-world interdependencies.
+- Best compatible with linear models and may not effectively evaluate the sensitivity of non-linear models.
+- Provides a visual representation without a numerical risk measure, potentially introducing subjectivity in interpretation.
diff --git a/docs/validmind/tests/model_validation/statsmodels/RegressionModelSummary.qmd b/docs/validmind/tests/model_validation/statsmodels/RegressionModelSummary.qmd
new file mode 100644
index 000000000..0d6ffd512
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/RegressionModelSummary.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionModelSummary"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionModelSummary
+
+
+
+::: {.signature}
+
+@tags('model_performance', 'regression')
+
+@tasks('regression')
+
+defRegressionModelSummary(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel):
+
+:::
+
+
+
+Evaluates regression model performance using metrics including R-Squared, Adjusted R-Squared, MSE, and RMSE.
+
+### Purpose
+
+The Regression Model Summary test evaluates the performance of regression models by measuring their predictive ability regarding dependent variables given changes in the independent variables. It uses conventional regression metrics such as R-Squared, Adjusted R-Squared, Mean Squared Error (MSE), and Root Mean Squared Error (RMSE) to assess the model's accuracy and fit.
+
+### Test Mechanism
+
+This test uses the sklearn library to calculate the R-Squared, Adjusted R-Squared, MSE, and RMSE. It outputs a table with the results of these metrics along with the feature columns used by the model.
+
+### Signs of High Risk
+
+- Low R-Squared and Adjusted R-Squared values.
+- High MSE and RMSE values.
+
+### Strengths
+
+- Offers an extensive evaluation of regression models by combining four key measures of model accuracy and fit.
+- Provides a comprehensive view of the model's performance.
+- Both the R-Squared and Adjusted R-Squared measures are readily interpretable.
+
+### Limitations
+
+- RMSE and MSE might be sensitive to outliers.
+- A high R-Squared or Adjusted R-Squared may not necessarily indicate a good model, especially in cases of overfitting.
diff --git a/docs/validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.qmd b/docs/validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.qmd
new file mode 100644
index 000000000..c541a9efe
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.qmd
@@ -0,0 +1,51 @@
+---
+title: "[validmind](/validmind/validmind.qmd).RegressionPermutationFeatureImportance"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## RegressionPermutationFeatureImportance
+
+
+
+::: {.signature}
+
+@tags('statsmodels', 'feature_importance', 'visualization')
+
+@tasks('regression')
+
+defRegressionPermutationFeatureImportance(dataset:validmind.vm_models.VMDataset,model:validmind.vm_models.VMModel,fontsize:int=12,figure_height:int=500):
+
+:::
+
+
+
+Assesses the significance of each feature in a model by evaluating the impact on model performance when feature values are randomly rearranged.
+
+### Purpose
+
+The primary purpose of this metric is to determine which features significantly impact the performance of a regression model developed using statsmodels. The metric measures how much the prediction accuracy deteriorates when each feature's values are permuted.
+
+### Test Mechanism
+
+This metric shuffles the values of each feature one at a time in the dataset, computes the model's performance after each permutation, and compares it to the baseline performance. A significant decrease in performance indicates the importance of the feature.
+
+### Signs of High Risk
+
+- Significant reliance on a feature that, when permuted, leads to a substantial decrease in performance, suggesting overfitting or high model dependency on that feature.
+- Features identified as unimportant despite known impacts from domain knowledge, suggesting potential issues in model training or data preprocessing.
+
+### Strengths
+
+- Directly assesses the impact of each feature on model performance, providing clear insights into model dependencies.
+- Model-agnostic within the scope of statsmodels, applicable to any regression model that outputs predictions.
+
+### Limitations
+
+- The metric is specific to statsmodels and cannot be used with other types of models without adaptation.
+- It does not capture interactions between features, which can lead to underestimating the importance of correlated features.
+- Assumes independence of features when calculating importance, which might not always hold true.
diff --git a/docs/validmind/tests/model_validation/statsmodels/ScorecardHistogram.qmd b/docs/validmind/tests/model_validation/statsmodels/ScorecardHistogram.qmd
new file mode 100644
index 000000000..e0b59fe45
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/ScorecardHistogram.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ScorecardHistogram"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## ScorecardHistogram
+
+
+
+::: {.signature}
+
+@tags('visualization', 'credit_risk', 'logistic_regression')
+
+@tasks('classification')
+
+defScorecardHistogram(dataset,title='Histogram of Scores',score_column='score'):
+
+:::
+
+
+
+The Scorecard Histogram test evaluates the distribution of credit scores between default and non-default instances, providing critical insights into the performance and generalizability of credit-risk models.
+
+### Purpose
+
+The Scorecard Histogram test metric provides a visual interpretation of the credit scores generated by a machine learning model for credit-risk classification tasks. It aims to compare the alignment of the model's scoring decisions with the actual outcomes of credit loan applications. It helps in identifying potential discrepancies between the model's predictions and real-world risk levels.
+
+### Test Mechanism
+
+This metric uses logistic regression to generate a histogram of credit scores for both default (negative class) and non-default (positive class) instances. Using both training and test datasets, the metric calculates the credit score of each instance with a scorecard method, considering the impact of different features on the likelihood of default. It includes the default point to odds (PDO) scaling factor and predefined target score and odds settings. Histograms for training and test sets are computed and plotted separately to offer insights into the model's generalizability to unseen data.
+
+### Signs of High Risk
+
+- Discrepancies between the distributions of training and testing data, indicating a model's poor generalization ability
+- Skewed distributions favoring specific scores or classes, representing potential bias
+
+### Strengths
+
+- Provides a visual interpretation of the model's credit scoring system, enhancing comprehension of model behavior
+- Enables a direct comparison between actual and predicted scores for both training and testing data
+- Its intuitive visualization helps understand the model's ability to differentiate between positive and negative classes
+- Can unveil patterns or anomalies not easily discerned through numerical metrics alone
+
+### Limitations
+
+- Despite its value for visual interpretation, it doesn't quantify the performance of the model and therefore may lack precision for thorough model evaluation
+- The quality of input data can strongly influence the metric, as bias or noise in the data will affect both the score calculation and resultant histogram
+- Its specificity to credit scoring models limits its applicability across a wider variety of machine learning tasks and models
+- The metric's effectiveness is somewhat tied to the subjective interpretation of the analyst, relying on their judgment of the characteristics and implications of the plot.
diff --git a/docs/validmind/tests/model_validation/statsmodels/statsutils.qmd b/docs/validmind/tests/model_validation/statsmodels/statsutils.qmd
new file mode 100644
index 000000000..723f5b37f
--- /dev/null
+++ b/docs/validmind/tests/model_validation/statsmodels/statsutils.qmd
@@ -0,0 +1,23 @@
+---
+title: "[validmind](/validmind/validmind.qmd).statsutils"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## adj_r2_score
+
+
+
+::: {.signature}
+
+defadj_r2_score(actual:np.ndarray,predicted:np.ndarray,rowcount:int,featurecount:int):
+
+:::
+
+
+
+Adjusted R2 Score
diff --git a/docs/validmind/tests/prompt_validation.qmd b/docs/validmind/tests/prompt_validation.qmd
new file mode 100644
index 000000000..5797eb873
--- /dev/null
+++ b/docs/validmind/tests/prompt_validation.qmd
@@ -0,0 +1,16 @@
+---
+title: "[validmind](/validmind/validmind.qmd).prompt_validation"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+- [ai_powered_test](prompt_validation/ai_powered_test.qmd)
+- [Bias](prompt_validation/Bias.qmd)
+- [Clarity](prompt_validation/Clarity.qmd)
+- [Conciseness](prompt_validation/Conciseness.qmd)
+- [Delimitation](prompt_validation/Delimitation.qmd)
+- [NegativeInstruction](prompt_validation/NegativeInstruction.qmd)
+- [Robustness](prompt_validation/Robustness.qmd)
+- [Specificity](prompt_validation/Specificity.qmd)
diff --git a/docs/validmind/tests/prompt_validation/Bias.qmd b/docs/validmind/tests/prompt_validation/Bias.qmd
new file mode 100644
index 000000000..0d4e4370b
--- /dev/null
+++ b/docs/validmind/tests/prompt_validation/Bias.qmd
@@ -0,0 +1,57 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Bias"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Bias
+
+
+
+::: {.signature}
+
+@tags('llm', 'few_shot')
+
+@tasks('text_classification', 'text_summarization')
+
+defBias(model,min_threshold=7):
+
+:::
+
+
+
+Assesses potential bias in a Large Language Model by analyzing the distribution and order of exemplars in the prompt.
+
+### Purpose
+
+The Bias Evaluation test calculates if and how the order and distribution of exemplars (examples) in a few-shot learning prompt affect the output of a Large Language Model (LLM). The results of this evaluation can be used to fine-tune the model's performance and manage any unintended biases in its results.
+
+### Test Mechanism
+
+This test uses two checks:
+
+1. **Distribution of Exemplars:** The number of positive vs. negative examples in a prompt is varied. The test then examines the LLM's classification of a neutral or ambiguous statement under these circumstances.
+1. **Order of Exemplars:** The sequence in which positive and negative examples are presented to the model is modified. Their resultant effect on the LLM's response is studied.
+
+For each test case, the LLM grades the input prompt on a scale of 1 to 10. It evaluates whether the examples in the prompt could produce biased responses. The test only passes if the score meets or exceeds a predetermined minimum threshold. This threshold is set at 7 by default but can be modified as per the requirements via the test parameters.
+
+### Signs of High Risk
+
+- A skewed result favoring either positive or negative responses may suggest potential bias in the model. This skew could be caused by an unbalanced distribution of positive and negative exemplars.
+- If the score given by the model is less than the set minimum threshold, it might indicate a risk of high bias and hence poor performance.
+
+### Strengths
+
+- This test provides a quantitative measure of potential bias, offering clear guidelines for developers about whether their Large Language Model (LLM) contains significant bias.
+- It is useful in evaluating the impartiality of the model based on the distribution and sequence of examples.
+- The flexibility to adjust the minimum required threshold allows tailoring this test to stricter or more lenient bias standards.
+
+### Limitations
+
+- The test may not pick up on more subtle forms of bias or biases that are not directly related to the distribution or order of exemplars.
+- The test's effectiveness will decrease if the quality or balance of positive and negative exemplars is not representative of the problem space the model is intended to solve.
+- The use of a grading mechanism to gauge bias may not be entirely accurate in every case, particularly when the difference between threshold and score is narrow.
diff --git a/docs/validmind/tests/prompt_validation/Clarity.qmd b/docs/validmind/tests/prompt_validation/Clarity.qmd
new file mode 100644
index 000000000..f864b8427
--- /dev/null
+++ b/docs/validmind/tests/prompt_validation/Clarity.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Clarity"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Clarity
+
+
+
+::: {.signature}
+
+@tags('llm', 'zero_shot', 'few_shot')
+
+@tasks('text_classification', 'text_summarization')
+
+defClarity(model,min_threshold=7):
+
+:::
+
+
+
+Evaluates and scores the clarity of prompts in a Large Language Model based on specified guidelines.
+
+### Purpose
+
+The Clarity evaluation metric is used to assess how clear the prompts of a Large Language Model (LLM) are. This assessment is particularly important because clear prompts assist the LLM in more accurately interpreting and responding to instructions.
+
+### Test Mechanism
+
+The evaluation uses an LLM to scrutinize the clarity of prompts, factoring in considerations such as the inclusion of relevant details, persona adoption, step-by-step instructions, usage of examples, and specification of desired output length. Each prompt is rated on a clarity scale of 1 to 10, and any prompt scoring at or above the preset threshold (default of 7) will be marked as clear. It is important to note that this threshold can be adjusted via test parameters, providing flexibility in the evaluation process.
+
+### Signs of High Risk
+
+- Prompts that consistently score below the clarity threshold
+- Repeated failure of prompts to adhere to guidelines for clarity, including detail inclusion, persona adoption, explicit step-by-step instructions, use of examples, and specification of output length
+
+### Strengths
+
+- Encourages the development of more effective prompts that aid the LLM in interpreting instructions accurately
+- Applies a quantifiable measure (a score from 1 to 10) to evaluate the clarity of prompts
+- Threshold for clarity is adjustable, allowing for flexible evaluation depending on the context
+
+### Limitations
+
+- Scoring system is subjective and relies on the AI’s interpretation of 'clarity'
+- The test assumes that all required factors (detail inclusion, persona adoption, step-by-step instructions, use of examples, and specification of output length) contribute equally to clarity, which might not always be the case
+- The evaluation may not be as effective if used on non-textual models
diff --git a/docs/validmind/tests/prompt_validation/Conciseness.qmd b/docs/validmind/tests/prompt_validation/Conciseness.qmd
new file mode 100644
index 000000000..446ae9fb0
--- /dev/null
+++ b/docs/validmind/tests/prompt_validation/Conciseness.qmd
@@ -0,0 +1,54 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Conciseness"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Conciseness
+
+
+
+::: {.signature}
+
+@tags('llm', 'zero_shot', 'few_shot')
+
+@tasks('text_classification', 'text_summarization')
+
+defConciseness(model,min_threshold=7):
+
+:::
+
+
+
+Analyzes and grades the conciseness of prompts provided to a Large Language Model.
+
+### Purpose
+
+The Conciseness Assessment is designed to evaluate the brevity and succinctness of prompts provided to a Language Learning Model (LLM). A concise prompt strikes a balance between offering clear instructions and eliminating redundant or unnecessary information, ensuring that the LLM receives relevant input without being overwhelmed.
+
+### Test Mechanism
+
+Using an LLM, this test conducts a conciseness analysis on input prompts. The analysis grades the prompt on a scale from 1 to 10, where the grade reflects how well the prompt delivers clear instructions without being verbose. Prompts that score equal to or above a predefined threshold (default set to 7) are deemed successfully concise. This threshold can be adjusted to meet specific requirements.
+
+### Signs of High Risk
+
+- Prompts that consistently score below the predefined threshold.
+- Prompts that are overly wordy or contain unnecessary information.
+- Prompts that create confusion or ambiguity due to excess or unnecessary information.
+
+### Strengths
+
+- Ensures clarity and effectiveness of the prompts.
+- Promotes brevity and preciseness in prompts without sacrificing essential information.
+- Useful for models like LLMs, where input prompt length and clarity greatly influence model performance.
+- Provides a quantifiable measure of prompt conciseness.
+
+### Limitations
+
+- The conciseness score is based on an AI's assessment, which might not fully capture human interpretation of conciseness.
+- The predefined threshold for conciseness could be subjective and might need adjustment based on application.
+- The test is dependent on the LLM’s understanding of conciseness, which might vary from model to model.
diff --git a/docs/validmind/tests/prompt_validation/Delimitation.qmd b/docs/validmind/tests/prompt_validation/Delimitation.qmd
new file mode 100644
index 000000000..9177caa52
--- /dev/null
+++ b/docs/validmind/tests/prompt_validation/Delimitation.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Delimitation"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Delimitation
+
+
+
+::: {.signature}
+
+@tags('llm', 'zero_shot', 'few_shot')
+
+@tasks('text_classification', 'text_summarization')
+
+defDelimitation(model,min_threshold=7):
+
+:::
+
+
+
+Evaluates the proper use of delimiters in prompts provided to Large Language Models.
+
+### Purpose
+
+The Delimitation Test aims to assess whether prompts provided to the Language Learning Model (LLM) correctly use delimiters to mark different sections of the input. Well-delimited prompts help simplify the interpretation process for the LLM, ensuring that the responses are precise and accurate.
+
+### Test Mechanism
+
+The test employs an LLM to examine prompts for appropriate use of delimiters such as triple quotation marks, XML tags, and section titles. Each prompt is assigned a score from 1 to 10 based on its delimitation integrity. Prompts with scores equal to or above the preset threshold (which is 7 by default, although it can be adjusted as necessary) pass the test.
+
+### Signs of High Risk
+
+- Prompts missing, improperly placed, or incorrectly used delimiters, leading to misinterpretation by the LLM.
+- High-risk scenarios with complex prompts involving multiple tasks or diverse data where correct delimitation is crucial.
+- Scores below the threshold, indicating a high risk.
+
+### Strengths
+
+- Ensures clarity in demarcating different components of given prompts.
+- Reduces ambiguity in understanding prompts, especially for complex tasks.
+- Provides a quantified insight into the appropriateness of delimiter usage, aiding continuous improvement.
+
+### Limitations
+
+- Only checks for the presence and placement of delimiters, not whether the correct delimiter type is used for the specific data or task.
+- May not fully reveal the impacts of poor delimitation on the LLM's final performance.
+- The preset score threshold may not be refined enough for complex tasks and prompts, requiring regular manual adjustment.
diff --git a/docs/validmind/tests/prompt_validation/NegativeInstruction.qmd b/docs/validmind/tests/prompt_validation/NegativeInstruction.qmd
new file mode 100644
index 000000000..847e34997
--- /dev/null
+++ b/docs/validmind/tests/prompt_validation/NegativeInstruction.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).NegativeInstruction"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## NegativeInstruction
+
+
+
+::: {.signature}
+
+@tags('llm', 'zero_shot', 'few_shot')
+
+@tasks('text_classification', 'text_summarization')
+
+defNegativeInstruction(model,min_threshold=7):
+
+:::
+
+
+
+Evaluates and grades the use of affirmative, proactive language over negative instructions in LLM prompts.
+
+### Purpose
+
+The Negative Instruction test is utilized to scrutinize the prompts given to a Large Language Model (LLM). The objective is to ensure these prompts are expressed using proactive, affirmative language. The focus is on instructions indicating what needs to be done rather than what needs to be avoided, thereby guiding the LLM more efficiently towards the desired output.
+
+### Test Mechanism
+
+An LLM is employed to evaluate each prompt. The prompt is graded based on its use of positive instructions with scores ranging between 1-10. This grade reflects how effectively the prompt leverages affirmative language while shying away from negative or restrictive instructions. A prompt that attains a grade equal to or above a predetermined threshold (7 by default) is regarded as adhering effectively to the best practices of positive instruction. This threshold can be custom-tailored through the test parameters.
+
+### Signs of High Risk
+
+- Low score obtained from the LLM analysis, indicating heavy reliance on negative instructions in the prompts.
+- Failure to surpass the preset minimum threshold.
+- The LLM generates ambiguous or undesirable outputs as a consequence of the negative instructions used in the prompt.
+
+### Strengths
+
+- Encourages the usage of affirmative, proactive language in prompts, aiding in more accurate and advantageous model responses.
+- The test result provides a comprehensible score, helping to understand how well a prompt follows the positive instruction best practices.
+
+### Limitations
+
+- Despite an adequate score, a prompt could still be misleading or could lead to undesired responses due to factors not covered by this test.
+- The test necessitates an LLM for evaluation, which might not be available or feasible in certain scenarios.
+- A numeric scoring system, while straightforward, may oversimplify complex issues related to prompt designing and instruction clarity.
+- The effectiveness of the test hinges significantly on the predetermined threshold level, which can be subjective and may need to be adjusted according to specific use-cases.
diff --git a/docs/validmind/tests/prompt_validation/Robustness.qmd b/docs/validmind/tests/prompt_validation/Robustness.qmd
new file mode 100644
index 000000000..f91730640
--- /dev/null
+++ b/docs/validmind/tests/prompt_validation/Robustness.qmd
@@ -0,0 +1,52 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Robustness"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Robustness
+
+
+
+::: {.signature}
+
+@tags('llm', 'zero_shot', 'few_shot')
+
+@tasks('text_classification', 'text_summarization')
+
+defRobustness(model,dataset,num_tests=10):
+
+:::
+
+
+
+Assesses the robustness of prompts provided to a Large Language Model under varying conditions and contexts. This test specifically measures the model's ability to generate correct classifications with the given prompt even when the inputs are edge cases or otherwise difficult to classify.
+
+### Purpose
+
+The Robustness test is meant to evaluate the resilience and reliability of prompts provided to a Language Learning Model (LLM). The aim of this test is to guarantee that the prompts consistently generate accurate and expected outputs, even in diverse or challenging scenarios. This test is only applicable to LLM-powered text classification tasks where the prompt has a single input variable.
+
+### Test Mechanism
+
+The Robustness test appraises prompts under various conditions, alterations, and contexts to ascertain their stability in producing consistent responses from the LLM. Factors evaluated include different phrasings, inclusion of potential distracting elements, and various input complexities. By default, the test generates 10 inputs for a prompt but can be adjusted according to test parameters.
+
+### Signs of High Risk
+
+- If the output from the tests diverges extensively from the expected results, this indicates high risk.
+- When the prompt doesn't give a consistent performance across various tests.
+- A high risk is indicated when the prompt is susceptible to breaking, especially when the output is expected to be of a specific type.
+
+### Strengths
+
+- The robustness test helps to ensure stable performance of the LLM prompts and lowers the chances of generating unexpected or off-target outputs.
+- This test is vital for applications where predictability and reliability of the LLM’s output are crucial.
+
+### Limitations
+
+- Currently, the test only supports single-variable prompts, which restricts its application to more complex models.
+- When there are too many target classes (over 10), the test is skipped, which can leave potential vulnerabilities unchecked in complex multi-class models.
+- The test may not account for all potential conditions or alterations that could show up in practical use scenarios.
diff --git a/docs/validmind/tests/prompt_validation/Specificity.qmd b/docs/validmind/tests/prompt_validation/Specificity.qmd
new file mode 100644
index 000000000..efb240235
--- /dev/null
+++ b/docs/validmind/tests/prompt_validation/Specificity.qmd
@@ -0,0 +1,53 @@
+---
+title: "[validmind](/validmind/validmind.qmd).Specificity"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## Specificity
+
+
+
+::: {.signature}
+
+@tags('llm', 'zero_shot', 'few_shot')
+
+@tasks('text_classification', 'text_summarization')
+
+defSpecificity(model,min_threshold=7):
+
+:::
+
+
+
+Evaluates and scores the specificity of prompts provided to a Large Language Model (LLM), based on clarity, detail, and relevance.
+
+### Purpose
+
+The Specificity Test evaluates the clarity, precision, and effectiveness of the prompts provided to a Language Model (LLM). It aims to ensure that the instructions embedded in a prompt are indisputably clear and relevant, thereby helping to remove ambiguity and steer the LLM towards desired outputs. This level of specificity significantly affects the accuracy and relevance of LLM outputs.
+
+### Test Mechanism
+
+The Specificity Test employs an LLM to grade each prompt based on clarity, detail, and relevance parameters within a specificity scale that extends from 1 to 10. On this scale, prompts scoring equal to or more than a predefined threshold (set to 7 by default) pass the evaluation, while those scoring below this threshold fail it. Users can adjust this threshold as per their requirements.
+
+### Signs of High Risk
+
+- Prompts scoring consistently below the established threshold
+- Vague or ambiguous prompts that do not provide clear direction to the LLM
+- Overly verbose prompts that may confuse the LLM instead of providing clear guidance
+
+### Strengths
+
+- Enables precise and clear communication with the LLM to achieve desired outputs
+- Serves as a crucial means to measure the effectiveness of prompts
+- Highly customizable, allowing users to set their threshold based on specific use cases
+
+### Limitations
+
+- This test doesn't consider the content comprehension capability of the LLM
+- High specificity score doesn't guarantee a high-quality response from the LLM, as the model's performance is also dependent on various other factors
+- Striking a balance between specificity and verbosity can be challenging, as overly detailed prompts might confuse or mislead the model
diff --git a/docs/validmind/tests/prompt_validation/ai_powered_test.qmd b/docs/validmind/tests/prompt_validation/ai_powered_test.qmd
new file mode 100644
index 000000000..e16ee2e89
--- /dev/null
+++ b/docs/validmind/tests/prompt_validation/ai_powered_test.qmd
@@ -0,0 +1,59 @@
+---
+title: "[validmind](/validmind/validmind.qmd).ai_powered_test"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## call_model
+
+
+
+::: {.signature}
+
+defcall_model(system_prompt:str,user_prompt:str,temperature:float=0.0,seed:int=42):
+
+:::
+
+
+
+Call LLM with the given prompts and return the response
+
+
+
+## get_explanation
+
+
+
+::: {.signature}
+
+defget_explanation(response:str):
+
+:::
+
+
+
+Get just the explanation from the response string TODO: use json response mode instead of this
+
+e.g. "Score: 8 Explanation: " -> ""
+
+
+
+## get_score
+
+
+
+::: {.signature}
+
+defget_score(response:str):
+
+:::
+
+
+
+Get just the score from the response string TODO: use json response mode instead of this
+
+e.g. "Score: 8 Explanation: " -> 8
diff --git a/docs/validmind/unit_metrics.qmd b/docs/validmind/unit_metrics.qmd
new file mode 100644
index 000000000..6fd0dddad
--- /dev/null
+++ b/docs/validmind/unit_metrics.qmd
@@ -0,0 +1,55 @@
+---
+title: "[validmind](/validmind/validmind.qmd).unit_metrics"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+## list_metrics
+
+
+
+::: {.signature}
+
+deflist_metrics(\*\*kwargs):
+
+:::
+
+
+
+List all metrics
+
+
+
+## describe_metric
+
+
+
+::: {.signature}
+
+defdescribe_metric(metric_id:str,\*\*kwargs):
+
+:::
+
+
+
+Describe a metric
+
+
+
+## run_metric
+
+
+
+::: {.signature}
+
+defrun_metric(metric_id:str,\*\*kwargs):
+
+:::
+
+
+
+Run a metric
diff --git a/docs/validmind/version.qmd b/docs/validmind/version.qmd
new file mode 100644
index 000000000..7264b4709
--- /dev/null
+++ b/docs/validmind/version.qmd
@@ -0,0 +1,14 @@
+---
+title: "[validmind](/validmind/validmind.qmd).__version__"
+sidebar: validmind-reference
+---
+
+
+
+
+
+::: {.signature}
+
+2.8.20
+
+:::
diff --git a/docs/validmind/vm_models.qmd b/docs/validmind/vm_models.qmd
new file mode 100644
index 000000000..8633c5f48
--- /dev/null
+++ b/docs/validmind/vm_models.qmd
@@ -0,0 +1,986 @@
+---
+title: "[validmind](/validmind/validmind.qmd).vm_models"
+sidebar: validmind-reference
+toc-depth: 4
+toc-expand: 4
+# module.qmd.jinja2
+---
+
+
+
+Models entrypoint
+
+## R_MODEL_TYPES
+
+
+
+::: {.signature}
+
+R_MODEL_TYPES=\['LogisticRegression', 'LinearRegression', 'XGBClassifier', 'XGBRegressor'\]:
+
+:::
+
+
+
+## VMInput
+
+
+
+::: {.signature}
+
+classVMInput(ABC):
+
+:::
+
+
+
+Base class for ValidMind Input types.
+
+### with_options
+
+
+
+::: {.signature}
+
+defwith_options(self,\*\*kwargs:Dict\[str, Any\]) → validmind.vm_models.VMInput:
+
+:::
+
+
+
+Allows for setting options on the input object that are passed by the user when using the input to run a test or set of tests.
+
+To allow options, just override this method in the subclass (see VMDataset) and ensure that it returns a new instance of the input with the specified options set.
+
+**Arguments**
+
+- `**kwargs`: Arbitrary keyword arguments that will be passed to the input object.
+
+**Returns**
+
+- A new instance of the input with the specified options set.
+
+
+
+## VMDataset
+
+
+
+::: {.signature}
+
+classVMDataset(VMInput):
+
+:::
+
+
+
+Base class for VM datasets.
+
+Child classes should be used to support new dataset types (tensor, polars etc.) by converting the user's dataset into a numpy array collecting metadata like column names and then call this (parent) class `__init__` method.
+
+This way we can support multiple dataset types but under the hood we only need to work with numpy arrays and pandas dataframes in this class.
+
+**Arguments**
+
+- `raw_dataset (np.ndarray)`: The raw dataset as a NumPy array.
+- `input_id (str)`: Identifier for the dataset.
+- `index (np.ndarray)`: The raw dataset index as a NumPy array.
+- `columns (Set[str])`: The column names of the dataset.
+- `target_column (str)`: The target column name of the dataset.
+- `feature_columns (List[str])`: The feature column names of the dataset.
+- `feature_columns_numeric (List[str])`: The numeric feature column names of the dataset.
+- `feature_columns_categorical (List[str])`: The categorical feature column names of the dataset.
+- `text_column (str)`: The text column name of the dataset for NLP tasks.
+- `target_class_labels (Dict)`: The class labels for the target columns.
+- `df (pd.DataFrame)`: The dataset as a pandas DataFrame.
+- `extra_columns (Dict)`: Extra columns to include in the dataset.
+
+### VMDataset
+
+
+
+::: {.signature}
+
+VMDataset(raw_dataset:np.ndarray,input_id:str=None,model:validmind.vm_models.VMModel=None,index:np.ndarray=None,index_name:str=None,date_time_index:bool=False,columns:list=None,target_column:str=None,feature_columns:list=None,text_column:str=None,extra_columns:dict=None,target_class_labels:dict=None)
+
+:::
+
+
+
+Initializes a VMDataset instance.
+
+**Arguments**
+
+- `raw_dataset (np.ndarray)`: The raw dataset as a NumPy array.
+- `input_id (str)`: Identifier for the dataset.
+- `model (VMModel)`: Model associated with the dataset.
+- `index (np.ndarray)`: The raw dataset index as a NumPy array.
+- `index_name (str)`: The raw dataset index name as a NumPy array.
+- `date_time_index (bool)`: Whether the index is a datetime index.
+- `columns (List[str], optional)`: The column names of the dataset. Defaults to None.
+- `target_column (str, optional)`: The target column name of the dataset. Defaults to None.
+- `feature_columns (str, optional)`: The feature column names of the dataset. Defaults to None.
+- `text_column (str, optional)`: The text column name of the dataset for nlp tasks. Defaults to None.
+- `target_class_labels (Dict, optional)`: The class labels for the target columns. Defaults to None.
+
+### add_extra_column
+
+
+
+::: {.signature}
+
+defadd_extra_column(self,column_name,column_values=None):
+
+:::
+
+
+
+Adds an extra column to the dataset without modifying the dataset `features` and `target` columns.
+
+**Arguments**
+
+- `column_name (str)`: The name of the extra column.
+- `column_values (np.ndarray)`: The values of the extra column.
+
+### assign_predictions
+
+
+
+::: {.signature}
+
+defassign_predictions(self,model:validmind.vm_models.VMModel,prediction_column:Optional\[str\]=None,prediction_values:Optional\[List\[Any\]\]=None,probability_column:Optional\[str\]=None,probability_values:Optional\[List\[float\]\]=None,prediction_probabilities:Optional\[List\[float\]\]=None,\*\*kwargs:Dict\[str, Any\]):
+
+:::
+
+
+
+Assign predictions and probabilities to the dataset.
+
+**Arguments**
+
+- `model (VMModel)`: The model used to generate the predictions.
+- `prediction_column (Optional[str])`: The name of the column containing the predictions.
+- `prediction_values (Optional[List[Any]])`: The values of the predictions.
+- `probability_column (Optional[str])`: The name of the column containing the probabilities.
+- `probability_values (Optional[List[float]])`: The values of the probabilities.
+- `prediction_probabilities (Optional[List[float]])`: DEPRECATED: The values of the probabilities.
+- `**kwargs`: Additional keyword arguments that will get passed through to the model's `predict` method.
+
+### prediction_column
+
+
+
+::: {.signature}
+
+defprediction_column(self,model:validmind.vm_models.VMModel,column_name:str=None) → str:
+
+:::
+
+
+
+Get or set the prediction column for a model.
+
+### probability_column
+
+
+
+::: {.signature}
+
+defprobability_column(self,model:validmind.vm_models.VMModel,column_name:str=None) → str:
+
+:::
+
+
+
+Get or set the probability column for a model.
+
+### target_classes
+
+
+
+::: {.signature}
+
+deftarget_classes(self):
+
+:::
+
+
+
+Returns the target class labels or unique values of the target column.
+
+### with_options
+
+
+
+::: {.signature}
+
+defwith_options(self,\*\*kwargs:Dict\[str, Any\]) → validmind.vm_models.VMDataset:
+
+:::
+
+
+
+Support options provided when passing an input to run_test or run_test_suite
+
+**Arguments**
+
+- `**kwargs`: Options:
+- columns: Filter columns in the dataset
+
+**Returns**
+
+- A new instance of the dataset with only the specified columns
+
+### x_df
+
+
+
+::: {.signature}
+
+defx_df(self):
+
+:::
+
+
+
+Returns a dataframe containing only the feature columns
+
+### y_df
+
+
+
+::: {.signature}
+
+defy_df(self) → pd.DataFrame:
+
+:::
+
+
+
+Returns a dataframe containing the target column
+
+### y_pred
+
+
+
+::: {.signature}
+
+defy_pred(self,model) → np.ndarray:
+
+:::
+
+
+
+Returns the predictions for a given model.
+
+Attempts to stack complex prediction types (e.g., embeddings) into a single, multi-dimensional array.
+
+**Arguments**
+
+- `model (VMModel)`: The model whose predictions are sought.
+
+**Returns**
+
+- The predictions for the model
+
+### y_pred_df
+
+
+
+::: {.signature}
+
+defy_pred_df(self,model) → pd.DataFrame:
+
+:::
+
+
+
+Returns a dataframe containing the predictions for a given model
+
+### y_prob
+
+
+
+::: {.signature}
+
+defy_prob(self,model) → np.ndarray:
+
+:::
+
+
+
+Returns the probabilities for a given model.
+
+**Arguments**
+
+- `model (str)`: The ID of the model whose predictions are sought.
+
+**Returns**
+
+- The probability variables.
+
+### y_prob_df
+
+
+
+::: {.signature}
+
+defy_prob_df(self,model) → pd.DataFrame:
+
+:::
+
+
+
+Returns a dataframe containing the probabilities for a given model
+
+### df{.property}
+
+
+
+::: {.signature}
+
+df():
+
+:::
+
+
+
+Returns the dataset as a pandas DataFrame.
+
+**Returns**
+
+- The dataset as a pandas DataFrame.
+
+### x{.property}
+
+
+
+::: {.signature}
+
+x():
+
+:::
+
+
+
+Returns the input features (X) of the dataset.
+
+**Returns**
+
+- The input features.
+
+### y{.property}
+
+
+
+::: {.signature}
+
+y():
+
+:::
+
+
+
+Returns the target variables (y) of the dataset.
+
+**Returns**
+
+- The target variables.
+
+
+
+## VMModel
+
+
+
+::: {.signature}
+
+classVMModel(VMInput):
+
+:::
+
+
+
+An base class that wraps a trained model instance and its associated data.
+
+**Arguments**
+
+- `model (object, optional)`: The trained model instance. Defaults to None.
+- `input_id (str, optional)`: The input ID for the model. Defaults to None.
+- `attributes (ModelAttributes, optional)`: The attributes of the model. Defaults to None.
+- `name (str, optional)`: The name of the model. Defaults to the class name.
+
+### VMModel
+
+
+
+::: {.signature}
+
+VMModel(input_id:str=None,model:object=None,attributes:validmind.vm_models.ModelAttributes=None,name:str=None,\*\*kwargs)
+
+:::
+
+### predict
+
+
+
+::: {.signature}
+
+@abstractmethod
+
+defpredict(self,\*args,\*\*kwargs):
+
+:::
+
+
+
+Predict method for the model. This is a wrapper around the model's
+
+### predict_proba
+
+
+
+::: {.signature}
+
+defpredict_proba(self,\*args,\*\*kwargs):
+
+:::
+
+
+
+Predict probabilties - must be implemented by subclass if needed
+
+### serialize
+
+
+
+::: {.signature}
+
+defserialize(self):
+
+:::
+
+
+
+Serializes the model to a dictionary so it can be sent to the API
+
+
+
+## Figure
+
+
+
+::: {.signature}
+
+@dataclass
+
+classFigure:
+
+:::
+
+
+
+Figure objects track the schema supported by the ValidMind API.
+
+### Figure
+
+
+
+::: {.signature}
+
+Figure(key:str,figure:Union\[matplotlib.validmind.vm_models.figure.Figure, go.Figure, go.validmind.vm_models.FigureWidget, bytes\],ref_id:str,\_type:str='plot')
+
+:::
+
+### serialize
+
+
+
+::: {.signature}
+
+defserialize(self):
+
+:::
+
+
+
+Serializes the Figure to a dictionary so it can be sent to the API.
+
+### serialize_files
+
+
+
+::: {.signature}
+
+defserialize_files(self):
+
+:::
+
+
+
+Creates a `requests`-compatible files object to be sent to the API.
+
+### to_widget
+
+
+
+::: {.signature}
+
+defto_widget(self):
+
+:::
+
+
+
+Returns the ipywidget compatible representation of the figure. Ideally we would render images as-is, but Plotly FigureWidgets don't work well on Google Colab when they are combined with ipywidgets.
+
+
+
+## ModelAttributes
+
+
+
+::: {.signature}
+
+@dataclass
+
+classModelAttributes:
+
+:::
+
+
+
+Model attributes definition.
+
+### ModelAttributes
+
+
+
+::: {.signature}
+
+ModelAttributes(architecture:str=None,framework:str=None,framework_version:str=None,language:str=None,task:validmind.vm_models.ModelTask=None)
+
+:::
+
+### from_dict
+
+
+
+::: {.signature}
+
+@classmethod
+
+deffrom_dict(cls,data):
+
+:::
+
+
+
+Creates a ModelAttributes instance from a dictionary.
+
+
+
+## ResultTable
+
+
+
+::: {.signature}
+
+@dataclass
+
+classResultTable:
+
+:::
+
+
+
+A dataclass that holds the table summary of result.
+
+### ResultTable
+
+
+
+::: {.signature}
+
+ResultTable(data:Union\[List\[Any\], pd.DataFrame\],title:Optional\[str\]=None)
+
+:::
+
+### serialize
+
+
+
+::: {.signature}
+
+defserialize(self):
+
+:::
+
+
+
+## TestResult
+
+
+
+::: {.signature}
+
+@dataclass
+
+classTestResult(Result):
+
+:::
+
+
+
+Test result.
+
+### TestResult
+
+
+
+::: {.signature}
+
+TestResult(result_id:str=None,name:str='Test Result',ref_id:str=None,title:Optional\[str\]=None,doc:Optional\[str\]=None,description:Optional\[Union\[str, validmind.vm_models.DescriptionFuture\]\]=None,metric:Optional\[Union\[int, float\]\]=None,tables:Optional\[List\[validmind.vm_models.ResultTable\]\]=None,raw_data:Optional\[validmind.vm_models.RawData\]=None,figures:Optional\[List\[Figure\]\]=None,passed:Optional\[bool\]=None,params:Optional\[Dict\[str, Any\]\]=None,inputs:Optional\[Dict\[str, Union\[List\[validmind.vm_models.VMInput\], validmind.vm_models.VMInput\]\]\]=None,metadata:Optional\[Dict\[str, Any\]\]=None,\_was_description_generated:bool=False,\_unsafe:bool=False,\_client_config_cache:Optional\[Any\]=None)
+
+:::
+
+### add_figure
+
+
+
+::: {.signature}
+
+defadd_figure(self,figure:Union\[matplotlib.validmind.vm_models.figure.Figure, go.Figure, go.validmind.vm_models.FigureWidget, bytes, Figure\]):
+
+:::
+
+
+
+Add a new figure to the result.
+
+**Arguments**
+
+- `figure`: The figure to add. Can be one of:
+- matplotlib.figure.Figure: A matplotlib figure
+- plotly.graph_objs.Figure: A plotly figure
+- plotly.graph_objs.FigureWidget: A plotly figure widget
+- bytes: A PNG image as raw bytes
+- validmind.vm_models.figure.Figure: A ValidMind figure object.
+
+**Returns**
+
+- None.
+
+### add_table
+
+
+
+::: {.signature}
+
+defadd_table(self,table:Union\[validmind.vm_models.ResultTable, pd.DataFrame, List\[Dict\[str, Any\]\]\],title:Optional\[str\]=None):
+
+:::
+
+
+
+Add a new table to the result.
+
+**Arguments**
+
+- `table (Union[ResultTable, pd.DataFrame, List[Dict[str, Any]]])`: The table to add.
+- `title (Optional[str])`: The title of the table (can optionally be provided for pd.DataFrame and List\[Dict\[str, Any\]\] tables).
+
+### check_result_id_exist
+
+
+
+::: {.signature}
+
+defcheck_result_id_exist(self):
+
+:::
+
+
+
+Check if the result_id exists in any test block across all sections.
+
+### log
+
+
+
+::: {.signature}
+
+deflog(self,section_id:str=None,position:int=None,unsafe:bool=False,config:Dict\[str, bool\]=None):
+
+:::
+
+
+
+Log the result to ValidMind.
+
+**Arguments**
+
+- `section_id (str)`: The section ID within the model document to insert the test result.
+- `position (int)`: The position (index) within the section to insert the test result.
+- `unsafe (bool)`: If True, log the result even if it contains sensitive data i.e. raw data from input datasets.
+- `config (Dict[str, bool])`: Configuration options for displaying the test result. Available config options:
+- hideTitle: Hide the title in the document view
+- hideText: Hide the description text in the document view
+- hideParams: Hide the parameters in the document view
+- hideTables: Hide tables in the document view
+- hideFigures: Hide figures in the document view
+
+### log_async
+
+
+
+::: {.signature}
+
+async deflog_async(self,section_id:str=None,position:int=None,config:Dict\[str, bool\]=None):
+
+:::
+
+### remove_figure
+
+
+
+::: {.signature}
+
+defremove_figure(self,index:int=0):
+
+:::
+
+
+
+Remove a figure from the result by index.
+
+**Arguments**
+
+- `index (int)`: The index of the figure to remove (default is 0).
+
+### remove_table
+
+
+
+::: {.signature}
+
+defremove_table(self,index:int):
+
+:::
+
+
+
+Remove a table from the result by index.
+
+**Arguments**
+
+- `index (int)`: The index of the table to remove (default is 0).
+
+### serialize
+
+
+
+::: {.signature}
+
+defserialize(self):
+
+:::
+
+
+
+Serialize the result for the API.
+
+### to_widget
+
+
+
+::: {.signature}
+
+defto_widget(self):
+
+:::
+
+### validate_log_config
+
+
+
+::: {.signature}
+
+defvalidate_log_config(self,config:Dict\[str, bool\]):
+
+:::
+
+
+
+Validate the configuration options for logging a test result
+
+**Arguments**
+
+- `config (Dict[str, bool])`: Configuration options to validate
+
+**Raises**
+
+- `InvalidParameterError`: If config contains invalid keys or non-boolean values
+
+### test_name{.property}
+
+
+
+::: {.signature}
+
+test_name():
+
+:::
+
+
+
+Get the test name, using custom title if available.
+
+
+
+## TestSuite
+
+
+
+::: {.signature}
+
+@dataclass
+
+classTestSuite:
+
+:::
+
+
+
+Base class for test suites. Test suites are used to define a grouping of tests that can be run as a suite against datasets and models. Test Suites can be defined by inheriting from this base class and defining the list of tests as a class variable.
+
+Tests can be a flat list of strings or may be nested into sections by using a dict.
+
+### TestSuite
+
+
+
+::: {.signature}
+
+TestSuite(sections:List\[validmind.vm_models.TestSuiteSection\]=None)
+
+:::
+
+### get_default_config
+
+
+
+::: {.signature}
+
+defget_default_config(self) → dict:
+
+:::
+
+
+
+Returns the default configuration for the test suite.
+
+Each test in a test suite can accept parameters and those parameters can have default values. Both the parameters and their defaults are set in the test class and a config object can be passed to the test suite's run method to override the defaults. This function returns a dictionary containing the parameters and their default values for every test to allow users to view and set values.
+
+**Returns**
+
+- A dictionary of test names and their default parameters.
+
+### get_tests
+
+
+
+::: {.signature}
+
+defget_tests(self) → List\[str\]:
+
+:::
+
+
+
+Get all test suite test objects from all sections.
+
+### num_tests
+
+
+
+::: {.signature}
+
+defnum_tests(self) → int:
+
+:::
+
+
+
+Returns the total number of tests in the test suite.
+
+
+
+## TestSuiteRunner
+
+
+
+::: {.signature}
+
+classTestSuiteRunner:
+
+:::
+
+
+
+Runs a test suite.
+
+### TestSuiteRunner
+
+
+
+::: {.signature}
+
+TestSuiteRunner(suite:validmind.vm_models.TestSuite,config:dict=None,inputs:dict=None)
+
+:::
+
+### log_results
+
+
+
+::: {.signature}
+
+async deflog_results(self):
+
+:::
+
+
+
+Logs the results of the test suite to ValidMind.
+
+This method will be called after the test suite has been run and all results have been collected. This method will log the results to ValidMind.
+
+### run
+
+
+
+::: {.signature}
+
+defrun(self,send:bool=True,fail_fast:bool=False):
+
+:::
+
+
+
+Runs the test suite, renders the summary and sends the results to ValidMind.
+
+**Arguments**
+
+- `send (bool, optional)`: Whether to send the results to ValidMind. Defaults to True.
+- `fail_fast (bool, optional)`: Whether to stop running tests after the first failure. Defaults to False.
+
+### summarize
+
+
+
+::: {.signature}
+
+defsummarize(self,show_link:bool=True):
+
+:::
diff --git a/notebooks/README.md b/notebooks/README.md
index 24a923b91..d4b8b9dbc 100644
--- a/notebooks/README.md
+++ b/notebooks/README.md
@@ -16,7 +16,7 @@ ValidMind enables organizations to identify, document, and manage model risks fo
If this is your first time trying out ValidMind, you can make use of the following resources alongside our sample notebooks:
- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work
-- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference
+- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference
## Contributing code samples
diff --git a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb
index 58c581362..6da5cffde 100644
--- a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb
+++ b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models.ipynb
@@ -82,7 +82,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb
index 32f9cbce2..a8548826b 100644
--- a/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb
+++ b/notebooks/code_samples/capital_markets/quickstart_option_pricing_models_quantlib.ipynb
@@ -120,7 +120,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/credit_risk/application_scorecard_demo.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_demo.ipynb
index 7b3bc0ec6..1bc8aa931 100644
--- a/notebooks/code_samples/credit_risk/application_scorecard_demo.ipynb
+++ b/notebooks/code_samples/credit_risk/application_scorecard_demo.ipynb
@@ -86,7 +86,7 @@
"\n",
"\n",
"### New to ValidMind?\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb
index 3ee2b1e6b..8025ece67 100644
--- a/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb
+++ b/notebooks/code_samples/credit_risk/application_scorecard_executive.ipynb
@@ -37,7 +37,7 @@
"\n",
"\n",
"### New to ValidMind?\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb
index 750ebc967..77e5b2ba0 100644
--- a/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb
+++ b/notebooks/code_samples/credit_risk/application_scorecard_full_suite.ipynb
@@ -37,7 +37,7 @@
"\n",
"\n",
"### New to ValidMind?\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb
index 59d507bf2..0b909ddda 100644
--- a/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb
+++ b/notebooks/code_samples/credit_risk/application_scorecard_with_bias.ipynb
@@ -75,7 +75,7 @@
"\n",
"\n",
"### New to ValidMind?\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.\n"
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n"
]
},
{
diff --git a/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb b/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb
index 26a983f10..961a92c5c 100644
--- a/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb
+++ b/notebooks/code_samples/credit_risk/application_scorecard_with_ml.ipynb
@@ -37,7 +37,7 @@
"\n",
"\n",
"### New to ValidMind?\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb b/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb
index 35999cde8..963179129 100644
--- a/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb
+++ b/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb
@@ -37,7 +37,9 @@
" - [Custom Test: External API Call](#toc9_2_) \n",
" - [Custom Test: Passing Parameters](#toc9_3_) \n",
" - [Custom Test: Multiple Tables and Plots in a Single Test](#toc9_4_) \n",
- " - [Custom Test: Images](#toc9_5_) \n",
+ " - [Custom Test: Images](#toc9_5_)\n",
+ " - [Custom Test: Description](#toc9_6_)\n",
+ "\n",
"- [Conclusion](#toc10_) \n",
"- [Next steps](#toc11_) \n",
" - [Work with your model documentation](#toc11_1_) \n",
@@ -78,7 +80,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
@@ -867,6 +869,66 @@
""
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Custom Test: Description\n",
+ "\n",
+ "If you want to write a custom test description for your custom test instead of it is interpreted through llm, you can do so by returning string in your test."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "@vm.test(\"my_custom_tests.MyCustomTest\")\n",
+ "def my_custom_test(dataset, model):\n",
+ " \"\"\"\n",
+ " This is a custom computed test that computes confusion matrix for a binary classification model and return a string as a test description.\n",
+ " \"\"\"\n",
+ " y_true = dataset.y\n",
+ " y_pred = dataset.y_pred(model)\n",
+ "\n",
+ " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n",
+ "\n",
+ " cm_display = metrics.ConfusionMatrixDisplay(\n",
+ " confusion_matrix=confusion_matrix, display_labels=[False, True]\n",
+ " )\n",
+ " cm_display.plot()\n",
+ "\n",
+ " plt.close() # close the plot to avoid displaying it\n",
+ "\n",
+ " return cm_display.figure_, \"Test Description - Confusion Matrix\", pd.DataFrame({\"Value\": [1, 2, 3]}) # return the figure object itself\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "You can see here test result description has been customized here. The same result description will be displayed in the UI."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = run_test(\n",
+ " \"my_custom_tests.MyCustomTest\",\n",
+ " inputs={\"model\": \"model\", \"dataset\": \"test_dataset\"},\n",
+ ")\n",
+ "result.log()"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
diff --git a/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb b/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb
index 414d4cfe2..d977817e7 100644
--- a/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb
+++ b/notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb
@@ -114,7 +114,7 @@
"If this is your first time trying out ValidMind, we recommend going through the following resources first:\n",
"\n",
"- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work\n",
- "- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference\n",
+ "- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference\n",
"\n",
"\n",
"\n",
diff --git a/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb b/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb
new file mode 100644
index 000000000..2ac83e369
--- /dev/null
+++ b/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb
@@ -0,0 +1,1831 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Validate an application scorecard model\n",
+ "\n",
+ "Learn how to independently assess an application scorecard model developed using the ValidMind Library as a validator. You'll evaluate the development of the model by conducting thorough testing and analysis, including the use of challenger models to benchmark performance.\n",
+ "\n",
+ "An *application scorecard model* is a type of statistical model used in credit scoring to evaluate the creditworthiness of potential borrowers by generating a score based on various characteristics of an applicant such as credit history, income, employment status, and other relevant financial data.\n",
+ "\n",
+ " - This score assists lenders in making informed decisions about whether to approve or reject loan applications, as well as in determining the terms of the loan, including interest rates and credit limits.\n",
+ " - Effective validation of application scorecard models ensures that lenders can manage risk efficiently while maintaining a fast and transparent loan application process for applicants.\n",
+ "\n",
+ "This interactive notebook provides a step-by-step guide for:\n",
+ "\n",
+ "- Verifying the data quality steps performed by the model development team\n",
+ "- Independently replicating the champion model's results and conducting additional tests to assess performance, stability, and robustness\n",
+ "- Setting up test inputs and challenger models for comparative analysis\n",
+ "- Running validation tests, analyzing results, and logging findings to ValidMind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "::: {.content-hidden when-format=\"html\"}\n",
+ "## Contents \n",
+ "- [About ValidMind](#toc1_) \n",
+ " - [Before you begin](#toc1_1_) \n",
+ " - [New to ValidMind?](#toc1_2_) \n",
+ " - [Key concepts](#toc1_3_) \n",
+ "- [Setting up](#toc2_) \n",
+ " - [Register a sample model](#toc2_1_) \n",
+ " - [Assign validator credentials](#toc2_1_1_) \n",
+ " - [Install the ValidMind Library](#toc2_2_) \n",
+ " - [Initialize the ValidMind Library](#toc2_3_) \n",
+ " - [Get your code snippet](#toc2_3_1_) \n",
+ " - [Importing the champion model](#toc2_4_) \n",
+ " - [Load the sample dataset](#toc2_5_) \n",
+ " - [Preprocess the dataset](#toc2_5_1_) \n",
+ " - [Apply feature engineering to the dataset](#toc2_5_2_) \n",
+ " - [Split the feature engineered dataset](#toc2_6_) \n",
+ "- [Developing potential challenger models](#toc3_) \n",
+ " - [Train potential challenger models](#toc3_1_) \n",
+ " - [Random forest classification model](#toc3_1_1_) \n",
+ " - [Logistic regression model](#toc3_1_2_) \n",
+ " - [Extract predicted probabilities](#toc3_2_) \n",
+ " - [Compute binary predictions](#toc3_2_1_) \n",
+ "- [Initializing the ValidMind objects](#toc4_) \n",
+ " - [Initialize the ValidMind datasets](#toc4_1_) \n",
+ " - [Initialize the model objects](#toc4_2_) \n",
+ " - [Assign predictions](#toc4_3_) \n",
+ " - [Compute credit risk scores](#toc4_4_) \n",
+ "- [Run data quality tests](#toc5_) \n",
+ " - [Run and log an individual data quality test](#toc5_1_) \n",
+ " - [Log multiple data quality tests](#toc5_2_) \n",
+ " - [Run data quality comparison tests](#toc5_3_) \n",
+ "- [Run performance tests](#toc6_) \n",
+ " - [Identify performance tests](#toc6_1_) \n",
+ " - [Run and log an individual performance test](#toc6_2_) \n",
+ " - [Log multiple performance tests](#toc6_3_) \n",
+ " - [Evaluate performance of the champion model](#toc6_4_) \n",
+ " - [Evaluate performance of challenger models](#toc6_5_) \n",
+ " - [Enable custom context for test descriptions](#toc6_5_1_) \n",
+ " - [Run performance comparison tests](#toc6_5_2_) \n",
+ "- [Adjust a ValidMind test](#toc7_) \n",
+ "- [Run diagnostic tests](#toc8_) \n",
+ "- [Run feature importance tests](#toc9_) \n",
+ "- [Implement a custom test](#toc10_) \n",
+ "- [Verify test runs](#toc11_) \n",
+ "- [Next steps](#toc12_) \n",
+ " - [Work with your validation report](#toc12_1_) \n",
+ " - [Discover more learning resources](#toc12_2_) \n",
+ "- [Upgrade ValidMind](#toc13_) \n",
+ "\n",
+ ":::\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## About ValidMind\n",
+ "\n",
+ "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n",
+ "\n",
+ "You use the ValidMind Library to automate comparison and other validation tests, and then use the ValidMind Platform to submit compliance assessments of champion models via comprehensive validation reports. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model developers."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Before you begin\n",
+ "\n",
+ "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n",
+ "\n",
+ "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### New to ValidMind?\n",
+ "\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
+ "\n",
+ "
For access to all features available in this notebook, create a free ValidMind account.\n",
+ " \n",
+ "Signing up is FREE — Register with ValidMind
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Key concepts\n",
+ "\n",
+ "**Validation report**: A comprehensive and structured assessment of a model’s development and performance, focusing on verifying its integrity, appropriateness, and alignment with its intended use. It includes analyses of model assumptions, data quality, performance metrics, outcomes of testing procedures, and risk considerations. The validation report supports transparency, regulatory compliance, and informed decision-making by documenting the validator’s independent review and conclusions.\n",
+ "\n",
+ "**Validation report template**: Serves as a standardized framework for conducting and documenting model validation activities. It outlines the required sections, recommended analyses, and expected validation tests, ensuring consistency and completeness across validation reports. The template helps guide validators through a systematic review process while promoting comparability and traceability of validation outcomes.\n",
+ "\n",
+ "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets.\n",
+ "\n",
+ "**Metrics**: A subset of tests that do not have thresholds. In the context of this notebook, metrics and tests can be thought of as interchangeable concepts.\n",
+ "\n",
+ "**Custom metrics**: Custom metrics are functions that you define to evaluate your model or dataset. These functions can be registered with the ValidMind Library to be used in the ValidMind Platform.\n",
+ "\n",
+ "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n",
+ "\n",
+ " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n",
+ " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n",
+ " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom metric.\n",
+ " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom metric. (Learn more: [Run tests with multiple datasets](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html))\n",
+ "\n",
+ "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a metric, customize its behavior, or provide additional context.\n",
+ "\n",
+ "**Outputs**: Custom metrics can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Setting up"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Register a sample model\n",
+ "\n",
+ "In a usual model lifecycle, a champion model will have been independently registered in your model inventory and submitted to you for validation by your model development team as part of the effective challenge process. (**Learn more:** [Submit for approval](https://docs.validmind.ai/guide/model-documentation/submit-for-approval.html))\n",
+ "\n",
+ "For this notebook, we'll have you register a dummy model in the ValidMind Platform inventory and assign yourself as the validator to familiarize you with the ValidMind interface and circumvent the need for an existing model:\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n",
+ "\n",
+ "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n",
+ "\n",
+ " For example, to register a model for use with this notebook, select:\n",
+ "\n",
+ " - Documentation template: `Credit Risk Scorecard`\n",
+ " - Use case: `Credit Risk — CECL`\n",
+ "\n",
+ " You can fill in other options according to your preference."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Assign validator credentials\n",
+ "\n",
+ "In order to log tests as a validator instead of as a developer, on the model details page that appears after you've successfully registered your sample model:\n",
+ "\n",
+ "1. Remove yourself as a developer: \n",
+ "\n",
+ " - Click on the **DEVELOPERS** tile.\n",
+ " - Click the **x** next to your name to remove yourself from that model's role.\n",
+ " - Click **Save** to apply your changes to that role.\n",
+ "\n",
+ "2. Add yourself as a validator: \n",
+ "\n",
+ " - Click on the **VALIDATORS** tile.\n",
+ " - Select your name from the drop-down menu.\n",
+ " - Click **Save** to apply your changes to that role."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Install the ValidMind Library\n",
+ "\n",
+ "
\n",
+ "\n",
+ "To install the library:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip install -q validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the ValidMind Library\n",
+ "\n",
+ "ValidMind generates a unique _code snippet_ for each registered model to connect with your validation environment. You initialize the ValidMind Library with this code snippet, which ensures that your test results are uploaded to the correct model when you run the notebook."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Get your code snippet\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this notebook.\n",
+ "\n",
+ "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
+ "\n",
+ "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load your model identifier credentials from an `.env` file\n",
+ "\n",
+ "%load_ext dotenv\n",
+ "%dotenv .env\n",
+ "\n",
+ "# Or replace with your code snippet\n",
+ "\n",
+ "import validmind as vm\n",
+ "\n",
+ "vm.init(\n",
+ " # api_host=\"...\",\n",
+ " # api_key=\"...\",\n",
+ " # api_secret=\"...\",\n",
+ " # model=\"...\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Importing the champion model\n",
+ "\n",
+ "With the ValidMind Library set up and ready to go, let's go ahead and import the champion model submitted by the model development team in the format of a `.pkl` file: **[xgb_model_champion.pkl](xgb_model_champion.pkl)**\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import xgboost as xgb\n",
+ "\n",
+ "#Load the saved model\n",
+ "xgb_model = xgb.XGBClassifier()\n",
+ "xgb_model.load_model(\"xgb_model_champion.pkl\")\n",
+ "xgb_model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Ensure that we have to appropriate order in feature names from Champion model and dataset\n",
+ "cols_when_model_builds = xgb_model.get_booster().feature_names"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Load the sample dataset\n",
+ "\n",
+ "Let's next import the public [Lending Club](https://www.kaggle.com/datasets/devanshi23/loan-data-2007-2014/data) dataset from Kaggle, which was used to develop the dummy champion model.\n",
+ "\n",
+ "- We'll use this dataset to review steps that should have been conducted during the initial development and documentation of the model to ensure that the model was built correctly.\n",
+ "- By independently performing steps such as preprocessing and feature engineering, we can confirm whether the model was built using appropriate and properly processed data.\n",
+ "\n",
+ "To be able to use the dataset, you'll need to import the dataset and load it into a pandas [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html), a two-dimensional tabular data structure that makes use of rows and columns:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from validmind.datasets.credit_risk import lending_club\n",
+ "\n",
+ "df = lending_club.load_data(source=\"offline\")\n",
+ "df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Preprocess the dataset\n",
+ "\n",
+ "We'll first quickly preprocess the dataset for data quality testing purposes using `lending_club.preprocess`. This function performs the following operations:\n",
+ "\n",
+ "- Filters the dataset to include only loans for debt consolidation or credit card purposes\n",
+ "- Removes loans classified under the riskier grades \"F\" and \"G\"\n",
+ "- Excludes uncommon home ownership types and standardizes employment length and loan terms into numerical formats\n",
+ "- Discards unnecessary fields and any entries with missing information to maintain a clean and robust dataset for modeling"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "preprocess_df = lending_club.preprocess(df)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Apply feature engineering to the dataset\n",
+ "\n",
+ "Feature engineering improves the dataset's structure to better match what our model expects, and ensures that the model performs optimally by leveraging additional insights from raw data.\n",
+ "\n",
+ "We'll apply the following transformations using the `ending_club.feature_engineering()` function to optimize the dataset for predictive modeling in our application scorecard:\n",
+ "\n",
+ "- **WoE encoding**: Converts both numerical and categorical features into Weight of Evidence (WoE) values. WoE is a statistical measure used in scorecard modeling that quantifies the relationship between a predictor variable and the binary target variable. It calculates the ratio of the distribution of good outcomes to the distribution of bad outcomes for each category or bin of a feature. This transformation helps to ensure that the features are predictive and consistent in their contribution to the model.\n",
+ "- **Integration of WoE bins**: Ensures that the WoE transformed values are integrated throughout the dataset, replacing the original feature values while excluding the target variable from this transformation. This transformation is used to maintain a consistent scale and impact of each variable within the model, which helps make the predictions more stable and accurate."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fe_df = lending_club.feature_engineering(preprocess_df)\n",
+ "fe_df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Split the feature engineered dataset\n",
+ "\n",
+ "With our dummy model imported and our independently preprocessed and feature engineered dataset ready to go, let's now **spilt our dataset into train and test** to start the validation testing process.\n",
+ "\n",
+ "Splitting our dataset into training and testing is essential for proper validation testing, as this helps assess how well the model generalizes to unseen data:\n",
+ "\n",
+ "- We begin by dividing our data, which is based on Weight of Evidence (WoE) features, into training and testing sets (`train_df`, `test_df`).\n",
+ "- With `lending_club.split`, we employ a simple random split, randomly allocating data points to each set to ensure a mix of examples in both."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Split the data\n",
+ "train_df, test_df = lending_club.split(fe_df, test_size=0.2)\n",
+ "\n",
+ "x_train = train_df.drop(lending_club.target_column, axis=1)\n",
+ "y_train = train_df[lending_club.target_column]\n",
+ "\n",
+ "x_test = test_df.drop(lending_club.target_column, axis=1)\n",
+ "y_test = test_df[lending_club.target_column]\n",
+ "\n",
+ "# Now let's apply the order of features from the champion model construction\n",
+ "x_train = x_train[cols_when_model_builds]\n",
+ "x_test = x_test[cols_when_model_builds]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cols_use = ['annual_inc_woe',\n",
+ " 'verification_status_woe',\n",
+ " 'emp_length_woe',\n",
+ " 'installment_woe',\n",
+ " 'term_woe',\n",
+ " 'home_ownership_woe',\n",
+ " 'purpose_woe',\n",
+ " 'open_acc_woe',\n",
+ " 'total_acc_woe',\n",
+ " 'int_rate_woe',\n",
+ " 'sub_grade_woe',\n",
+ " 'grade_woe','loan_status']\n",
+ "\n",
+ "\n",
+ "train_df = train_df[cols_use]\n",
+ "test_df = test_df[cols_use]\n",
+ "test_df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Developing potential challenger models"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Train potential challenger models\n",
+ "\n",
+ "We're curious how alternate models compare to our champion model, so let's train two challenger models as basis for our testing.\n",
+ "\n",
+ "Our selected options below offer decreased complexity in terms of implementation — such as lessened manual preprocessing — which can reduce the amount of risk for implementation. However, model risk is not calculated in isolation from a single factor, but rather in consideration with trade-offs in predictive performance, ease of interpretability, and overall alignment with business objectives."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Random forest classification model\n",
+ "\n",
+ "A *random forest classification model* is an ensemble machine learning algorithm that uses multiple decision trees to classify data. In ensemble learning, multiple models are combined to improve prediction accuracy and robustness.\n",
+ "\n",
+ "Random forest classification models generally have higher accuracy because they capture complex, non-linear relationships, but as a result they lack transparency in their predictions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Import the Random Forest Classification model\n",
+ "from sklearn.ensemble import RandomForestClassifier\n",
+ "\n",
+ "# Create the model instance with 50 decision trees\n",
+ "rf_model = RandomForestClassifier(\n",
+ " n_estimators=50,\n",
+ " random_state=42,\n",
+ ")\n",
+ "\n",
+ "# Train the model\n",
+ "rf_model.fit(x_train, y_train)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Logistic regression model\n",
+ "\n",
+ "A *logistic regression model* is a statistical machine learning algorithm that uses a linear equation (straight-line relationship between variables) and the logistic function (or sigmoid function, which maps any real-valued number to a range between `0` and `1`) to classify data. In statistical modeling, a single equation is used to estimate the probability of an outcome based on input features.\n",
+ "\n",
+ "Logistic regression models are simple and interpretable because they provide clear probability estimates and feature coefficients (numerical value that represents the influence of a particular input feature on the model's prediction), but they may struggle with capturing complex, non-linear relationships in the data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Import the Logistic Regression model\n",
+ "from sklearn.linear_model import LogisticRegression\n",
+ "\n",
+ "# Logistic Regression grid params\n",
+ "log_reg_params = {\n",
+ " \"penalty\": [\"l1\", \"l2\"],\n",
+ " \"C\": [0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
+ " \"solver\": [\"liblinear\"],\n",
+ "}\n",
+ "\n",
+ "# Grid search for Logistic Regression\n",
+ "from sklearn.model_selection import GridSearchCV\n",
+ "\n",
+ "grid_log_reg = GridSearchCV(LogisticRegression(), log_reg_params)\n",
+ "grid_log_reg.fit(x_train, y_train)\n",
+ "\n",
+ "# Logistic Regression best estimator\n",
+ "log_reg = grid_log_reg.best_estimator_\n",
+ "log_reg"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Extract predicted probabilities\n",
+ "\n",
+ "With our challenger models trained, let's extract the predicted probabilities from our three models:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Champion — Application scorecard model\n",
+ "train_xgb_prob = xgb_model.predict_proba(x_train)[:, 1]\n",
+ "test_xgb_prob = xgb_model.predict_proba(x_test)[:, 1]\n",
+ "\n",
+ "# Challenger — Random forest classification model\n",
+ "train_rf_prob = rf_model.predict_proba(x_train)[:, 1]\n",
+ "test_rf_prob = rf_model.predict_proba(x_test)[:, 1]\n",
+ "\n",
+ "# Challenger — Logistic regression model\n",
+ "train_log_prob = log_reg.predict_proba(x_train)[:, 1]\n",
+ "test_log_prob = log_reg.predict_proba(x_test)[:, 1]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Compute binary predictions\n",
+ "\n",
+ "Next, we'll convert the probability predictions from our three models into a binary, based on a threshold of `0.3`:\n",
+ "\n",
+ "- If the probability is greater than `0.3`, the prediction becomes `1` (positive).\n",
+ "- Otherwise, it becomes `0` (negative)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cut_off_threshold = 0.3\n",
+ "\n",
+ "# Champion — Application scorecard model\n",
+ "train_xgb_binary_predictions = (train_xgb_prob > cut_off_threshold).astype(int)\n",
+ "test_xgb_binary_predictions = (test_xgb_prob > cut_off_threshold).astype(int)\n",
+ "\n",
+ "# Challenger — Random forest classification model\n",
+ "train_rf_binary_predictions = (train_rf_prob > cut_off_threshold).astype(int)\n",
+ "test_rf_binary_predictions = (test_rf_prob > cut_off_threshold).astype(int)\n",
+ "\n",
+ "# Challenger — Logistic regression model\n",
+ "train_log_binary_predictions = (train_log_prob > cut_off_threshold).astype(int)\n",
+ "test_log_binary_predictions = (test_log_prob > cut_off_threshold).astype(int)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Initializing the ValidMind objects"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the ValidMind datasets\n",
+ "\n",
+ "Before you can run tests, you'll need to connect your data with a ValidMind `Dataset` object. **This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind,** but you only need to do it once per dataset.\n",
+ "\n",
+ "Initialize a ValidMind dataset object using the [`init_dataset` function](https://docs.validmind.ai/validmind/validmind.html#init_dataset) from the ValidMind (`vm`) module. For this example, we'll pass in the following arguments:\n",
+ "\n",
+ "- **`dataset`** — The raw dataset that you want to provide as input to tests.\n",
+ "- **`input_id`** — A unique identifier that allows tracking what inputs are used when running each individual test.\n",
+ "- **`target_column`** — A required argument if tests require access to true values. This is the name of the target column in the dataset."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize the raw dataset\n",
+ "vm_raw_dataset = vm.init_dataset(\n",
+ " dataset=df,\n",
+ " input_id=\"raw_dataset\",\n",
+ " target_column=lending_club.target_column,\n",
+ ")\n",
+ "\n",
+ "# Initialize the preprocessed dataset\n",
+ "vm_preprocess_dataset = vm.init_dataset(\n",
+ " dataset=preprocess_df,\n",
+ " input_id=\"preprocess_dataset\",\n",
+ " target_column=lending_club.target_column,\n",
+ ")\n",
+ "\n",
+ "# Initialize the feature engineered dataset\n",
+ "vm_fe_dataset = vm.init_dataset(\n",
+ " dataset=fe_df,\n",
+ " input_id=\"fe_dataset\",\n",
+ " target_column=lending_club.target_column,\n",
+ ")\n",
+ "\n",
+ "# Initialize the training dataset\n",
+ "vm_train_ds = vm.init_dataset(\n",
+ " dataset=train_df,\n",
+ " input_id=\"train_dataset\",\n",
+ " target_column=lending_club.target_column,\n",
+ ")\n",
+ "\n",
+ "# Initialize the test dataset\n",
+ "vm_test_ds = vm.init_dataset(\n",
+ " dataset=test_df,\n",
+ " input_id=\"test_dataset\",\n",
+ " target_column=lending_club.target_column,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "After initialization, you can pass the ValidMind `Dataset` objects `vm_raw_dataset`, `vm_preprocess_dataset`, `vm_fe_dataset`, `vm_train_ds`, and `vm_test_ds` into any ValidMind tests."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the model objects\n",
+ "\n",
+ "You'll also need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data for each of our three models.\n",
+ "\n",
+ "You simply initialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize the champion application scorecard model\n",
+ "vm_xgb_model = vm.init_model(\n",
+ " xgb_model,\n",
+ " input_id=\"xgb_model_developer_champion\",\n",
+ ")\n",
+ "\n",
+ "# Initialize the challenger random forest classification model\n",
+ "vm_rf_model = vm.init_model(\n",
+ " rf_model,\n",
+ " input_id=\"rf_model\",\n",
+ ")\n",
+ "\n",
+ "# Initialize the challenger logistic regression model\n",
+ "vm_log_model = vm.init_model(\n",
+ " log_reg,\n",
+ " input_id=\"log_model\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Assign predictions\n",
+ "\n",
+ "With our models registered, we'll move on to assigning both the predictive probabilities coming directly from each model's predictions, and the binary prediction after applying the cutoff threshold described in the Compute binary predictions step above.\n",
+ "\n",
+ "- The [`assign_predictions()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#VMDataset.assign_predictions) from the `Dataset` object can link existing predictions to any number of models.\n",
+ "- This method links the model's class prediction values and probabilities to our `vm_train_ds` and `vm_test_ds` datasets."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Champion — Application scorecard model\n",
+ "vm_train_ds.assign_predictions(\n",
+ " model=vm_xgb_model,\n",
+ " prediction_values=train_xgb_binary_predictions,\n",
+ " prediction_probabilities=train_xgb_prob,\n",
+ ")\n",
+ "\n",
+ "vm_test_ds.assign_predictions(\n",
+ " model=vm_xgb_model,\n",
+ " prediction_values=test_xgb_binary_predictions,\n",
+ " prediction_probabilities=test_xgb_prob,\n",
+ ")\n",
+ "\n",
+ "# Challenger — Random forest classification model\n",
+ "vm_train_ds.assign_predictions(\n",
+ " model=vm_rf_model,\n",
+ " prediction_values=train_rf_binary_predictions,\n",
+ " prediction_probabilities=train_rf_prob,\n",
+ ")\n",
+ "\n",
+ "vm_test_ds.assign_predictions(\n",
+ " model=vm_rf_model,\n",
+ " prediction_values=test_rf_binary_predictions,\n",
+ " prediction_probabilities=test_rf_prob,\n",
+ ")\n",
+ "\n",
+ "\n",
+ "# Challenger — Logistic regression model\n",
+ "vm_train_ds.assign_predictions(\n",
+ " model=vm_log_model,\n",
+ " prediction_values=train_log_binary_predictions,\n",
+ " prediction_probabilities=train_log_prob,\n",
+ ")\n",
+ "\n",
+ "vm_test_ds.assign_predictions(\n",
+ " model=vm_log_model,\n",
+ " prediction_values=test_log_binary_predictions,\n",
+ " prediction_probabilities=test_log_prob,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Compute credit risk scores\n",
+ "\n",
+ "Finally, we'll translate model predictions into actionable scores using probability estimates generated by our trained model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Compute the scores\n",
+ "train_xgb_scores = lending_club.compute_scores(train_xgb_prob)\n",
+ "test_xgb_scores = lending_club.compute_scores(test_xgb_prob)\n",
+ "train_rf_scores = lending_club.compute_scores(train_rf_prob)\n",
+ "test_rf_scores = lending_club.compute_scores(test_rf_prob)\n",
+ "train_log_scores = lending_club.compute_scores(train_log_prob)\n",
+ "test_log_scores = lending_club.compute_scores(test_log_prob)\n",
+ "\n",
+ "# Assign scores to the datasets\n",
+ "vm_train_ds.add_extra_column(\"xgb_scores\", train_xgb_scores)\n",
+ "vm_test_ds.add_extra_column(\"xgb_scores\", test_xgb_scores)\n",
+ "vm_train_ds.add_extra_column(\"rf_scores\", train_rf_scores)\n",
+ "vm_test_ds.add_extra_column(\"rf_scores\", test_rf_scores)\n",
+ "vm_train_ds.add_extra_column(\"log_scores\", train_log_scores)\n",
+ "vm_test_ds.add_extra_column(\"log_scores\", test_log_scores)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Run data quality tests\n",
+ "\n",
+ "With everything ready to go, let's explore some of ValidMind's available tests. Using ValidMind’s repository of tests streamlines your validation testing, and helps you ensure that your models are being validated appropriately.\n",
+ "\n",
+ "We want to narrow down the tests we want to run from the selection provided by ValidMind, so we'll use the [`vm.tests.list_tasks_and_tags()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tasks_and_tags) to list which `tags` are associated with each `task` type:\n",
+ "\n",
+ "- **`tasks`** represent the kind of modeling task associated with a test. Here we'll focus on `classification` tasks.\n",
+ "- **`tags`** are free-form descriptions providing more details about the test, for example, what category the test falls into. Here we'll focus on the `data_quality` tag."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.list_tasks_and_tags()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Then we'll call [the `vm.tests.list_tests()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to list all the data quality tests for classification:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.list_tests(\n",
+ " tags=[\"data_quality\"], task=\"classification\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "
Want to learn more about navigating ValidMind tests?\n",
+ " \n",
+ "Refer to our notebook outlining the utilities available for viewing and understanding available ValidMind tests: Explore tests
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Run and log an individual data quality test\n",
+ "\n",
+ "Next, we'll use our previously initialized preprocessed dataset (`vm_preprocess_dataset`) as input to run an individual test, then log the result to the ValidMind Platform.\n",
+ "\n",
+ "- You run validation tests by calling [the `run_test` function](https://docs.validmind.ai/validmind/validmind/tests.html#run_test) provided by the `validmind.tests` module.\n",
+ "- Every test result returned by the `run_test()` function has a [`.log()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#TestResult.log) that can be used to send the test results to the ValidMind Platform.\n",
+ "\n",
+ "Here, we'll use the [`HighPearsonCorrelation` test](https://docs.validmind.ai/tests/data_validation/HighPearsonCorrelation.html) as an example:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_preprocess_dataset\n",
+ " }\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "
Note the output returned indicating that a test-driven block doesn't currently exist in your model's documentation for some test IDs. \n",
+ " \n",
+ "That's expected, as when we run validations tests the results logged need to be manually added to your report as part of your compliance assessment process within the ValidMind Platform. You'll continue to see this message throughout this notebook as we run and log more tests.
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Log multiple data quality tests\n",
+ "\n",
+ "Now that we understand how to run a test with ValidMind, we want to run all the tests that were returned for our `classification` tasks focusing on `data_quality`.\n",
+ "\n",
+ "We'll store the identified tests in `dq` in preparation for batch running these tests and logging their results to the ValidMind Platform:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dq = vm.tests.list_tests(tags=[\"data_quality\"], task=\"classification\",pretty=False)\n",
+ "dq"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "With our data quality tests stored, let's run our first batch of tests using the same preprocessed dataset (`vm_preprocess_dataset`) and log their results."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for test in dq:\n",
+ " vm.tests.run_test(\n",
+ " test,\n",
+ " inputs={\n",
+ " \"dataset\": vm_preprocess_dataset\n",
+ " }\n",
+ " ).log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Run data quality comparison tests\n",
+ "\n",
+ "Next, let's reuse the tests in `dq` to perform comparison tests between the raw (`vm_raw_dataset`) and preprocessed (`vm_preprocess_dataset`) dataset, again logging the results to the ValidMind Platform:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for test in dq:\n",
+ " vm.tests.run_test(\n",
+ " test,\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_raw_dataset,vm_preprocess_dataset]\n",
+ " }\n",
+ " ).log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Run performance tests\n",
+ "\n",
+ "We'll also run some performance tests, beginning with independent testing of our champion application scorecard model, then moving on to our potential challenger models."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Identify performance tests\n",
+ "\n",
+ "Use `vm.tests.list_tests()` to this time identify all the model performance tests for classification:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "vm.tests.list_tests(tags=[\"model_performance\"], task=\"classification\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Run and log an individual performance test\n",
+ "\n",
+ "Before we run our batch of performance tests, we'll use our previously initialized testing dataset (`vm_test_ds`) as input to run an individual test, then log the result to the ValidMind Platform.\n",
+ "\n",
+ "When running individual tests, you can use a custom `result_id` to tag the individual result with a unique identifier by appending this `result_id` to the `test_id` with a `:` separator. We'll append an identifier for our champion model here (`xgboost_champion`):\n",
+ "\n",
+ "Here, we'll use the [`ClassifierPerformance` test](https://docs.validmind.ai/tests/model_validation/sklearn/ClassifierPerformance.html) as an example:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.run_test(\n",
+ " test_id=\"validmind.model_validation.sklearn.ClassifierPerformance:xgboost_champion\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_test_ds, \"model\" : vm_xgb_model\n",
+ " }\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Log multiple performance tests\n",
+ "\n",
+ "We only want to run a few other tests that were returned for our `classification` tasks focusing on `model_performance`, so we'll isolate the specific tests we want to batch run in `mpt`:\n",
+ "\n",
+ "- `ClassifierPerformance`\n",
+ "- [`ConfusionMatrix`](https://docs.validmind.ai/tests/model_validation/sklearn/ConfusionMatrix.html)\n",
+ "- [`MinimumAccuracy`](https://docs.validmind.ai/tests/model_validation/sklearn/MinimumAccuracy.html)\n",
+ "- [`MinimumF1Score`](https://docs.validmind.ai/tests/model_validation/sklearn/MinimumF1Score.html)\n",
+ "- [`ROCCurve`](https://docs.validmind.ai/tests/model_validation/sklearn/ROCCurve.html)\n",
+ "\n",
+ "Note the custom `result_id`s appended to the `test_id`s for our champion model (`xgboost_champion`):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mpt = [\n",
+ " \"validmind.model_validation.sklearn.ClassifierPerformance:xgboost_champion\",\n",
+ " \"validmind.model_validation.sklearn.ConfusionMatrix:xgboost_champion\",\n",
+ " \"validmind.model_validation.sklearn.MinimumAccuracy:xgboost_champion\",\n",
+ " \"validmind.model_validation.sklearn.MinimumF1Score:xgboost_champion\",\n",
+ " \"validmind.model_validation.sklearn.ROCCurve:xgboost_champion\"\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Evaluate performance of the champion model\n",
+ "\n",
+ "Now, let's run and log our batch of model performance tests using our testing dataset (`vm_test_ds`) for our champion model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for test in mpt:\n",
+ " vm.tests.run_test(\n",
+ " test,\n",
+ " inputs={\n",
+ " \"dataset\": vm_test_ds, \"model\" : vm_xgb_model\n",
+ " },\n",
+ " ).log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Evaluate performance of challenger models\n",
+ "\n",
+ "We've now conducted similar tests as the model development team for our champion model, with the aim of verifying their test results.\n",
+ "\n",
+ "Next, let's see how our challenger models compare. We'll use the same batch of tests here as we did in `mpt`, but append a different `result_id` to indicate that these results should be associated with our challenger models:\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mpt_chall = [\n",
+ " \"validmind.model_validation.sklearn.ClassifierPerformance:xgboost_champion_vs_challengers\",\n",
+ " \"validmind.model_validation.sklearn.ConfusionMatrix:xgboost_champion_vs_challengers\",\n",
+ " \"validmind.model_validation.sklearn.MinimumAccuracy:xgboost_champion_vs_challengers\",\n",
+ " \"validmind.model_validation.sklearn.MinimumF1Score:xgboost_champion_vs_challengers\",\n",
+ " \"validmind.model_validation.sklearn.ROCCurve:xgboost_champion_vs_challengers\"\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Enable custom context for test descriptions"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "When you run ValidMind tests, test descriptions are automatically generated with LLM using the test results, the test name, and the static test definitions provided in the test’s docstring. While this metadata offers valuable high-level overviews of tests, insights produced by the LLM-based descriptions may not always align with your specific use cases or incorporate organizational policy requirements.\n",
+ "\n",
+ "Before we run our next batch of tests, we'll include some custom use case context to focus on comparison testing going forward, improving the relevancy, insight, and format of the test descriptions returned. By default, custom context for LLM-generated descriptions is disabled, meaning that the output will not include any additional context. To enable custom use case context, set the `VALIDMIND_LLM_DESCRIPTIONS_CONTEXT_ENABLED` environment variable to `1`.\n",
+ "\n",
+ "This is a global setting that will affect all tests for your linked model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "os.environ[\"VALIDMIND_LLM_DESCRIPTIONS_CONTEXT_ENABLED\"] = \"1\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0d1e90ba",
+ "metadata": {},
+ "source": [
+ "Enabling use case context allows you to pass in additional context to the LLM-generated text descriptions within `context`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "os.environ[\"VALIDMIND_LLM_DESCRIPTIONS_CONTEXT_ENABLED\"] = \"1\"\n",
+ "\n",
+ "context = \"\"\"\n",
+ "FORMAT FOR THE LLM DESCRIPTIONS: \n",
+ " **** is designed to .\n",
+ "\n",
+ " The test operates by \n",
+ "\n",
+ " The primary advantages of this test include \n",
+ "\n",
+ " Users should be aware that \n",
+ "\n",
+ " **Key Insights:**\n",
+ "\n",
+ " The test results reveal:\n",
+ "\n",
+ " - ****: \n",
+ " - ****: \n",
+ " ...\n",
+ "\n",
+ " Based on these results, \n",
+ "\n",
+ "ADDITIONAL INSTRUCTIONS:\n",
+ "\n",
+ " The champion model as the basis for comparison is called \"xgb_model_developer_champion\" and emphasis should be on the following:\n",
+ " - The metrics for the champion model compared against the challenger models\n",
+ " - Which model potentially outperforms the champion model based on the metrics, this should be highlighted and emphasized\n",
+ "\n",
+ "\n",
+ " For each metric in the test results, include in the test overview:\n",
+ " - The metric's purpose and what it measures\n",
+ " - Its mathematical formula\n",
+ " - The range of possible values\n",
+ " - What constitutes good/bad performance\n",
+ " - How to interpret different values\n",
+ "\n",
+ " Each insight should progressively cover:\n",
+ " 1. Overall scope and distribution\n",
+ " 2. Complete breakdown of all elements with specific values\n",
+ " 3. Natural groupings and patterns\n",
+ " 4. Comparative analysis between datasets/categories\n",
+ " 5. Stability and variations\n",
+ " 6. Notable relationships or dependencies\n",
+ "\n",
+ " Remember:\n",
+ " - Champion model (xgb_model_developer_champion) is the selection and challenger models are used to challenge the selection\n",
+ " - Keep all insights at the same level (no sub-bullets or nested structures)\n",
+ " - Make each insight complete and self-contained\n",
+ " - Include specific numerical values and ranges\n",
+ " - Cover all elements in the results comprehensively\n",
+ " - Maintain clear, concise language\n",
+ " - Use only \"- **Title**: Description\" format for insights\n",
+ " - Progress naturally from general to specific observations\n",
+ "\n",
+ "\"\"\".strip()\n",
+ "\n",
+ "os.environ[\"VALIDMIND_LLM_DESCRIPTIONS_CONTEXT\"] = context"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Run performance comparison tests\n",
+ "\n",
+ "With the use case context set, we'll run each test in `mpt_chall` once for each model with the same `vm_test_ds` dataset to compare them:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for test in mpt_chall:\n",
+ " vm.tests.run_test(\n",
+ " test,\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds], \"model\" : [vm_xgb_model,vm_log_model,vm_rf_model]\n",
+ " }\n",
+ " ).log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "
Based on the performance metrics, we can conclude that the random forest classification model is not a viable candidate for our use case and can be disregarded in our tests going forward.\n",
+ " \n",
+ "In the next section, we'll dive a bit deeper into some tests comparing our champion application scorecard model and our remaining challenger logistic regression model, including tests that will allow us to customize parameters and thresholds for performance standards.
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Adjust a ValidMind test\n",
+ "\n",
+ "Let's dig deeper into the `MinimumF1Score` test we ran previously in Run performance tests to ensure that the models maintain a minimum acceptable balance between *precision* and *recall*. Precision refers to how many out of the positive predictions made by the model were actually correct, and recall refers to how many out of the actual positive cases did the model correctly identify.\n",
+ "\n",
+ "Use `run_test()` with our testing dataset (`vm_test_ds`) to run the test in isolation again for our two remaining models without logging the result to have the output to compare with a subsequent iteration:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.run_test(\n",
+ " \"validmind.model_validation.sklearn.MinimumF1Score:xgboost_champion_vs_challengers\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\": [vm_xgb_model, vm_log_model]\n",
+ " },\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "As `MinimumF1Score` allows us to customize parameters and thresholds for performance standards, let's adjust the threshold to see if it improves metrics:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = vm.tests.run_test(\n",
+ " \"validmind.model_validation.sklearn.MinimumF1Score:AdjThreshold\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\": [vm_xgb_model, vm_log_model],\n",
+ " \"params\": {\"min_threshold\": 0.35}\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Run diagnostic tests\n",
+ "\n",
+ "Next we want to inspect the robustness and stability testing comparison between our champion and challenger model.\n",
+ "\n",
+ "Use `list_tests()` to identify all the model diagnosis tests for classification:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.list_tests(tags=[\"model_diagnosis\"], task=\"classification\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's see if models suffer from any *overfit* potentials and also where there are potential sub-segments of issues with the [`OverfitDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/OverfitDiagnosis.html). \n",
+ "\n",
+ "Overfitting occurs when a model learns the training data too well, capturing not only the true pattern but noise and random fluctuations resulting in excellent performance on the training dataset but poor generalization to new, unseen data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.run_test(\n",
+ " test_id=\"validmind.model_validation.sklearn.OverfitDiagnosis:Champion_vs_LogRegression\",\n",
+ " input_grid={\n",
+ " \"datasets\": [[vm_train_ds,vm_test_ds]],\n",
+ " \"model\" : [vm_xgb_model,vm_log_model]\n",
+ " }\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's also conduct *robustness* and *stability* testing of the two models with the [`RobustnessDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/RobustnessDiagnosis.html).\n",
+ "\n",
+ "Robustness refers to a model's ability to maintain consistent performance, and stability refers to a model's ability to produce consistent outputs over time across different data subsets.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.run_test(\n",
+ " test_id=\"validmind.model_validation.sklearn.RobustnessDiagnosis:Champion_vs_LogRegression\",\n",
+ " input_grid={\n",
+ " \"datasets\": [[vm_train_ds,vm_test_ds]],\n",
+ " \"model\" : [vm_xgb_model,vm_log_model]\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Run feature importance tests\n",
+ "\n",
+ "We also want to verify the relative influence of different input features on our models' predictions, as well as inspect the differences between our champion and challenger model to see if a certain model offers more understandable or logical importance scores for features.\n",
+ "\n",
+ "Use `list_tests()` to identify all the feature importance tests for classification:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Store the feature importance tests\n",
+ "FI = vm.tests.list_tests(tags=[\"feature_importance\"], task=\"classification\",pretty=False)\n",
+ "FI"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Run and log our feature importance tests for both models for the testing dataset\n",
+ "for test in FI:\n",
+ " vm.tests.run_test(\n",
+ " \"\".join((test,':Champion_vs_LogisticRegression')),\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds], \"model\" : [vm_xgb_model,vm_log_model]\n",
+ " },\n",
+ " ).log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Implement a custom test\n",
+ "\n",
+ "Let's finish up testing by implementing a custom *inline test* that outputs a FICO score-type score. An inline test refers to a test written and executed within the same environment as the code being tested — in this case, right in this Jupyter Notebook — without requiring a separate test file or framework.\n",
+ "\n",
+ "The [`@vm.test` wrapper](https://docs.validmind.ai/validmind/validmind.html#test) allows you to create a reusable test:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import plotly.graph_objects as go\n",
+ "\n",
+ "@vm.test(\"my_custom_tests.ScoreToOdds\")\n",
+ "def score_to_odds_analysis(dataset, score_column='score', score_bands=[410, 440, 470]):\n",
+ " \"\"\"\n",
+ " Analyzes the relationship between score bands and odds (good:bad ratio).\n",
+ " Good odds = (1 - default_rate) / default_rate\n",
+ " \n",
+ " Higher scores should correspond to higher odds of being good.\n",
+ "\n",
+ " If there are multiple scores provided through score_column, this means that there are two different models and the scores reflect each model\n",
+ "\n",
+ " If there are more scores provided in the score_column then focus the assessment on the differences between the two scores and indicate through evidence which one is preferred.\n",
+ " \"\"\"\n",
+ " df = dataset.df\n",
+ " \n",
+ " # Create score bands\n",
+ " df['score_band'] = pd.cut(\n",
+ " df[score_column],\n",
+ " bins=[-np.inf] + score_bands + [np.inf],\n",
+ " labels=[f'<{score_bands[0]}'] + \n",
+ " [f'{score_bands[i]}-{score_bands[i+1]}' for i in range(len(score_bands)-1)] +\n",
+ " [f'>{score_bands[-1]}']\n",
+ " )\n",
+ " \n",
+ " # Calculate metrics per band\n",
+ " results = df.groupby('score_band').agg({\n",
+ " dataset.target_column: ['mean', 'count']\n",
+ " })\n",
+ " \n",
+ " results.columns = ['Default Rate', 'Total']\n",
+ " results['Good Count'] = results['Total'] - (results['Default Rate'] * results['Total'])\n",
+ " results['Bad Count'] = results['Default Rate'] * results['Total']\n",
+ " results['Odds'] = results['Good Count'] / results['Bad Count']\n",
+ " \n",
+ " # Create visualization\n",
+ " fig = go.Figure()\n",
+ " \n",
+ " # Add odds bars\n",
+ " fig.add_trace(go.Bar(\n",
+ " name='Odds (Good:Bad)',\n",
+ " x=results.index,\n",
+ " y=results['Odds'],\n",
+ " marker_color='blue'\n",
+ " ))\n",
+ " \n",
+ " fig.update_layout(\n",
+ " title='Score-to-Odds Analysis',\n",
+ " yaxis=dict(title='Odds Ratio (Good:Bad)'),\n",
+ " showlegend=False\n",
+ " )\n",
+ " \n",
+ " return fig"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "With the custom test available, run and log the test for our champion and challenger models with our testing dataset (`vm_test_ds`):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = vm.tests.run_test(\n",
+ " \"my_custom_tests.ScoreToOdds:Champion_vs_Challenger\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_test_ds,\n",
+ " },\n",
+ " param_grid={\n",
+ " \"score_column\": [\"xgb_scores\",\"log_scores\"],\n",
+ " \"score_bands\": [[500, 540, 570]],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "
Want to learn more about custom tests?\n",
+ " \n",
+ "Refer to our in-depth introduction to custom tests: Implement custom tests
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Verify test runs\n",
+ "\n",
+ "Our final task is to verify that all the tests provided by the model development team were run and reported accurately. Note the appended `result_ids` to delineate which dataset we ran the test with for the relevant tests.\n",
+ "\n",
+ "Here, we'll specify all the tests we'd like to independently rerun in a dictionary called `test_config`. **Note here that `inputs` and `input_grid` expect the `input_id` of the dataset or model as the value rather than the variable name we specified**:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_config = {\n",
+ " # Run with the raw dataset\n",
+ " 'validmind.data_validation.DatasetDescription:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'}\n",
+ " },\n",
+ " 'validmind.data_validation.DescriptiveStatistics:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'}\n",
+ " },\n",
+ " 'validmind.data_validation.MissingValues:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'min_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.ClassImbalance:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'min_percent_threshold': 10}\n",
+ " },\n",
+ " 'validmind.data_validation.Duplicates:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'min_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.HighCardinality:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {\n",
+ " 'num_threshold': 100,\n",
+ " 'percent_threshold': 0.1,\n",
+ " 'threshold_type': 'percent'\n",
+ " }\n",
+ " },\n",
+ " 'validmind.data_validation.Skewness:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'max_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.UniqueRows:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'min_percent_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.TooManyZeroValues:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'max_percent_threshold': 0.03}\n",
+ " },\n",
+ " 'validmind.data_validation.IQROutliersTable:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'threshold': 5}\n",
+ " },\n",
+ " # Run with the preprocessed dataset\n",
+ " 'validmind.data_validation.DescriptiveStatistics:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'preprocess_dataset'}\n",
+ " },\n",
+ " 'validmind.data_validation.TabularDescriptionTables:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'preprocess_dataset'}\n",
+ " },\n",
+ " 'validmind.data_validation.MissingValues:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'preprocess_dataset'},\n",
+ " 'params': {'min_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.TabularNumericalHistograms:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'preprocess_dataset'}\n",
+ " },\n",
+ " 'validmind.data_validation.TabularCategoricalBarPlots:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'preprocess_dataset'}\n",
+ " },\n",
+ " 'validmind.data_validation.TargetRateBarPlots:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'preprocess_dataset'},\n",
+ " 'params': {'default_column': 'loan_status'}\n",
+ " },\n",
+ " # Run with the training and test datasets\n",
+ " 'validmind.data_validation.DescriptiveStatistics:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset', 'test_dataset']}\n",
+ " },\n",
+ " 'validmind.data_validation.TabularDescriptionTables:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset', 'test_dataset']}\n",
+ " },\n",
+ " 'validmind.data_validation.ClassImbalance:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset', 'test_dataset']},\n",
+ " 'params': {'min_percent_threshold': 10}\n",
+ " },\n",
+ " 'validmind.data_validation.UniqueRows:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset', 'test_dataset']},\n",
+ " 'params': {'min_percent_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.TabularNumericalHistograms:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset', 'test_dataset']}\n",
+ " },\n",
+ " 'validmind.data_validation.MutualInformation:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset', 'test_dataset']},\n",
+ " 'params': {'min_threshold': 0.01}\n",
+ " },\n",
+ " 'validmind.data_validation.PearsonCorrelationMatrix:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset', 'test_dataset']}\n",
+ " },\n",
+ " 'validmind.data_validation.HighPearsonCorrelation:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset', 'test_dataset']},\n",
+ " 'params': {'max_threshold': 0.3, 'top_n_correlations': 10}\n",
+ " },\n",
+ " 'validmind.model_validation.ModelMetadata': {\n",
+ " 'input_grid': {'model': ['xgb_model_developer_champion', 'rf_model']}\n",
+ " },\n",
+ " 'validmind.model_validation.sklearn.ModelParameters': {\n",
+ " 'input_grid': {'model': ['xgb_model_developer_champion', 'rf_model']}\n",
+ " },\n",
+ " 'validmind.model_validation.sklearn.ROCCurve': {\n",
+ " 'input_grid': {'dataset': ['train_dataset', 'test_dataset'], 'model': ['xgb_model_developer_champion']}\n",
+ " },\n",
+ " 'validmind.model_validation.sklearn.MinimumROCAUCScore': {\n",
+ " 'input_grid': {'dataset': ['train_dataset', 'test_dataset'], 'model': ['xgb_model_developer_champion']},\n",
+ " 'params': {'min_threshold': 0.5}\n",
+ " }\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Then batch run and log our tests in `test_config`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for t in test_config:\n",
+ " print(t)\n",
+ " try:\n",
+ " # Check if test has input_grid\n",
+ " if 'input_grid' in test_config[t]:\n",
+ " # For tests with input_grid, pass the input_grid configuration\n",
+ " if 'params' in test_config[t]:\n",
+ " vm.tests.run_test(t, input_grid=test_config[t]['input_grid'], params=test_config[t]['params']).log()\n",
+ " else:\n",
+ " vm.tests.run_test(t, input_grid=test_config[t]['input_grid']).log()\n",
+ " else:\n",
+ " # Original logic for regular inputs\n",
+ " if 'params' in test_config[t]:\n",
+ " vm.tests.run_test(t, inputs=test_config[t]['inputs'], params=test_config[t]['params']).log()\n",
+ " else:\n",
+ " vm.tests.run_test(t, inputs=test_config[t]['inputs']).log()\n",
+ " except Exception as e:\n",
+ " print(f\"Error running test {t}: {str(e)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Next steps"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Work with your validation report\n",
+ "\n",
+ "Now that you've logged all your test results and verified the work done by the model development team, head to the ValidMind Platform to wrap up your validation report:\n",
+ "\n",
+ "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n",
+ "\n",
+ "2. In the left sidebar that appears for your model, click **Validation Report**.\n",
+ "\n",
+ "Include your logged test results as evidence, create risk assessment notes, add findings, and assess compliance, then submit your report for review when it's ready. **Learn more:** [Preparing validation reports](https://docs.validmind.ai/guide/model-validation/preparing-validation-reports.html)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Discover more learning resources\n",
+ "\n",
+ "All notebook samples can be found in the following directories of the ValidMind Library GitHub repository:\n",
+ "\n",
+ "- [Code samples](https://github.com/validmind/validmind-library/tree/main/notebooks/code_samples)\n",
+ "- [How-to guides](https://github.com/validmind/validmind-library/tree/main/notebooks/how_to)\n",
+ "\n",
+ "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Upgrade ValidMind\n",
+ "\n",
+ "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n",
+ "\n",
+ "Retrieve the information for the currently installed version of ValidMind:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip show validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n",
+ "\n",
+ "```bash\n",
+ "%pip install --upgrade validmind\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "You may need to restart your kernel after running the upgrade package for changes to be applied."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "ValidMind Library",
+ "language": "python",
+ "name": "validmind"
+ },
+ "language_info": {
+ "name": "python",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/code_samples/model_validation/xgb_model_champion.pkl b/notebooks/code_samples/model_validation/xgb_model_champion.pkl
new file mode 100644
index 000000000..bfe7349b6
Binary files /dev/null and b/notebooks/code_samples/model_validation/xgb_model_champion.pkl differ
diff --git a/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb b/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb
index e76caab88..5fb7b6221 100644
--- a/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb
+++ b/notebooks/code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb
@@ -22,7 +22,7 @@
"If this is your first time trying out ValidMind, we recommend going through the following resources first:\n",
"\n",
"- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work\n",
- "- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
+ "- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
]
},
{
@@ -322,7 +322,7 @@
"\n",
"What you can see now is a more easily consumable version of the prompt validation testing you just performed, along with other parts of your model documentation that still need to be completed.\n",
"\n",
- "If you want to learn more about where you are in the model documentation process, take a look at [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html).\n"
+ "If you want to learn more about where you are in the model documentation process, take a look our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html).\n"
]
},
{
diff --git a/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb
index ef70b9ddd..9ae50dbb9 100644
--- a/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb
+++ b/notebooks/code_samples/nlp_and_llm/foundation_models_summarization_demo.ipynb
@@ -37,7 +37,7 @@
"If this is your first time trying out ValidMind, we recommend going through the following resources first:\n",
"\n",
"- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work\n",
- "- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
+ "- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
]
},
{
@@ -367,7 +367,7 @@
"\n",
"What you can see now is a more easily consumable version of the prompt validation testing you just performed, along with other parts of your model documentation that still need to be completed.\n",
"\n",
- "If you want to learn more about where you are in the model documentation process, take a look at [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html).\n"
+ "If you want to learn more about where you are in the model documentation process, take a look our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html).\n"
]
},
{
diff --git a/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb b/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb
index 0c5d996ad..ee51ab20b 100644
--- a/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb
+++ b/notebooks/code_samples/nlp_and_llm/hugging_face_integration_demo.ipynb
@@ -22,7 +22,7 @@
"If this is your first time trying out ValidMind, we recommend going through the following resources first:\n",
"\n",
"- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work\n",
- "- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
+ "- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
]
},
{
@@ -294,7 +294,7 @@
"\n",
"What you can see now is a more easily consumable version of the prompt validation testing you just performed, along with other parts of your model documentation that still need to be completed.\n",
"\n",
- "If you want to learn more about where you are in the model documentation process, take a look at [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html).\n"
+ "If you want to learn more about where you are in the model documentation process, take a look our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html).\n"
]
},
{
diff --git a/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb
index f4042e960..c68e562d7 100644
--- a/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb
+++ b/notebooks/code_samples/nlp_and_llm/hugging_face_summarization_demo.ipynb
@@ -23,7 +23,7 @@
"If this is your first time trying out ValidMind, we recommend going through the following resources first:\n",
"\n",
"- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work\n",
- "- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
+ "- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
]
},
{
@@ -326,7 +326,7 @@
"\n",
"What you can see now is a more easily consumable version of the prompt validation testing you just performed, along with other parts of your model documentation that still need to be completed.\n",
"\n",
- "If you want to learn more about where you are in the model documentation process, take a look at [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html).\n"
+ "If you want to learn more about where you are in the model documentation process, take a look our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html).\n"
]
},
{
diff --git a/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb b/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb
index 8fef07bd3..e7ac59684 100644
--- a/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb
+++ b/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.ipynb
@@ -45,7 +45,7 @@
"If this is your first time trying out ValidMind, you can make use of the following resources alongside this notebook:\n",
"\n",
"- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work\n",
- "- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
+ "- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
]
},
{
@@ -814,7 +814,7 @@
"\n",
"What you can see now is a more easily consumable version of the prompt validation testing you just performed, along with other parts of your model documentation that still need to be completed.\n",
"\n",
- "If you want to learn more about where you are in the model documentation process, take a look at [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html)."
+ "If you want to learn more about where you are in the model documentation process, take a look at our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html)."
]
},
{
diff --git a/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb b/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb
index b32845f16..055028eca 100644
--- a/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb
+++ b/notebooks/code_samples/nlp_and_llm/prompt_validation_demo.ipynb
@@ -66,7 +66,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb b/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb
new file mode 100644
index 000000000..329092a4b
--- /dev/null
+++ b/notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb
@@ -0,0 +1,1635 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# RAG Model Benchmarking Demo\n",
+ "\n",
+ "In this notebook, we are going to implement a simple RAG Model for automating the process of answering RFP questions using GenAI. We will see how we can initialize an embedding model, a retrieval model and a generator model with LangChain components and use them within the ValidMind Library to run tests against them. We'll demonstrate how to set up multiple models for benchmarking at each stage of the RAG pipeline - specifically two embedding models, two retrieval models with different parameters, and two LLM models (GPT-3.5 and GPT-4o) - allowing for comparison of performance across different configurations. Finally, we will see how we can put them together in a Pipeline and run that to get e2e results and run tests against that."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## About ValidMind\n",
+ "\n",
+ "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n",
+ "\n",
+ "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n",
+ "\n",
+ "\n",
+ "\n",
+ "### Before you begin\n",
+ "\n",
+ "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n",
+ "\n",
+ "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n",
+ "\n",
+ "\n",
+ "\n",
+ "### New to ValidMind?\n",
+ "\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
+ "\n",
+ "
For access to all features available in this notebook, create a free ValidMind account.\n",
+ " \n",
+ "Signing up is FREE — Register with ValidMind
\n",
+ "\n",
+ "\n",
+ "\n",
+ "### Key concepts\n",
+ "\n",
+ "- **FunctionModels**: ValidMind offers support for creating `VMModel` instances from Python functions. This enables us to support any \"model\" by simply using the provided function as the model's `predict` method.\n",
+ "- **PipelineModels**: ValidMind models (`VMModel` instances) of any type can be piped together to create a model pipeline. This allows model components to be created and tested/documented independently, and then combined into a single model for end-to-end testing and documentation. We use the `|` operator to pipe models together.\n",
+ "- **RAG**: RAG stands for Retrieval Augmented Generation and refers to a wide range of GenAI applications where some form of retrieval is used to add context to the prompt so that the LLM that generates content can refer to it when creating its output. In this notebook, we are going to implement a simple RAG setup using LangChain components.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Prerequisites\n",
+ "\n",
+ "Let's go ahead and install the `validmind` library if its not already installed... Then we can install the `qdrant-client` library for our vector store and `langchain` for everything else:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip install -q \"validmind[llm]\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip install -q qdrant-client langchain langchain-openai sentencepiece"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Initialize the ValidMind Library\n",
+ "\n",
+ "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n",
+ "\n",
+ "### Get your code snippet\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Model Inventory** and click **+ Register Model**.\n",
+ "\n",
+ "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n",
+ "\n",
+ " For example, to register a model for use with this notebook, select:\n",
+ "\n",
+ " - Documentation template: `Gen AI RAG Template`\n",
+ " - Use case: `Marketing/Sales - Analytics`\n",
+ "\n",
+ " You can fill in other options according to your preference.\n",
+ "\n",
+ "4. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
+ "\n",
+ "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load your model identifier credentials from an `.env` file\n",
+ "\n",
+ "%load_ext dotenv\n",
+ "%dotenv .env\n",
+ "\n",
+ "# Or replace with your code snippet\n",
+ "\n",
+ "import validmind as vm\n",
+ "\n",
+ "vm.init(\n",
+ " api_host = \"https://api.prod.validmind.ai/api/v1/tracking\",\n",
+ " api_key = \"...\",\n",
+ " api_secret = \"...\",\n",
+ " model = \"...\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Read Open AI API Key\n",
+ "\n",
+ "We will need to have an OpenAI API key to be able to use their `text-embedding-3-small` and `text-embedding-3-large` models for our embeddings, `gpt-3.5-turbo` and `gpt-4o` models for our generator and `gpt-4o` model for our LLM-as-Judge tests. If you don't have an OpenAI API key, you can get one by signing up at [OpenAI](https://platform.openai.com/signup). Then you can create a `.env` file in the root of your project and the following cell will load it from there. Alternatively, you can just uncomment the line below to directly set the key (not recommended for security reasons)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# load openai api key\n",
+ "import os\n",
+ "\n",
+ "import dotenv\n",
+ "import nltk\n",
+ "\n",
+ "dotenv.load_dotenv()\n",
+ "nltk.download('stopwords')\n",
+ "nltk.download('punkt_tab')\n",
+ "\n",
+ "# os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n",
+ "\n",
+ "if not \"OPENAI_API_KEY\" in os.environ:\n",
+ " raise ValueError(\"OPENAI_API_KEY is not set\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Dataset Loader\n",
+ "\n",
+ "Great, now that we have all of our dependencies installed, the ValidMind Library initialized and connected to our model and our OpenAI API key setup, we can go ahead and load our datasets. We will use the synthetic `RFP` dataset included with ValidMind for this notebook. This dataset contains a variety of RFP questions and ground truth answers that we can use both as the source where our Retriever will search for similar question-answer pairs as well as our test set for evaluating the performance of our RAG model. To do this, we just have to load it and call the preprocess function to get a split of the data into train and test sets."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Import the sample dataset from the library\n",
+ "from validmind.datasets.llm.rag import rfp\n",
+ "\n",
+ "raw_df = rfp.load_data()\n",
+ "train_df, test_df = rfp.preprocess(raw_df)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_train_ds = vm.init_dataset(\n",
+ " train_df,\n",
+ " text_column=\"question\",\n",
+ " target_column=\"ground_truth\",\n",
+ ")\n",
+ "\n",
+ "vm_test_ds = vm.init_dataset(\n",
+ " test_df,\n",
+ " text_column=\"question\",\n",
+ " target_column=\"ground_truth\",\n",
+ ")\n",
+ "\n",
+ "vm_test_ds.df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Data validation\n",
+ "\n",
+ "Now that we have loaded our dataset, we can go ahead and run some data validation tests right away to start assessing and documenting the quality of our data. Since we are using a text dataset, we can use ValidMind's built-in array of text data quality tests to check that things like number of duplicates, missing values, and other common text data issues are not present in our dataset. We can also run some tests to check the sentiment and toxicity of our data."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Duplicates\n",
+ "\n",
+ "First, let's check for duplicates in our dataset. We can use the `validmind.data_validation.Duplicates` test and pass our dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from validmind.tests import run_test\n",
+ "\n",
+ "run_test(\n",
+ " test_id=\"validmind.data_validation.Duplicates\",\n",
+ " inputs={\"dataset\": vm_train_ds},\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Stop Words\n",
+ "\n",
+ "Next, let's check for stop words in our dataset. We can use the `validmind.data_validation.StopWords` test and pass our dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " test_id=\"validmind.data_validation.nlp.StopWords\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_train_ds,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Punctuations\n",
+ "\n",
+ "Next, let's check for punctuations in our dataset. We can use the `validmind.data_validation.Punctuations` test:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " test_id=\"validmind.data_validation.nlp.Punctuations\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_train_ds,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Common Words\n",
+ "\n",
+ "Next, let's check for common words in our dataset. We can use the `validmind.data_validation.CommonWord` test:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " test_id=\"validmind.data_validation.nlp.CommonWords\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_train_ds,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Language Detection\n",
+ "\n",
+ "For documentation purposes, we can detect and log the languages used in the dataset with the `validmind.data_validation.LanguageDetection` test:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " test_id=\"validmind.data_validation.nlp.LanguageDetection\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_train_ds,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Toxicity Score\n",
+ "\n",
+ "Now, let's go ahead and run the `validmind.data_validation.nlp.Toxicity` test to compute a toxicity score for our dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.data_validation.nlp.Toxicity\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_train_ds,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Polarity and Subjectivity\n",
+ "\n",
+ "We can also run the `validmind.data_validation.nlp.PolarityAndSubjectivity` test to compute the polarity and subjectivity of our dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.data_validation.nlp.PolarityAndSubjectivity\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_train_ds,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Sentiment\n",
+ "\n",
+ "Finally, we can run the `validmind.data_validation.nlp.Sentiment` test to plot the sentiment of our dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.data_validation.nlp.Sentiment\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_train_ds,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Embedding Model\n",
+ "\n",
+ "Now that we have our dataset loaded and have run some data validation tests to assess and document the quality of our data, we can go ahead and initialize our embedding model. We will use `text-embedding-3-small` and `text-embedding-3-large` models from OpenAI for this purpose wrapped in the `OpenAIEmbeddings` class from LangChain. This model will be used to \"embed\" our questions both for inserting the question-answer pairs from the \"train\" set into the vector store and for embedding the question from inputs when making predictions with our RAG model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from langchain_openai import OpenAIEmbeddings\n",
+ "\n",
+ "embedding_small_client = OpenAIEmbeddings(model=\"text-embedding-3-small\")\n",
+ "\n",
+ "\n",
+ "def embed_small(input):\n",
+ " \"\"\"Returns a text embedding for the given text\"\"\"\n",
+ " return embedding_small_client.embed_query(input[\"question\"])\n",
+ "\n",
+ "\n",
+ "vm_embedder_small = vm.init_model(input_id=\"embedding_small_model\", predict_fn=embed_small)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "embedding_large_client = OpenAIEmbeddings(model=\"text-embedding-3-large\")\n",
+ "\n",
+ "\n",
+ "def embed_large(input):\n",
+ " \"\"\"Returns a text embedding for the given text\"\"\"\n",
+ " return embedding_large_client.embed_query(input[\"question\"])\n",
+ "\n",
+ "\n",
+ "vm_embedder_large = vm.init_model(input_id=\"embedding_large_model\", predict_fn=embed_large)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "What we have done here is to initialize the `OpenAIEmbeddings` class so it uses OpenAI's `text-embedding-3-small` and `text-embedding-3-large` models. We then created an `embed` function that takes in an `input` dictionary and uses the `embed_query` method of the embedding client to compute the embeddings of the `question`. We use an `embed` function since that is how ValidMind supports any custom model. We will use this strategy for the retrieval and generator models as well but you could also use, say, a HuggingFace model directly. See the documentation for more information on which model types are directly supported - [ValidMind Documentation](https://docs.validmind.ai/validmind/validmind.html)... Finally, we use the `init_model` function from the ValidMind Library to create a `VMModel` object that can be used in ValidMind tests. This also logs the model to our model documentation and any test that uses the model will be linked to the logged model and its metadata."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Assign Predictions\n",
+ "\n",
+ "To precompute the embeddings for our test set, we can call the `assign_predictions` method of our `vm_test_ds` object we created above. This will compute the embeddings for each question in the test set and store them in the a special prediction column of the test set thats linked to our `vm_embedder` model. This will allow us to use these embeddings later when we run tests against our embedding model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_test_ds.assign_predictions(vm_embedder_small)\n",
+ "vm_test_ds.assign_predictions(vm_embedder_large)\n",
+ "print(vm_test_ds)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Run tests\n",
+ "\n",
+ "Now that everything is setup for the embedding model, we can go ahead and run some tests to assess and document the quality of our embeddings. We will use the `validmind.model_validation.embeddings.*` tests to compute a variety of metrics against our model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.embeddings.StabilityAnalysisRandomNoise\",\n",
+ " input_grid={\n",
+ " \"model\": [vm_embedder_small, vm_embedder_large],\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " },\n",
+ " params={\n",
+ " \"probability\": 0.3,\n",
+ " \"mean_similarity_threshold\": 0.7,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.embeddings.StabilityAnalysisSynonyms\",\n",
+ " input_grid={\n",
+ " \"model\": [vm_embedder_small, vm_embedder_large],\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " },\n",
+ " params={\n",
+ " \"probability\": 0.3,\n",
+ " \"mean_similarity_threshold\": 0.7,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.embeddings.StabilityAnalysisTranslation\",\n",
+ " input_grid={\n",
+ " \"model\": [vm_embedder_small, vm_embedder_large],\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " },\n",
+ " params={\n",
+ " \"source_lang\": \"en\",\n",
+ " \"target_lang\": \"fr\",\n",
+ " \"mean_similarity_threshold\": 0.7,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.embeddings.CosineSimilarityHeatmap\",\n",
+ " input_grid={\n",
+ " \"model\": [vm_embedder_small, vm_embedder_large],\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.embeddings.CosineSimilarityDistribution\",\n",
+ " input_grid={\n",
+ " \"model\": [vm_embedder_small, vm_embedder_large],\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.embeddings.PCAComponentsPairwisePlots\",\n",
+ " input_grid={\n",
+ " \"model\": [vm_embedder_small, vm_embedder_large],\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " },\n",
+ " params={\n",
+ " \"n_components\": 3,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Setup Vector Store\n",
+ "\n",
+ "Great, so now that we have assessed our embedding model and verified that it is performing well, we can go ahead and use it to compute embeddings for our question-answer pairs in the \"train\" set. We will then use these embeddings to insert the question-answer pairs into a vector store. We will use an in-memory `qdrant` vector database for demo purposes but any option would work just as well here. We will use the `QdrantClient` class from LangChain to interact with the vector store. This class will allow us to insert and search for embeddings in the vector store."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Generate embeddings for the Train Set\n",
+ "\n",
+ "We can use the same `assign_predictions` method from earlier except this time we will use the `vm_train_ds` object to compute the embeddings for the question-answer pairs in the \"train\" set."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_train_ds.assign_predictions(vm_embedder_small)\n",
+ "print(vm_train_ds)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Insert embeddings and questions into Vector DB\n",
+ "\n",
+ "Now that we have computed the embeddings for our question-answer pairs in the \"train\" set, we can go ahead and insert them into the vector store:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from langchain_community.vectorstores import Qdrant\n",
+ "from langchain_community.document_loaders import DataFrameLoader\n",
+ "\n",
+ "# load documents from dataframe\n",
+ "loader = DataFrameLoader(train_df, page_content_column=\"question\")\n",
+ "docs = loader.load()\n",
+ "\n",
+ "# setup vector datastore\n",
+ "qdrant = Qdrant.from_documents(\n",
+ " docs,\n",
+ " embedding_small_client,\n",
+ " location=\":memory:\", # Local mode with in-memory storage only\n",
+ " collection_name=\"rfp_rag_collection\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Retrieval Model\n",
+ "\n",
+ "Now that we have an embedding model and a vector database setup and loaded with our data, we need a Retrieval model that can search for similar question-answer pairs for a given input question. Once created, we can initialize this as a ValidMind model and `assign_predictions` to it just like our embedding model. In this example, we'll create two retrieval models with different `k` parameters (the number of documents retrieved) to benchmark and compare their performance. This approach allows us to evaluate how retrieval depth affects the overall system quality."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def retrieve(input):\n",
+ " contexts = []\n",
+ "\n",
+ " for result in qdrant.similarity_search_with_score(input[\"question\"], k=5):\n",
+ " document, score = result\n",
+ " context = f\"Q: {document.page_content}\\n\"\n",
+ " context += f\"A: {document.metadata['ground_truth']}\\n\"\n",
+ "\n",
+ " contexts.append(context)\n",
+ "\n",
+ " return contexts\n",
+ "\n",
+ "\n",
+ "vm_retriever_k5 = vm.init_model(input_id=\"retrieval_k5_model\", predict_fn=retrieve)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def retrieve(input):\n",
+ " contexts = []\n",
+ "\n",
+ " for result in qdrant.similarity_search_with_score(input[\"question\"], k=10):\n",
+ " document, score = result\n",
+ " context = f\"Q: {document.page_content}\\n\"\n",
+ " context += f\"A: {document.metadata['ground_truth']}\\n\"\n",
+ "\n",
+ " contexts.append(context)\n",
+ "\n",
+ " return contexts\n",
+ "\n",
+ "\n",
+ "vm_retriever_k10 = vm.init_model(input_id=\"retrieval_k10_model\", predict_fn=retrieve)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_test_ds.assign_predictions(model=vm_retriever_k5)\n",
+ "vm_test_ds.assign_predictions(model=vm_retriever_k10)\n",
+ "print(vm_test_ds)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_test_ds._df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Generation Model\n",
+ "\n",
+ "As the final piece of this simple RAG pipeline, we can create and initialize a generation model that will use the retrieved context to generate an answer to the input question. We will use the `gpt-3.5-turbo` and `gpt-4o` models from OpenAI. Since we have two retrieval models (with different `k` values) and want to test two different LLMs, we'll create a total of four generator models - pairing each retrieval configuration with each LLM to comprehensively evaluate how both retrieval depth and model capability affect response quality."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from openai import OpenAI\n",
+ "\n",
+ "from validmind.models import Prompt\n",
+ "\n",
+ "\n",
+ "system_prompt = \"\"\"\n",
+ "You are an expert RFP AI assistant.\n",
+ "You are tasked with answering new RFP questions based on existing RFP questions and answers.\n",
+ "You will be provided with the existing RFP questions and answer pairs that are the most relevant to the new RFP question.\n",
+ "After that you will be provided with a new RFP question.\n",
+ "You will generate an answer and respond only with the answer.\n",
+ "Ignore your pre-existing knowledge and answer the question based on the provided context.\n",
+ "\"\"\".strip()\n",
+ "\n",
+ "openai_client = OpenAI()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def generate(input):\n",
+ " \n",
+ " response = openai_client.chat.completions.create(\n",
+ " model=\"gpt-3.5-turbo\",\n",
+ " messages=[\n",
+ " {\"role\": \"system\", \"content\": system_prompt},\n",
+ " {\"role\": \"user\", \"content\": \"\\n\\n\".join(input[\"retrieval_k5_model\"])},\n",
+ " {\"role\": \"user\", \"content\": input[\"question\"]},\n",
+ " ],\n",
+ " )\n",
+ " \n",
+ " return response.choices[0].message.content\n",
+ "\n",
+ "\n",
+ "vm_generator_k5_gpt35 = vm.init_model(\n",
+ " input_id=\"generation_k5_gpt35_model\",\n",
+ " predict_fn=generate,\n",
+ " prompt=Prompt(template=system_prompt),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def generate(input):\n",
+ " response = openai_client.chat.completions.create(\n",
+ " model=\"gpt-3.5-turbo\",\n",
+ " messages=[\n",
+ " {\"role\": \"system\", \"content\": system_prompt},\n",
+ " {\"role\": \"user\", \"content\": \"\\n\\n\".join(input[\"retrieval_k10_model\"])},\n",
+ " {\"role\": \"user\", \"content\": input[\"question\"]},\n",
+ " ],\n",
+ " )\n",
+ "\n",
+ " return response.choices[0].message.content\n",
+ "\n",
+ "\n",
+ "vm_generator_k10_gpt35 = vm.init_model(\n",
+ " input_id=\"generation_k10_gpt35_model\",\n",
+ " predict_fn=generate,\n",
+ " prompt=Prompt(template=system_prompt),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def generate(input):\n",
+ " \n",
+ " response = openai_client.chat.completions.create(\n",
+ " model=\"gpt-4o\",\n",
+ " messages=[\n",
+ " {\"role\": \"system\", \"content\": system_prompt},\n",
+ " {\"role\": \"user\", \"content\": \"\\n\\n\".join(input[\"retrieval_k5_model\"])},\n",
+ " {\"role\": \"user\", \"content\": input[\"question\"]},\n",
+ " ],\n",
+ " )\n",
+ " \n",
+ " return response.choices[0].message.content\n",
+ "\n",
+ "\n",
+ "vm_generator_k5_gpt4o = vm.init_model(\n",
+ " input_id=\"generation_k5_gpt4o_model\",\n",
+ " predict_fn=generate,\n",
+ " prompt=Prompt(template=system_prompt),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def generate(input):\n",
+ " response = openai_client.chat.completions.create(\n",
+ " model=\"gpt-4o\",\n",
+ " messages=[\n",
+ " {\"role\": \"system\", \"content\": system_prompt},\n",
+ " {\"role\": \"user\", \"content\": \"\\n\\n\".join(input[\"retrieval_k10_model\"])},\n",
+ " {\"role\": \"user\", \"content\": input[\"question\"]},\n",
+ " ],\n",
+ " )\n",
+ "\n",
+ " return response.choices[0].message.content\n",
+ "\n",
+ "\n",
+ "vm_generator_k10_gpt4o = vm.init_model(\n",
+ " input_id=\"generation_k10_gpt4o_model\",\n",
+ " predict_fn=generate,\n",
+ " prompt=Prompt(template=system_prompt),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's test it out real quick:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "vm_generator_k5_gpt35.predict(\n",
+ " pd.DataFrame(\n",
+ " {\"retrieval_k5_model\": [[\"My name is anil\"]], \"question\": [\"what is my name\"]}\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_generator_k5_gpt4o.predict(\n",
+ " pd.DataFrame(\n",
+ " {\"retrieval_k5_model\": [[\"My name is anil\"]], \"question\": [\"what is my name\"]}\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Prompt Evaluation\n",
+ "\n",
+ "Now that we have our generator model initialized, we can run some LLM-as-Judge tests to evaluate the system prompt. This will allow us to get an initial sense of how well the prompt meets a few best practices for prompt engineering. These tests use an LLM to rate the prompt on a scale of 1-10 against the following criteria:\n",
+ "\n",
+ "- **Examplar Bias**: When using multi-shot prompting, does the prompt contain an unbiased distribution of examples?\n",
+ "- **Delimitation**: When using complex prompts containing examples, contextual information, or other elements, is the prompt formatted in such a way that each element is clearly separated?\n",
+ "- **Clarity**: How clearly the prompt states the task.\n",
+ "- **Conciseness**: How succinctly the prompt states the task.\n",
+ "- **Instruction Framing**: Whether the prompt contains negative instructions.\n",
+ "- **Specificity**: How specific the prompt defines the task."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.prompt_validation.Bias\",\n",
+ " inputs={\n",
+ " \"model\": vm_generator_k5_gpt4o,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.prompt_validation.Clarity\",\n",
+ " inputs={\n",
+ " \"model\": vm_generator_k5_gpt4o,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.prompt_validation.Conciseness\",\n",
+ " inputs={\n",
+ " \"model\": vm_generator_k5_gpt4o,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.prompt_validation.Delimitation\",\n",
+ " inputs={\n",
+ " \"model\": vm_generator_k5_gpt4o,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.prompt_validation.NegativeInstruction\",\n",
+ " inputs={\n",
+ " \"model\": vm_generator_k5_gpt4o,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.prompt_validation.Specificity\",\n",
+ " inputs={\n",
+ " \"model\": vm_generator_k5_gpt4o,\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Setup RAG Pipeline Model\n",
+ "\n",
+ "Now that we have all of our individual \"component\" models setup and initialized we need some way to put them all together in a single \"pipeline\". We can use the `PipelineModel` class to do this. This ValidMind model type simply wraps any number of other ValidMind models and runs them in sequence. We can use a pipe(`|`) operator - in Python this is normally an `or` operator but we have overloaded it for easy pipeline creation - to chain together our models. We can then initialize this pipeline model and assign predictions to it just like any other model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_rag_k5_gpt35_model = vm.init_model(vm_retriever_k5 | vm_generator_k5_gpt35, input_id=\"rag_k5_gpt35_model\")\n",
+ "vm_rag_k10_gpt35_model = vm.init_model(vm_retriever_k10 | vm_generator_k10_gpt35, input_id=\"rag_k10_gpt35_model\")\n",
+ "vm_rag_k5_gpt4o_model = vm.init_model(vm_retriever_k5 | vm_generator_k5_gpt4o, input_id=\"rag_k5_gpt4o_model\")\n",
+ "vm_rag_k10_gpt4o_model = vm.init_model(vm_retriever_k10 | vm_generator_k10_gpt4o, input_id=\"rag_k10_gpt4o_model\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We can `assign_predictions` to the pipeline model just like we did with the individual models. This will run the pipeline on the test set and store the results in the test set for later use."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_test_ds.assign_predictions(model=vm_rag_k5_gpt35_model)\n",
+ "vm_test_ds.assign_predictions(model=vm_rag_k10_gpt35_model)\n",
+ "vm_test_ds.assign_predictions(model=vm_rag_k5_gpt4o_model)\n",
+ "vm_test_ds.assign_predictions(model=vm_rag_k10_gpt4o_model)\n",
+ "print(vm_test_ds)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_test_ds._df.head(5)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Run tests\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## RAGAS evaluation\n",
+ "\n",
+ "Let's go ahead and run some of our new RAG tests against our model...\n",
+ "\n",
+ "> Note: these tests are still being developed and are not yet in a stable state. We are using advanced tests here that use LLM-as-Judge and other strategies to assess things like the relevancy of the retrieved context to the input question and the correctness of the generated answer when compared to the ground truth. There is more to come in this area so stay tuned!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 46,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import warnings\n",
+ "\n",
+ "warnings.filterwarnings(\"ignore\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Semantic Similarity\n",
+ "\n",
+ "The concept of Answer Semantic Similarity pertains to the assessment of the semantic resemblance between the generated answer and the ground truth. This evaluation is based on the ground truth and the answer, with values falling within the range of 0 to 1. A higher score signifies a better alignment between the generated answer and the ground truth.\n",
+ "\n",
+ "Measuring the semantic similarity between answers can offer valuable insights into the quality of the generated response. This evaluation utilizes a cross-encoder model to calculate the semantic similarity score."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.SemanticSimilarity\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid={\n",
+ " \"response_column\": [\"rag_k5_gpt35_model_prediction\", \"rag_k5_gpt4o_model_prediction\"],\n",
+ " \"reference_column\": [\"ground_truth\"],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Context Entity Recall\n",
+ "\n",
+ "This test gives the measure of recall of the retrieved context, based on the number of entities present in both ground_truths and contexts relative to the number of entities present in the ground_truths alone. Simply put, it is a measure of what fraction of entities are recalled from ground_truths. This test is useful in fact-based use cases like tourism help desk, historical QA, etc. This test can help evaluate the retrieval mechanism for entities, based on comparison with entities present in ground_truths, because in cases where entities matter, we need the contexts which cover them."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.ContextEntityRecall\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid={\n",
+ " \"reference_column\": [\"ground_truth\"],\n",
+ " \"retrieved_contexts_column\": [\"retrieval_k5_model_prediction\", \"retrieval_k10_model_prediction\"],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Context Precision\n",
+ "\n",
+ "Context Precision is a test that evaluates whether all of the ground-truth relevant items present in the contexts are ranked higher or not. Ideally all the relevant chunks must appear at the top ranks. This test is computed using the question, ground_truth and the contexts, with values ranging between 0 and 1, where higher scores indicate better precision."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.ContextPrecision\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid={\n",
+ " \"user_input_column\": [\"question\"],\n",
+ " \"retrieved_contexts_column\": [\"retrieval_k5_model_prediction\", \"retrieval_k10_model_prediction\"],\n",
+ " \"reference_column\": [\"ground_truth\"],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Context Precision Without Reference\n",
+ "\n",
+ "This test evaluates whether retrieved contexts align well with the expected response for a given user input, without requiring a ground-truth reference. This test assesses the relevance of each retrieved context chunk by comparing it directly to the response."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.ContextPrecisionWithoutReference\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid=[\n",
+ " {\"user_input_column\": \"question\",\n",
+ " \"retrieved_contexts_column\": \"retrieval_k5_model_prediction\",\n",
+ " \"response_column\": \"rag_k5_gpt4o_model_prediction\"\n",
+ " },\n",
+ " {\"user_input_column\": \"question\",\n",
+ " \"retrieved_contexts_column\": \"retrieval_k10_model_prediction\",\n",
+ " \"response_column\": \"rag_k10_gpt4o_model_prediction\"\n",
+ " },\n",
+ " ],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.ContextPrecisionWithoutReference\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid={\n",
+ " \"user_input_column\": [\"question\"],\n",
+ " \"retrieved_contexts_column\": [\"retrieval_k5_model_prediction\"],\n",
+ " \"response_column\": [\"rag_k5_gpt35_model_prediction\", \"rag_k5_gpt4o_model_prediction\"],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Faithfulness\n",
+ "\n",
+ "This measures the factual consistency of the generated answer against the given context. It is calculated from answer and retrieved context. The answer is scaled to (0,1) range. Higher the better.\n",
+ "\n",
+ "The generated answer is regarded as faithful if all the claims that are made in the answer can be inferred from the given context. To calculate this a set of claims from the generated answer is first identified. Then each one of these claims are cross checked with given context to determine if it can be inferred from given context or not."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.Faithfulness\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid={\n",
+ " \"user_input_column\": [\"question\"],\n",
+ " \"response_column\": [\"rag_k5_gpt35_model_prediction\", \"rag_k5_gpt4o_model_prediction\"],\n",
+ " \"retrieved_contexts_column\": [\"retrieval_k5_model_prediction\"],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Response Relevancy\n",
+ "\n",
+ "The Response Relevancy test, focuses on assessing how pertinent the generated answer is to the given prompt. A lower score is assigned to answers that are incomplete or contain redundant information and higher scores indicate better relevancy. This test is computed using the question, the context and the answer.\n",
+ "\n",
+ "The Response Relevancy is defined as the mean cosine similartiy of the original question to a number of artifical questions, which where generated (reverse engineered) based on the answer.\n",
+ "\n",
+ "Please note, that eventhough in practice the score will range between 0 and 1 most of the time, this is not mathematically guranteed, due to the nature of the cosine similarity ranging from -1 to 1.\n",
+ "\n",
+ "> Note: This is a reference free test. If you’re looking to compare ground truth answer with generated answer refer to Answer Correctness.\n",
+ "\n",
+ "An answer is deemed relevant when it directly and appropriately addresses the original question. Importantly, our assessment of answer relevance does not consider factuality but instead penalizes cases where the answer lacks completeness or contains redundant details. To calculate this score, the LLM is prompted to generate an appropriate question for the generated answer multiple times, and the mean cosine similarity between these generated questions and the original question is measured. The underlying idea is that if the generated answer accurately addresses the initial question, the LLM should be able to generate questions from the answer that align with the original question."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.ResponseRelevancy\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid={\n",
+ " \"user_input_column\": [\"question\"],\n",
+ " \"response_column\": [\"rag_k5_gpt35_model_prediction\", \"rag_k5_gpt4o_model_prediction\"],\n",
+ " \"retrieved_contexts_column\": [\"retrieval_k5_model_prediction\"],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Context Recall\n",
+ "\n",
+ "Context recall measures the extent to which the retrieved context aligns with the annotated answer, treated as the ground truth. It is computed based on the ground truth and the retrieved context, and the values range between 0 and 1, with higher values indicating better performance.\n",
+ "\n",
+ "To estimate context recall from the ground truth answer, each sentence in the ground truth answer is analyzed to determine whether it can be attributed to the retrieved context or not. In an ideal scenario, all sentences in the ground truth answer should be attributable to the retrieved context."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.ContextRecall\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid={\n",
+ " \"user_input_column\": [\"question\"],\n",
+ " \"retrieved_contexts_column\": [\"retrieval_k5_model_prediction\", \"retrieval_k10_model_prediction\"],\n",
+ " \"reference_column\": [\"ground_truth\"],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Answer Correctness\n",
+ "\n",
+ "The assessment of Answer Correctness involves gauging the accuracy of the generated answer when compared to the ground truth. This evaluation relies on the ground truth and the answer, with scores ranging from 0 to 1. A higher score indicates a closer alignment between the generated answer and the ground truth, signifying better correctness.\n",
+ "\n",
+ "Answer correctness encompasses two critical aspects: semantic similarity between the generated answer and the ground truth, as well as factual similarity. These aspects are combined using a weighted scheme to formulate the answer correctness score.\n",
+ "\n",
+ "Factual correctness quantifies the factual overlap between the generated answer and the ground truth answer. This is done using the concepts of:\n",
+ "\n",
+ "- TP (True Positive): Facts or statements that are present in both the ground truth and the generated answer.\n",
+ "- FP (False Positive): Facts or statements that are present in the generated answer but not in the ground truth.\n",
+ "- FN (False Negative): Facts or statements that are present in the ground truth but not in the generated answer."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.AnswerCorrectness\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid={\n",
+ " \"user_input_column\": [\"question\"],\n",
+ " \"response_column\": [\"rag_k5_gpt35_model_prediction\", \"rag_k5_gpt4o_model_prediction\"],\n",
+ " \"reference_column\": [\"ground_truth\"],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Aspect Critic\n",
+ "\n",
+ "This is designed to assess submissions based on predefined aspects such as harmlessness and correctness. Additionally, users have the flexibility to define their own aspects for evaluating submissions according to their specific criteria. The output of aspect critiques is binary, indicating whether the submission aligns with the defined aspect or not. This evaluation is performed using the ‘answer’ as input.\n",
+ "\n",
+ "Critiques within the LLM evaluators evaluate submissions based on the provided aspect. Ragas Critiques offers a range of predefined aspects like correctness, harmfulness, etc. Users can also define their own aspects for evaluating submissions based on their specific criteria. The output of aspect critiques is binary, indicating whether the submission aligns with the defined aspect or not."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.AspectCritic\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid={\n",
+ " \"user_input_column\": [\"question\"],\n",
+ " \"response_column\": [\"rag_k5_gpt35_model_prediction\", \"rag_k5_gpt4o_model_prediction\"],\n",
+ " \"retrieved_contexts_column\": [\"retrieval_k5_model_prediction\"],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Noise Sensitivity\n",
+ "\n",
+ "This test is designed to evaluate the robustness of the RAG pipeline model against noise in the retrieved context. It works by checking how well the \"claims\" in the generated answer match up with the \"claims\" in the ground truth answer. If the generated answer contains \"claims\" from the contexts that the ground truth answer does not contain, those claims are considered incorrect. The score for each answer is the number of incorrect claims divided by the total number of claims. This *can* be interpreted as a measure of how sensitive the LLM is to \"noise\" in the context where \"noise\" is information that is relevant but should not be included in the answer since the ground truth answer does not contain it."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ragas.NoiseSensitivity\",\n",
+ " inputs={\"dataset\": vm_test_ds},\n",
+ " param_grid={\n",
+ " \"user_input_column\": [\"question\"],\n",
+ " \"response_column\": [\"rag_k5_gpt35_model_prediction\", \"rag_k5_gpt4o_model_prediction\"],\n",
+ " \"reference_column\": [\"ground_truth\"],\n",
+ " \"retrieved_contexts_column\": [\"retrieval_k5_model_prediction\"],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Generation quality\n",
+ "\n",
+ "In this section, we evaluate the alignment and relevance of generated responses to reference outputs within our retrieval-augmented generation (RAG) application. We use metrics that assess various quality dimensions of the generated responses, including semantic similarity, structural alignment, and phrasing overlap. Semantic similarity metrics compare embeddings of generated and reference text to capture deeper contextual alignment, while overlap and alignment measures quantify how well the phrasing and structure of generated responses match the intended outputs."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Token Disparity\n",
+ "\n",
+ "This test assesses the difference in token counts between the reference texts (ground truth) and the answers generated by the RAG model. It helps evaluate how well the model's outputs align with the expected length and level of detail in the reference texts. A significant disparity in token counts could signal issues with generation quality, such as excessive verbosity or insufficient detail. Consistently low token counts in generated answers compared to references might suggest that the model’s outputs are incomplete or overly concise, missing important contextual information."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.TokenDisparity\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\": [vm_rag_k5_gpt35_model, vm_rag_k5_gpt4o_model],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### ROUGE Score\n",
+ "\n",
+ "This test evaluates the quality of answers generated by the RAG model by measuring overlaps in n-grams, word sequences, and word pairs between the model output and the reference (ground truth) text. ROUGE, short for Recall-Oriented Understudy for Gisting Evaluation, assesses both precision and recall, providing a balanced view of how well the generated response captures the reference content. ROUGE precision measures the proportion of n-grams in the generated text that match the reference, highlighting relevance and conciseness, while ROUGE recall assesses the proportion of reference n-grams present in the generated text, indicating completeness and thoroughness. \n",
+ "\n",
+ "Low precision scores might reveal that the generated text includes redundant or irrelevant information, while low recall scores suggest omissions of essential details from the reference. Consistently low ROUGE scores could indicate poor overall alignment with the ground truth, suggesting the model may be missing key content or failing to capture the intended meaning."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.RougeScore\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\": [vm_rag_k5_gpt35_model, vm_rag_k5_gpt4o_model],\n",
+ " },\n",
+ " params={\n",
+ " \"metric\": \"rouge-1\",\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### BLEU Score\n",
+ "\n",
+ "The BLEU Score test evaluates the quality of answers generated by the RAG model by measuring n-gram overlap between the generated text and the reference (ground truth) text, with a specific focus on exact precision in phrasing. While ROUGE precision also assesses overlap, BLEU differs in two main ways: first, it applies a geometric average across multiple n-gram levels, capturing precise phrase alignment, and second, it includes a brevity penalty to prevent overly short outputs from inflating scores artificially. This added precision focus is valuable in RAG applications where strict adherence to reference language is essential, as BLEU emphasizes the match to exact phrasing. In contrast, ROUGE precision evaluates general content overlap without penalizing brevity, offering a broader sense of content alignment."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.BleuScore\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\": [vm_rag_k5_gpt35_model, vm_rag_k5_gpt4o_model],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### BERT Score\n",
+ "\n",
+ "This test evaluates the quality of the RAG generated answers using BERT embeddings to measure precision, recall, and F1 scores based on semantic similarity, rather than exact n-gram matches as in BLEU and ROUGE. This approach captures contextual meaning, making it valuable when wording differs but the intended message closely aligns with the reference. In RAG applications, the BERT score is especially useful for ensuring that generated answers convey the reference text’s meaning, even if phrasing varies. Consistently low scores indicate a lack of semantic alignment, suggesting the model may miss or misrepresent key content. Low precision may reflect irrelevant or redundant details, while low recall can indicate omissions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.BertScore\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\": [vm_rag_k5_gpt35_model, vm_rag_k5_gpt4o_model],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### METEOR Score\n",
+ "\n",
+ "This test evaluates the quality of the generated answers by measuring alignment with the ground truth, emphasizing both accuracy and fluency. Unlike BLEU and ROUGE, which focus on n-gram matches, METEOR combines precision, recall, synonym matching, and word order, focusing at how well the generated text conveys meaning and reads naturally. This metric is especially useful for RAG applications where sentence structure and natural flow are crucial for clear communication. Lower scores may suggest alignment issues, indicating that the answers may lack fluency or key content. Discrepancies in word order or high fragmentation penalties can reveal problems with how the model constructs sentences, potentially affecting readability."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.MeteorScore\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\": [vm_rag_k5_gpt35_model, vm_rag_k5_gpt4o_model],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Bias and Toxicity\n",
+ "\n",
+ "In this section, we use metrics like Toxicity Score and Regard Score to evaluate both the generated responses and the ground truth. These tests helps us detect any harmful, offensive, or inappropriate language and evaluate the level of bias and neutrality enabling us to assess and mitigate potential biases in both the model's responses and the original dataset."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Toxicity Score\n",
+ "\n",
+ "This test measures the level of harmful or offensive content in the generated answers. The test uses a preloaded toxicity detection tool from Hugging Face, which identifies language that may be inappropriate, aggressive, or derogatory. High toxicity scores indicate potentially toxic content, while consistently elevated scores across multiple outputs may signal underlying issues in the model’s generation process that require attention to prevent the spread of harmful language."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.ToxicityScore\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\": [vm_rag_k5_gpt35_model, vm_rag_k5_gpt4o_model],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Regard Score\n",
+ "\n",
+ "This test evaluates the sentiment and perceived regard—categorized as positive, negative, neutral, or other—in answers generated by the RAG model. This is important for identifying any biases or sentiment tendencies in responses, ensuring that generated answers are balanced and appropriate for the context. The uses a preloaded regard evaluation tool from Hugging Face to compute scores for each response. High skewness in regard scores, especially if the generated responses consistently diverge from expected sentiments in the reference texts, may reveal biases in the model’s generation, such as overly positive or negative tones where neutrality is expected."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_test(\n",
+ " \"validmind.model_validation.RegardScore\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\": [vm_rag_k5_gpt35_model, vm_rag_k5_gpt4o_model],\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Conclusion\n",
+ "\n",
+ "In this notebook, we have seen how we can use LangChain and ValidMind together to build, evaluate and document a simple RAG Model as its developed. This is a great example of the interactive development experience that ValidMind is designed to support. We can quickly iterate on our model and document as we go... We have seen how ValidMind supports non-traditional \"models\" using a functional interface and how we can build pipelines of many models to support complex GenAI workflows.\n",
+ "\n",
+ "This is still a work in progress and we are actively developing new tests to support more advanced GenAI workflows. We are also keeping an eye on the most popular GenAI models and libraries to explore direct integrations. Stay tuned for more updates and new features in this area!"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Upgrade ValidMind\n",
+ "\n",
+ "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n",
+ "\n",
+ "Retrieve the information for the currently installed version of ValidMind:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip show validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n",
+ "\n",
+ "```bash\n",
+ "%pip install --upgrade validmind\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "You may need to restart your kernel after running the upgrade package for changes to be applied."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "ValidMind Library",
+ "language": "python",
+ "name": "validmind"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.15"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb b/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb
index 1070f604d..dfc9ccc15 100644
--- a/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb
+++ b/notebooks/code_samples/nlp_and_llm/rag_documentation_demo.ipynb
@@ -33,7 +33,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb b/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb
index ab5d6d4bf..04ca7aa1c 100644
--- a/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb
+++ b/notebooks/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb
@@ -33,7 +33,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb b/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb
index d489beb2d..5396ee1aa 100644
--- a/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb
+++ b/notebooks/code_samples/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb
@@ -74,7 +74,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/quickstart_customer_churn_full_suite.ipynb b/notebooks/code_samples/quickstart_customer_churn_full_suite.ipynb
index 21c1aa17e..d7a9c6b87 100644
--- a/notebooks/code_samples/quickstart_customer_churn_full_suite.ipynb
+++ b/notebooks/code_samples/quickstart_customer_churn_full_suite.ipynb
@@ -75,7 +75,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
@@ -545,9 +545,9 @@
},
"gpuClass": "standard",
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "ValidMind Library",
"language": "python",
- "name": "python3"
+ "name": "validmind"
},
"language_info": {
"codemirror_mode": {
diff --git a/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb b/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb
index e1e329e13..461d8c289 100644
--- a/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb
+++ b/notebooks/code_samples/regression/quickstart_regression_full_suite.ipynb
@@ -29,7 +29,7 @@
"If this is your first time trying out ValidMind, you can make use of the following resources alongside this notebook:\n",
"\n",
"- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work\n",
- "- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
+ "- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
]
},
{
@@ -447,7 +447,7 @@
"\n",
"What you can see now is a much more easily consumable version of the documentation, including the results of the tests you just performed, along with other parts of your model documentation that still need to be completed. There is a wealth of information that gets uploaded when you run the full test suite, so take a closer look around, especially at test results that might need attention (hint: some of the tests in **2.1 Data description** look like they need some attention).\n",
"\n",
- "If you want to learn more about where you are in the model documentation process, take a look at [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html).\n"
+ "If you want to learn more about where you are in the model documentation process, take a look our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html).\n"
]
},
{
diff --git a/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb b/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb
index 174b05686..c674ffd42 100644
--- a/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb
+++ b/notebooks/code_samples/time_series/quickstart_time_series_full_suite.ipynb
@@ -77,7 +77,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb b/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb
index 444532750..8418dcd0a 100644
--- a/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb
+++ b/notebooks/code_samples/time_series/quickstart_time_series_high_code.ipynb
@@ -77,7 +77,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb b/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb
index 43142c928..44d997de0 100644
--- a/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb
+++ b/notebooks/code_sharing/clustering/quickstart_cluster_demo.ipynb
@@ -30,7 +30,7 @@
"If this is your first time trying out ValidMind, you can make use of the following resources alongside this notebook:\n",
"\n",
"- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work\n",
- "- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference"
+ "- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference"
]
},
{
@@ -406,8 +406,7 @@
"\n",
"What you can see now is a much more easily consumable version of the documentation, including the results of the tests you just performed, along with other parts of your model documentation that still need to be completed. There is a wealth of information that gets uploaded when you run the full test suite, so take a closer look around, especially at test results that might need attention (hint: some of the tests in 2.1 Data description look like they need some attention).\n",
"\n",
- "If you want to learn more about where you are in the model documentation process, take a look at [Get started with the ValidMind Library\n",
- "](https://docs.validmind.ai/developer/get-started-validmind-library.html).\n"
+ "If you want to learn more about where you are in the model documentation process, take a look at our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html).\n"
]
},
{
diff --git a/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb b/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb
index ce49c7307..f8583992e 100644
--- a/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb
+++ b/notebooks/code_sharing/embeddings/quickstart_embeddings_demo.ipynb
@@ -22,7 +22,7 @@
"If this is your first time trying out ValidMind, you can make use of the following resources alongside this notebook:\n",
"\n",
"- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work\n",
- "- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference"
+ "- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference"
]
},
{
diff --git a/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb b/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb
index 6d25409b9..b98aa9d67 100644
--- a/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb
+++ b/notebooks/code_sharing/operational_deposit/operational_deposit_poc.ipynb
@@ -37,7 +37,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb b/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb
index 9cda0afca..c787a8831 100644
--- a/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb
+++ b/notebooks/code_sharing/output_templates/customizing_tests_with_output_templates.ipynb
@@ -85,7 +85,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/code_sharing/r/r_custom_tests.Rmd b/notebooks/code_sharing/r/r_custom_tests.Rmd
index fa835989d..2a89f051b 100644
--- a/notebooks/code_sharing/r/r_custom_tests.Rmd
+++ b/notebooks/code_sharing/r/r_custom_tests.Rmd
@@ -31,7 +31,7 @@ If you encounter errors due to missing modules in your Python environment, insta
### New to ValidMind?
-If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.
+If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.
For access to all features available in this notebook, create a free ValidMind account.
diff --git a/notebooks/code_sharing/regression/regression_unit_metrics.ipynb b/notebooks/code_sharing/regression/regression_unit_metrics.ipynb
index b66bed7bd..66568527a 100644
--- a/notebooks/code_sharing/regression/regression_unit_metrics.ipynb
+++ b/notebooks/code_sharing/regression/regression_unit_metrics.ipynb
@@ -29,7 +29,7 @@
"If this is your first time trying out ValidMind, you can make use of the following resources alongside this notebook:\n",
"\n",
"- [Get started](https://docs.validmind.ai/get-started/get-started.html) — The basics, including key concepts, and how our products work\n",
- "- [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
+ "- [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html) — The path for developers, more code samples, and our developer reference\n"
]
},
{
diff --git a/notebooks/how_to/add_context_to_llm_descriptions.ipynb b/notebooks/how_to/add_context_to_llm_descriptions.ipynb
index 6951e5285..45736d2c7 100644
--- a/notebooks/how_to/add_context_to_llm_descriptions.ipynb
+++ b/notebooks/how_to/add_context_to_llm_descriptions.ipynb
@@ -23,7 +23,7 @@
"- [Initialize the Python environment](#toc3_) \n",
"- [Load the sample dataset](#toc4_) \n",
" - [Preprocess the raw dataset](#toc4_1_) \n",
- "- [Initialize the ValidMind objects](#toc5_) \n",
+ "- [Initializing the ValidMind objects](#toc5_) \n",
" - [Initialize the datasets](#toc5_1_) \n",
" - [Initialize a model object](#toc5_2_) \n",
" - [Assign predictions to the datasets](#toc5_3_) \n",
@@ -230,7 +230,7 @@
"source": [
"\n",
"\n",
- "## Initialize the ValidMind objects"
+ "## Initializing the ValidMind objects"
]
},
{
@@ -349,7 +349,7 @@
"\n",
"By default, custom context for LLM-generated descriptions is disabled, meaning that the output will not include any additional context.\n",
"\n",
- "Let's generate an initial test description for the `DatasetDescription` test for comparision with later iterations:"
+ "Let's generate an initial test description for the `DatasetDescription` test for comparison with later iterations:"
]
},
{
@@ -376,7 +376,7 @@
"\n",
"To enable custom use case context, set the `VALIDMIND_LLM_DESCRIPTIONS_CONTEXT_ENABLED` environment variable to `1`.\n",
"\n",
- "This is a global setting that will affect all tests for your linked model:"
+ "This is a global setting that will affect all tests for your linked model for the duration of your ValidMind Library session:"
]
},
{
@@ -431,7 +431,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "With the use case context set, generate an updated test description for the `DatasetDescription` test for comparision with default output:"
+ "With the use case context set, generate an updated test description for the `DatasetDescription` test for comparison with default output:"
]
},
{
@@ -458,7 +458,7 @@
"\n",
"To disable custom use case context, set the `VALIDMIND_LLM_DESCRIPTIONS_CONTEXT_ENABLED` environment variable to `0`.\n",
"\n",
- "This is a global setting that will affect all tests for your linked model:"
+ "This is a global setting that will affect all tests for your linked model for the duration of your ValidMind Library session:"
]
},
{
@@ -474,7 +474,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "With the use case context disabled again, generate another test description for the `DatasetDescription` test for comparision with previous custom output:"
+ "With the use case context disabled again, generate another test description for the `DatasetDescription` test for comparison with previous custom output:"
]
},
{
diff --git a/notebooks/how_to/configure_dataset_features.ipynb b/notebooks/how_to/configure_dataset_features.ipynb
index 43be9c4d2..2e381bd89 100644
--- a/notebooks/how_to/configure_dataset_features.ipynb
+++ b/notebooks/how_to/configure_dataset_features.ipynb
@@ -67,7 +67,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb b/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb
index 2a46f0f20..2529727af 100644
--- a/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb
+++ b/notebooks/how_to/document_multiple_results_for_the_same_test.ipynb
@@ -82,7 +82,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/explore_test_suites.ipynb b/notebooks/how_to/explore_test_suites.ipynb
index 05bf40703..4fb36d894 100644
--- a/notebooks/how_to/explore_test_suites.ipynb
+++ b/notebooks/how_to/explore_test_suites.ipynb
@@ -63,7 +63,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/explore_tests.ipynb b/notebooks/how_to/explore_tests.ipynb
index 9a60f9c08..672c98fc3 100644
--- a/notebooks/how_to/explore_tests.ipynb
+++ b/notebooks/how_to/explore_tests.ipynb
@@ -81,1191 +81,1786 @@
"data": {
"text/html": [
"\n",
- "
\n",
+ "
\n",
" \n",
"
\n",
- "
ID
\n",
- "
Name
\n",
- "
Description
\n",
- "
Required Inputs
\n",
- "
Params
\n",
+ "
ID
\n",
+ "
Name
\n",
+ "
Description
\n",
+ "
Required Inputs
\n",
+ "
Params
\n",
+ "
Tags
\n",
+ "
Tasks
\n",
"
\n",
" \n",
" \n",
"
\n",
- "
validmind.prompt_validation.Bias
\n",
- "
Bias
\n",
- "
Evaluates bias in a Large Language Model based on the order and distribution of exemplars in a prompt....
\n",
- "
['model.prompt']
\n",
- "
{'min_threshold': 7}
\n",
- "
\n",
- "
\n",
- "
validmind.prompt_validation.Clarity
\n",
- "
Clarity
\n",
- "
Evaluates and scores the clarity of prompts in a Large Language Model based on specified guidelines....
\n",
- "
['model.prompt']
\n",
- "
{'min_threshold': 7}
\n",
- "
\n",
- "
\n",
- "
validmind.prompt_validation.Specificity
\n",
- "
Specificity
\n",
- "
Evaluates and scores the specificity of prompts provided to a Large Language Model (LLM), based on clarity,...
\n",
- "
['model.prompt']
\n",
- "
{'min_threshold': 7}
\n",
- "
\n",
- "
\n",
- "
validmind.prompt_validation.Robustness
\n",
- "
Robustness
\n",
- "
Assesses the robustness of prompts provided to a Large Language Model under varying conditions and contexts....
\n",
- "
['model']
\n",
- "
{'num_tests': 10}
\n",
- "
\n",
- "
\n",
- "
validmind.prompt_validation.NegativeInstruction
\n",
- "
Negative Instruction
\n",
- "
Evaluates and grades the use of affirmative, proactive language over negative instructions in LLM prompts....
\n",
- "
['model.prompt']
\n",
- "
{'min_threshold': 7}
\n",
- "
\n",
- "
\n",
- "
validmind.prompt_validation.Conciseness
\n",
- "
Conciseness
\n",
- "
Analyzes and grades the conciseness of prompts provided to a Large Language Model....
\n",
- "
['model.prompt']
\n",
- "
{'min_threshold': 7}
\n",
- "
\n",
- "
\n",
- "
validmind.prompt_validation.Delimitation
\n",
- "
Delimitation
\n",
- "
Evaluates the proper use of delimiters in prompts provided to Large Language Models....
Evaluates regression model performance using residual distribution and actual vs. predicted plots....
\n",
- "
['model', 'dataset']
\n",
- "
{'bin_size': 0.1}
\n",
- "
\n",
- "
\n",
- "
validmind.model_validation.FeaturesAUC
\n",
- "
Features AUC
\n",
- "
Evaluates the discriminatory power of each individual feature within a binary classification model by calculating the Area Under the Curve (AUC) for each feature separately....
\n",
- "
['model', 'dataset']
\n",
- "
{'fontsize': 12, 'figure_height': 500}
\n",
- "
\n",
- "
\n",
- "
validmind.model_validation.ContextualRecall
\n",
- "
Contextual Recall
\n",
- "
Evaluates a Natural Language Generation model's ability to generate contextually relevant and factually correct text, visualizing the results through histograms and bar charts, alongside compiling a comprehensive table of descriptive statistics for contextual recall scores....
\n",
- "
['dataset', 'model']
\n",
- "
{}
\n",
- "
\n",
- "
\n",
- "
validmind.model_validation.MeteorScore
\n",
- "
Meteor Score
\n",
- "
Computes and visualizes the METEOR score for each text generation instance, assessing translation quality....
\n",
- "
['dataset', 'model']
\n",
- "
{}
\n",
- "
\n",
- "
\n",
- "
validmind.model_validation.RougeScore
\n",
- "
Rouge Score
\n",
- "
Evaluates the quality of machine-generated text using ROUGE metrics and visualizes the results through histograms...
\n",
- "
['dataset', 'model']
\n",
- "
{'metric': 'rouge-1'}
\n",
- "
\n",
- "
\n",
- "
validmind.model_validation.ModelMetadata
\n",
- "
Model Metadata
\n",
- "
Extracts and summarizes critical metadata from a machine learning model instance for comprehensive analysis....
Evaluates the discriminatory power of each individual feature within a binary classification model by calculating the Area Under the Curve (AUC) for each feature separately....
\n",
- "
['model', 'dataset']
\n",
- "
{'fontsize': 12, 'figure_height': 500}
\n",
- "
\n",
- "
\n",
- "
validmind.model_validation.ModelMetadata
\n",
- "
Model Metadata
\n",
- "
Extracts and summarizes critical metadata from a machine learning model instance for comprehensive analysis....
Detects and visualizes overfit reg…"
+ "Accordion(children=(HTML(value='\\n
\\n
Overfit Diagnosis
\\n
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/log_metrics_over_time.ipynb b/notebooks/how_to/log_metrics_over_time.ipynb
index 9cef4c540..bbf4b94cb 100644
--- a/notebooks/how_to/log_metrics_over_time.ipynb
+++ b/notebooks/how_to/log_metrics_over_time.ipynb
@@ -78,7 +78,7 @@
"\n",
"\n",
"### New to ValidMind?\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/run_documentation_sections.ipynb b/notebooks/how_to/run_documentation_sections.ipynb
index baca51bfd..9066d637f 100644
--- a/notebooks/how_to/run_documentation_sections.ipynb
+++ b/notebooks/how_to/run_documentation_sections.ipynb
@@ -73,7 +73,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/run_documentation_tests_with_config.ipynb b/notebooks/how_to/run_documentation_tests_with_config.ipynb
index 82c969c47..8ad291fe8 100644
--- a/notebooks/how_to/run_documentation_tests_with_config.ipynb
+++ b/notebooks/how_to/run_documentation_tests_with_config.ipynb
@@ -77,7 +77,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb b/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb
index 8ed766f90..acfb923f8 100644
--- a/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb
+++ b/notebooks/how_to/run_tests/1_run_dataset_based_tests.ipynb
@@ -72,7 +72,7 @@
"\n",
"\n",
"### New to ValidMind?\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb b/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb
index 2564d8ef4..ab8880c4c 100644
--- a/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb
+++ b/notebooks/how_to/run_tests/2_run_comparison_tests.ipynb
@@ -79,7 +79,7 @@
"\n",
"\n",
"### New to ValidMind?\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb b/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb
index 1d9cdb02b..184594bf2 100644
--- a/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb
+++ b/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb
@@ -75,7 +75,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/run_unit_metrics.ipynb b/notebooks/how_to/run_unit_metrics.ipynb
index 77c92066d..163306ae3 100644
--- a/notebooks/how_to/run_unit_metrics.ipynb
+++ b/notebooks/how_to/run_unit_metrics.ipynb
@@ -104,7 +104,7 @@
"\n",
"### New to ValidMind? \n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/how_to/use_dataset_model_objects.ipynb b/notebooks/how_to/use_dataset_model_objects.ipynb
index 975f6aa25..abf449cb0 100644
--- a/notebooks/how_to/use_dataset_model_objects.ipynb
+++ b/notebooks/how_to/use_dataset_model_objects.ipynb
@@ -79,7 +79,7 @@
"\n",
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/templates/README.md b/notebooks/templates/README.md
index 70a46d4d7..8ad589ab7 100644
--- a/notebooks/templates/README.md
+++ b/notebooks/templates/README.md
@@ -15,4 +15,8 @@ The template generation script/notebook draws from the following mini-templates,
- [`about-validmind.ipynb`](about-validmind.ipynb): Conceptual overview of ValidMind & prerequisites.
- [`install-initialize-validmind.ipynb`](install-initialize-validmind.ipynb): ValidMind Library installation & initialization instructions.
- [`next-steps.ipynb`](next-steps.ipynb): Directions to review the generated documentation within the ValidMind Platform & additional learning resources.
-- [`upgrade-validmind.ipynb`](upgrade-validmind.ipynb): Instructions for comparing & upgrading versions of the ValidMind Library.
\ No newline at end of file
+- [`upgrade-validmind.ipynb`](upgrade-validmind.ipynb): Instructions for comparing & upgrading versions of the ValidMind Library.
+
+## Add table of contents
+
+For lengthy notebooks, we recommend that you add a table of contents with the [**Simplified table of contents for Jupyter Notebooks extension**](https://github.com/validbeck/jupyter-notebook-toc/tree/main/installation).
diff --git a/notebooks/templates/about-validmind.ipynb b/notebooks/templates/about-validmind.ipynb
index 006df0e7a..e982e60d0 100644
--- a/notebooks/templates/about-validmind.ipynb
+++ b/notebooks/templates/about-validmind.ipynb
@@ -31,7 +31,7 @@
"source": [
"### New to ValidMind?\n",
"\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/guide/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
"\n",
"
For access to all features available in this notebook, create a free ValidMind account.\n",
" \n",
diff --git a/notebooks/tutorials/intro_for_model_developers.ipynb b/notebooks/tutorials/intro_for_model_developers.ipynb
index 2b1ece7ba..cd103546b 100644
--- a/notebooks/tutorials/intro_for_model_developers.ipynb
+++ b/notebooks/tutorials/intro_for_model_developers.ipynb
@@ -4,1789 +4,26 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# ValidMind Introduction for Model Developers\n",
+ "# ValidMind introduction for model developers\n",
"\n",
- "As a model developer, learn how the end-to-end documentation process works based on common scenarios you encounter in model development settings.\n",
+ "Learn how to use ValidMind for your end-to-end model documentation process based on common model development scenarios with our *ValidMind for model development* series of four introductory notebooks:\n",
"\n",
- "As a prerequisite, a model documentation template must be available on the ValidMind Platform. You can [view the available templates](https://docs.validmind.com/guide/model-documentation/swap-documentation-templates.html#view-current-templates) to see what templates been set up for your organization.\n",
+ "1. [1 — Set up the ValidMind Library](/notebooks/tutorials/model_development/1-set_up_validmind.ipynb)\n",
+ "2. [2 — Start the model development process](/notebooks/tutorials/model_development/2-start_development_process.ipynb)\n",
+ "3. [3 — Integrate custom tests](/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb)\n",
+ "4. [4 — Finalize testing and documentation](/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb)\n",
"\n",
- "This notebook uses a binary classification model as an example, but the same principles shown here apply to other model types.\n",
- "\n",
- "\n",
- "\n",
- "## Overview of the notebook\n",
- "\n",
- "**1. Initializing the ValidMind Library**\n",
- "\n",
- "The ValidMind Library provides a rich collection of documentation tools and test suites, from documenting descriptions of datasets to validation and testing of models using a variety of open-source testing frameworks.\n",
- "\n",
- "**2. Start the model development process with raw data, run out-of-the box tests, and add evidence to model documentation**\n",
- "\n",
- "Learn how to access ValidMind's test repository of individual tests that you will use as building blocks to ensure a model is being built appropriately. The goal is to show how to run tests, investigate results, and add tests results or evidence to the documentation.\n",
- "\n",
- "For a full list of out-of-the-box tests, see [Test descriptions](https://docs.validmind.ai/developer/model-testing/test-descriptions.html) or try the interactive [Test sandbox](https://docs.validmind.ai/developer/model-testing/test-sandbox.html).\n",
- "\n",
- "**3. Implementing custom tests**\n",
- "\n",
- "Usually, model developers have their own custom tests and it is important to include this within the model documentation. We will show you how to include custom tests and then how they can be added to documentation as additional evidence.\n",
- "\n",
- "**4. Finalize testing and documentation**\n",
- "\n",
- "Learn how you can ensure that model documentation includes custom tests and how to make test configuration changes that apply to all tests in the model documentation template. At the end of this section you should have a fully documented model ready for review.\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Contents\n",
- "\n",
- "- [Overview of the notebook](#toc1_)\n",
- "- [About ValidMind](#toc2_)\n",
- " - [Before you begin](#toc2_1_)\n",
- " - [New to ValidMind?](#toc2_2_)\n",
- " - [Key concepts](#toc2_3_)\n",
- "- [1. Initializing the ValidMind Library](#toc4_)\n",
- " - [Install the ValidMind Library](#toc4_1_)\n",
- " - [Initialize the ValidMind Library](#toc4_2_)\n",
- " - [Get your code snippet](#toc4_2_1_)\n",
- " - [Verify & preview the documentation template](#toc4_3_)\n",
- "- [2. Start the model development process with raw data, run out-of-the box tests, and add evidence to model documentation](#toc5_)\n",
- " - [Initialize the ValidMind datasets](#toc5_1_)\n",
- " - [Run some tabular data tests](#toc5_2_)\n",
- " - [Utilize test output](#toc5_3_)\n",
- " - [Documenting the results based on two datasets](#toc5_4_)\n",
- " - [Run `run_documentation_tests()` using `vm_raw_dataset_preprocessed` as input](#toc5_4_1_)\n",
- " - [Log the individual result of the high correlation test that used `vm_balanced_raw_dataset` (that had a highly correlated `Age` column) as input](#toc5_4_2_)\n",
- " - [Add individual test results to model documentation](#toc5_5_)\n",
- " - [Model Testing](#toc5_6_)\n",
- " - [Initialize model evaluation objects and assigning predictions](#toc5_7_)\n",
- " - [Run the model evaluation tests](#toc5_8_)\n",
- "- [3. Implementing custom tests](#toc6_)\n",
- " - [Create a confusion matrix plot](#toc6_1_)\n",
- " - [Add parameters to custom tests](#toc6_2_)\n",
- " - [Pass parameters to custom tests](#toc6_3_)\n",
- " - [Log the confusion matrix results](#toc6_4_)\n",
- " - [Using external test providers](#toc6_5_)\n",
- " - [Create a folder of custom tests from existing inline tests](#toc6_5_1_)\n",
- " - [Save an inline test to a file](#toc6_5_2_)\n",
- " - [Define and register a `LocalTestProvider` that points to that folder](#toc6_5_3_)\n",
- " - [Initializing a local test provider](#toc6_6_)\n",
- " - [Run test provider tests](#toc6_6_1_)\n",
- " - [Add the test results to your documentation](#toc6_6_2_)\n",
- "- [4. Finalize testing and documentation](#toc7_)\n",
- " - [Use `run_documentation_tests()` to ensure custom test results are included in your documentation](#toc7_1_)\n",
- " - [Viewing and updating the configuration for the entire model documentation template](#toc7_2_)\n",
- " - [Update the config](#toc7_2_1_)\n",
- "- [Where to go from here](#toc8_)\n",
- " - [Use cases](#toc8_1_)\n",
- " - [More how-to guides and code samples](#toc8_2_)\n",
- " - [Discover more learning resources](#toc8_3_)\n",
- "- [Upgrade ValidMind](#toc9_)\n",
- "\n",
- "\n",
- "\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "## About ValidMind\n",
- "\n",
- "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models. You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n",
- "\n",
- "\n",
- "\n",
- "### Before you begin\n",
- "\n",
- "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language.\n",
- "\n",
- "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n",
- "\n",
- "\n",
- "\n",
- "### New to ValidMind?\n",
- "\n",
- "If you haven't already seen our [Get started with the ValidMind Library](https://docs.validmind.ai/developer/get-started-validmind-library.html), we recommend you explore the available resources for developers at some point. There, you can learn more about documenting models, find code samples, or read our developer reference.\n",
- "\n",
- "
For access to all features available in this notebook, create a free ValidMind account.\n",
+ "
Or, take our Developer Fundamentals course which walks you through the basics of ValidMind paired with this notebook series.\n",
" \n",
- "Signing up is FREE — Register with ValidMind
\n",
- "\n",
- "\n",
- "\n",
- "### Key concepts\n",
- "\n",
- "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n",
- "\n",
- "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n",
- "\n",
- "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n",
- "\n",
- "**Custom tests**: Custom tests are functions that you define to evaluate your model or dataset. These functions can be registered via the ValidMind Library to be used with the ValidMind Platform.\n",
- "\n",
- "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n",
- "\n",
- "- **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n",
- "- **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n",
- "- **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom test.\n",
- "- **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom test. See this [example](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html) for more information.\n",
- "\n",
- "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a test, customize its behavior, or provide additional context.\n",
- "\n",
- "**Outputs**: Custom tests can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n",
- "\n",
- "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n",
- "\n",
- "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases.\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "## 1. Initializing the ValidMind Library\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Install the ValidMind Library\n",
- "\n",
- "Please note the following recommended Python versions to use:\n",
- "\n",
- "- Python 3.8 <= x <= 3.11\n",
- "\n",
- "To install the library:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "%pip install -q validmind"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Initialize the ValidMind Library\n",
- "\n",
- "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n",
- "\n",
- "\n",
- "\n",
- "### Get your code snippet\n",
- "\n",
- "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
- "\n",
- "2. In the left sidebar, navigate to **Model Inventory** and click **+ Register Model**.\n",
- "\n",
- "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n",
- "\n",
- " For example, to register a model for use with this notebook, select:\n",
- "\n",
- " - Documentation template: `Binary classification`\n",
- " - Use case: `Marketing/Sales - Attrition/Churn Management`\n",
- "\n",
- " You can fill in other options according to your preference.\n",
- "\n",
- "4. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
- "\n",
- "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "# Load your model identifier credentials from an `.env` file\n",
- "\n",
- "%load_ext dotenv\n",
- "%dotenv .env\n",
- "\n",
- "# Or replace with your code snippet\n",
- "\n",
- "import validmind as vm\n",
- "\n",
- "vm.init(\n",
- " # api_host=\"...\",\n",
- " # api_key=\"...\",\n",
- " # api_secret=\"...\",\n",
- " # model=\"...\",\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Verify & preview the documentation template\n",
- "\n",
- "Let's verify that you have connected to ValidMind and that the appropriate template is selected. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n",
- "\n",
- "You will upload documentation and test results for this template later on. For now, take a look at the structure that the template provides with the `vm.preview_template()` function from the ValidMind library and note the empty sections:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "vm.preview_template()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Before learning how to run tests, let's explore the list of all available tests in the ValidMind Library. You can see that the documentation template for this model has references to some of the test IDs listed below.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "vm.tests.list_tests()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "## 2. Start the model development process with raw data, run out-of-the box tests, and add evidence to model documentation\n",
- "\n",
- "In this section you learn how to explore the individual tests available in ValidMind and how to run them and change parameters as necessary. You will use a public dataset from Kaggle that models a bank customer churn prediction use case. The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n",
- "\n",
- "You can find more information about this dataset [here](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction).\n",
- "\n",
- "The ValidMind Library provides a wrapper to automatically load the dataset as a Pandas DataFrame object.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "from validmind.datasets.classification import customer_churn as demo_dataset\n",
- "\n",
- "print(\n",
- " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n",
- ")\n",
- "\n",
- "raw_df = demo_dataset.load_data()\n",
- "raw_df.head()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Let's do some data quality assessments by running a few individual tests related to data assessment. You will use the `vm.tests.list_tests()` function introduced above in combination with `vm.tests.list_tags()` and `vm.tests.list_tasks()` to find which prebuilt tests are relevant for data quality assessment.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "# Get the list of available tags\n",
- "sorted(vm.tests.list_tags())"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "# Get the list of available task types\n",
- "sorted(vm.tests.list_tasks())"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "You can pass `tags` and `tasks` as parameters to the `vm.tests.list_tests()` function to filter the tests based on the tags and task types. For example, to find tests related to tabular data quality for classification models, you can call `list_tests()` like this:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "vm.tests.list_tests(task=\"classification\", tags=[\"tabular_data\", \"data_quality\"])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Initialize the ValidMind datasets\n",
- "\n",
- "Now, assume we have identified some tests we want to run with regards to the data we are intending to use. The next step is to connect your data with a ValidMind `Dataset` object. This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind. You only need to do it one time per dataset.\n",
- "\n",
- "You can initialize a ValidMind dataset object using the [`init_dataset`](https://docs.validmind.ai/validmind/validmind.html#init_dataset) function from the ValidMind (`vm`) module.\n",
- "\n",
- "This function takes a number of arguments:\n",
- "\n",
- "- `dataset` — the raw dataset that you want to provide as input to tests\n",
- "- `input_id` - a unique identifier that allows tracking what inputs are used when running each individual test\n",
- "- `target_column` — a required argument if tests require access to true values. This is the name of the target column in the dataset\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "# vm_raw_dataset is now a VMDataset object that you can pass to any ValidMind test\n",
- "vm_raw_dataset = vm.init_dataset(\n",
- " dataset=raw_df,\n",
- " input_id=\"raw_dataset\",\n",
- " target_column=\"Exited\",\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Run some tabular data tests\n",
- "\n",
- "Individual tests can be easily run by calling the `run_test` function provided by the `validmind.tests` module. The function takes the following arguments:\n",
- "\n",
- "- `test_id`: The ID of the test to run. To find a particular test and get its ID, refer to the [explore_tests](../how_to/explore_tests.ipynb) notebook. Look above for example after running 'vm.test_suites.describe_suite' as column 'Test ID' will contain the id.\n",
- "- `params`: A dictionary of parameters for the test. These will override any `default_params` set in the test definition. Refer to the [explore_tests](../how_to/explore_tests.ipynb) notebook to find the default parameters for a test. See below for examples.\n",
- "\n",
- "The inputs expected by a test can also be found in the test definition. Let's take `validmind.data_validation.DescriptiveStatistics` as an example. Note that the output of the `describe_test()` function below shows that this test expects a `dataset` as input:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "vm.tests.describe_test(\"validmind.data_validation.DescriptiveStatistics\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now, let's run a few tests to assess the quality of the dataset.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "result = vm.tests.run_test(\n",
- " test_id=\"validmind.data_validation.DescriptiveStatistics\",\n",
- " inputs={\"dataset\": vm_raw_dataset},\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "result2 = vm.tests.run_test(\n",
- " test_id=\"validmind.data_validation.ClassImbalance\",\n",
- " inputs={\"dataset\": vm_raw_dataset},\n",
- " params={\"min_percent_threshold\": 30},\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "You can see that the class imbalance test did not pass according to the value of `min_percent_threshold` we have set. Here is how you can re-run the test on some processed data to address this data quality issue. In this case we apply a very simple rebalancing technique to the dataset.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "import pandas as pd\n",
- "\n",
- "raw_copy_df = raw_df.sample(frac=1) # Create a copy of the raw dataset\n",
- "\n",
- "# Create a balanced dataset with the same number of exited and not exited customers\n",
- "exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 1]\n",
- "not_exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 0].sample(n=exited_df.shape[0])\n",
- "\n",
- "balanced_raw_df = pd.concat([exited_df, not_exited_df])\n",
- "balanced_raw_df = balanced_raw_df.sample(frac=1, random_state=42)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "With this new raw dataset, you can re-run the individual test to see if it passes the class imbalance test requirement. Remember to register new VM Dataset object since that is the type of input required by `run_test()`:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "# Register new data and now 'balanced_raw_dataset' is the new dataset object of interest\n",
- "vm_balanced_raw_dataset = vm.init_dataset(\n",
- " dataset=balanced_raw_df,\n",
- " input_id=\"balanced_raw_dataset\",\n",
- " target_column=\"Exited\",\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "result = vm.tests.run_test(\n",
- " test_id=\"validmind.data_validation.ClassImbalance\",\n",
- " inputs={\"dataset\": vm_balanced_raw_dataset},\n",
- " params={\"min_percent_threshold\": 30},\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Utilize test output\n",
- "\n",
- "Here is an example for how you can utilize the output from a ValidMind test for futher use, for example, if you want to remove highly correlated features. The example below shows how you can get the list of features with the highest correlation coefficients and use them to reduce the final list of features for modeling.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "corr_result = vm.tests.run_test(\n",
- " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n",
- " params={\"max_threshold\": 0.3},\n",
- " inputs={\"dataset\": vm_balanced_raw_dataset},\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Let's assume we want to remove highly correlated features from the dataset. `corr_result` is an object of type `TestResult`. We can inspect the result object to see what the test has produced."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "print(type(corr_result))\n",
- "print(\"Result ID: \", corr_result.result_id)\n",
- "print(\"Params: \", corr_result.params)\n",
- "print(\"Passed: \", corr_result.passed)\n",
- "print(\"Tables: \", corr_result.tables)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Let's check out the table in the result and extract a list of features that failed the test:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "features_df = corr_result.tables[0].data\n",
- "features_df"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Remove the highly correlated features and create a new VM dataset object. Note the use of different `input_id`s. This allows tracking the inputs used when running each individual test.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "high_correlation_features = features_df[features_df[\"Pass/Fail\"] == \"Fail\"][\"Columns\"].tolist()\n",
- "high_correlation_features"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Extract the feature names from the list of strings (e.g. '(Age, Exited)' -> 'Age')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "high_correlation_features = [feature.split(\",\")[0].strip(\"()\") for feature in high_correlation_features]\n",
- "high_correlation_features"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 42,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "# Remove the highly correlated features from the dataset\n",
- "balanced_raw_no_age_df = balanced_raw_df.drop(columns=high_correlation_features)\n",
- "\n",
- "# Re-initialize the dataset object\n",
- "vm_raw_dataset_preprocessed = vm.init_dataset(\n",
- " dataset=balanced_raw_no_age_df,\n",
- " input_id=\"raw_dataset_preprocessed\",\n",
- " target_column=\"Exited\",\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Re-running the test with the reduced feature set should pass the test. You can also plot the correlation matrix to visualize the new correlation between features:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "corr_result = vm.tests.run_test(\n",
- " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n",
- " params={\"max_threshold\": 0.3},\n",
- " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "corr_result = vm.tests.run_test(\n",
- " test_id=\"validmind.data_validation.PearsonCorrelationMatrix\",\n",
- " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Documenting the results based on two datasets\n",
- "\n",
- "We have now done some analysis on two different datasets and we should able to document why certain things were done to the raw data with testing to support it. Every test result returned by the `run_test()` function has a `.log()` method that can be used to log the test results to ValidMind. When logging individual results to ValidMind you need to manually add those results in a specific section of the model documentation.\n",
- "\n",
- "When using `run_documentation_tests()`, it's possible to automatically populate a section with the results of all tests that were registered in the documentation template.\n",
- "\n",
- "To show how to add individual results to any documentation section, we're going to populate the entire `data_preparation` section of the documentation using the clean `vm_raw_dataset_preprocessed` dataset as input, and then we're going to document an additional result for the highly correlated dataset `vm_balanced_raw_dataset`. The following two steps will accomplish this:\n",
- "\n",
- "1. Run `run_documentation_tests()` using `vm_raw_dataset_preprocessed` as input. This populates the entire data preparation section for every test that is already part of the documentation template.\n",
- "2. Log the individual result of the high correlation test that used `vm_balanced_raw_dataset` (that had a highly correlated `Age` column) as input\n",
- "\n",
- "After adding the result of step #2 to the documentation you will be able to explain the changes made to the raw data by editing the default description of the test result within the ValidMind Platform.\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "#### Run `run_documentation_tests()` using `vm_raw_dataset_preprocessed` as input\n",
- "\n",
- "`run_documentation_tests()` allows you to run multiple tests at once and log the results to the documentation. The function takes the following arguments:\n",
- "\n",
- "- `inputs`: any inputs to be passed to the tests\n",
- "- `config`: a dictionary `:` that allows configuring each test individually. Each test config has the following form:\n",
- " - `params`: individual test parameters\n",
- " - `inputs`: individual test inputs. When passed, this overrides any inputs passed from the `run_documentation_tests()` function\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "test_config = {\n",
- " \"validmind.data_validation.ClassImbalance\": {\n",
- " \"params\": {\"min_percent_threshold\": 30},\n",
- " },\n",
- " \"validmind.data_validation.HighPearsonCorrelation\": {\n",
- " \"params\": {\"max_threshold\": 0.3},\n",
- " },\n",
- "}\n",
- "\n",
- "tests_suite = vm.run_documentation_tests(\n",
- " inputs={\n",
- " \"dataset\": vm_raw_dataset_preprocessed,\n",
- " },\n",
- " config=test_config,\n",
- " section=[\"data_preparation\"],\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "#### Log the individual result of the high correlation test that used `vm_balanced_raw_dataset` (that had a highly correlated `Age` column) as input\n",
- "\n",
- "Here you can use a custom `result_id` to tag the individual result with a unique identifier. This `result_id` can be appended to `test_id` with a `:` separator. The `balanced_raw_dataset` result identifier will correspond to the `balanced_raw_dataset` input, the dataset that still has the `Age` column.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "result = vm.tests.run_test(\n",
- " test_id=\"validmind.data_validation.HighPearsonCorrelation:balanced_raw_dataset\",\n",
- " params={\"max_threshold\": 0.3},\n",
- " inputs={\"dataset\": vm_balanced_raw_dataset},\n",
- ")\n",
- "result.log()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Add individual test results to model documentation\n",
- "\n",
- "You can now visit the documentation page for the model you connected to at the beginning of this notebook and add a new content block in the relevant section.\n",
- "\n",
- "To do this, go to the documentation page of your model and navigate to the `Data Preparation` -> `Correlations and Interactions` section. Then hover after the \"Pearson Correlation Matrix\" content block to reveal the `+` button as shown in the screenshot below.\n",
- "\n",
- "\n",
- "\n",
- "Click on the `+` button and select `Test-Driven Block`. This will open a dialog where you can select `Threshold Test` as the type of the test-driven content block, and then select `High Pearson Correlation Vm Raw Dataset Test`. This will show a preview of the result and it should match the results shown above.\n",
- "\n",
- "\n",
- "\n",
- "Finally, click on the `Insert block` button to add the test result to the documentation. You'll now see two individual results for the high correlation test in the `Correlations and Interactions` section of the documentation. To finalize the documentation, you can edit the test result's description block to explain the changes made to the raw data and the reasons behind them as we can see in the screenshot below.\n",
- "\n",
- "\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Model Testing\n",
- "\n",
- "We have focused so far on the data assessment and pre-processing that usually occurs prior to any models being built. Now we are going to assume we have built a model and we want to incorporate some model results in our documentation.\n",
- "\n",
- "Let's train a simple logistic regression model on the dataset and evaluate its performance. You will use the `LogisticRegression` class from the `sklearn.linear_model` and use ValidMind tests to evaluate the model's performance.\n",
- "\n",
- "Before training the model, we need to encode the categorical features in the dataset. You will use the `OneHotEncoder` class from the `sklearn.preprocessing` module to encode the categorical features. The categorical features in the dataset are `Geography` and `Gender`.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "balanced_raw_no_age_df.head()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "balanced_raw_no_age_df = pd.get_dummies(\n",
- " balanced_raw_no_age_df, columns=[\"Geography\", \"Gender\"], drop_first=True\n",
- ")\n",
- "balanced_raw_no_age_df.head()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 49,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "from sklearn.linear_model import LogisticRegression\n",
- "from sklearn.model_selection import train_test_split\n",
- "\n",
- "# Split the input and target variables\n",
- "X = balanced_raw_no_age_df.drop(\"Exited\", axis=1)\n",
- "y = balanced_raw_no_age_df[\"Exited\"]\n",
- "X_train, X_test, y_train, y_test = train_test_split(\n",
- " X,\n",
- " y,\n",
- " test_size=0.2,\n",
- " random_state=42,\n",
- ")\n",
- "\n",
- "# Logistic Regression grid params\n",
- "log_reg_params = {\n",
- " \"penalty\": [\"l1\", \"l2\"],\n",
- " \"C\": [0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
- " \"solver\": [\"liblinear\"],\n",
- "}\n",
- "\n",
- "# Grid search for Logistic Regression\n",
- "from sklearn.model_selection import GridSearchCV\n",
- "\n",
- "grid_log_reg = GridSearchCV(LogisticRegression(), log_reg_params)\n",
- "grid_log_reg.fit(X_train, y_train)\n",
- "\n",
- "# Logistic Regression best estimator\n",
- "log_reg = grid_log_reg.best_estimator_"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Initialize model evaluation objects and assigning predictions\n",
- "\n",
- "The last step for evaluating the model's performance is to initialize the ValidMind `Dataset` and `Model` objects and assign model predictions to each dataset. You will use the `init_dataset`, `init_model` and `assign_predictions` functions to initialize these objects.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 50,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "train_df = X_train\n",
- "train_df[\"Exited\"] = y_train\n",
- "test_df = X_test\n",
- "test_df[\"Exited\"] = y_test\n",
- "\n",
- "vm_train_ds = vm.init_dataset(\n",
- " input_id=\"train_dataset_final\",\n",
- " dataset=train_df,\n",
- " target_column=\"Exited\",\n",
- ")\n",
- "\n",
- "vm_test_ds = vm.init_dataset(\n",
- " input_id=\"test_dataset_final\",\n",
- " dataset=test_df,\n",
- " target_column=\"Exited\",\n",
- ")\n",
- "\n",
- "# Register the model\n",
- "vm_model = vm.init_model(log_reg, input_id=\"log_reg_model_v1\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Once the model has been registered you can assign model predictions to the training and test datasets. The `assign_predictions()` method from the `Dataset` object can link existing predictions to any number of models. If no prediction values are passed, the method will compute predictions automatically:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "vm_train_ds.assign_predictions(model=vm_model)\n",
- "vm_test_ds.assign_predictions(model=vm_model)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Run the model evaluation tests\n",
- "\n",
- "In this part, we focus on running the tests within the model development section of the model documentation. Only tests associated with this section will be executed, and the corresponding results will be updated in the model documentation. In the example below, you will focus on only running tests for the `model development` section of the document.\n",
- "\n",
- "Note the additional config that is passed to `run_documentation_tests()`. This allows you to override inputs or params in certain tests. In our case, we want to explicitly use the `vm_train_ds` for the `validmind.model_validation.sklearn.ClassifierPerformance:in_sample` test, since it's supposed to run on the training dataset and not the test dataset.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "test_config = {\n",
- " \"validmind.model_validation.sklearn.ClassifierPerformance:in_sample\": {\n",
- " \"inputs\": {\n",
- " \"dataset\": vm_train_ds,\n",
- " \"model\": vm_model,\n",
- " },\n",
- " }\n",
- "}\n",
- "results = vm.run_documentation_tests(\n",
- " section=[\"model_development\"],\n",
- " inputs={\n",
- " \"dataset\": vm_test_ds, # Any test that requires a single dataset will use vm_test_ds\n",
- " \"model\": vm_model,\n",
- " \"datasets\": (\n",
- " vm_train_ds,\n",
- " vm_test_ds,\n",
- " ), # Any test that requires multiple datasets will use vm_train_ds and vm_test_ds\n",
- " },\n",
- " config=test_config,\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "## 3. Implementing custom tests\n",
- "\n",
- "This section assumes that model developers already have a repository of custom made tests that they consider critical to include in the documentation. Here we provide details on how to easily integrate custom tests with ValidMind.\n",
- "\n",
- "For a more in-depth introduction to custom tests, refer to this [notebook](../code_samples/custom_tests/implement_custom_tests.ipynb).\n",
- "\n",
- "A custom test is any function that takes a set of inputs and parameters as arguments and returns one or more outputs. The function can be as simple or as complex as you need it to be. It can use external libraries, make API calls, or do anything else that you can do in Python. The only requirement is that the function signature and return values can be \"understood\" and handled by the ValidMind Library. As such, custom tests offer added flexibility by extending the default tests provided by ValidMind, enabling you to document any type of model or use case.\n",
- "\n",
- "In the following example, you will learn how to implement a custom `inline` test that calculates the confusion matrix for a binary classification model. You will see that the custom test function is just a regular Python function that can include and require any Python library as you see fit.\n",
- "\n",
- "**NOTE**: in the context of Jupyter notebooks, we will use the word `inline` to refer to functions (or code) defined in the same notebook where they are used (this one) and not in a separate file, as we will see later with test providers.\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Create a confusion matrix plot\n",
- "\n",
- "To understand how to create a custom test from anything, let's first create a confusion matrix plot using the `confusion_matrix` function from the `sklearn.metrics` module.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "import matplotlib.pyplot as plt\n",
- "from sklearn import metrics\n",
- "\n",
- "# Get the predicted classes\n",
- "y_pred = log_reg.predict(vm_test_ds.x)\n",
- "\n",
- "confusion_matrix = metrics.confusion_matrix(y_test, y_pred)\n",
- "\n",
- "cm_display = metrics.ConfusionMatrixDisplay(\n",
- " confusion_matrix=confusion_matrix, display_labels=[False, True]\n",
- ")\n",
- "cm_display.plot()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We will now create a @vm.test wrapper that will allow you to create a reusable test. Note the following changes in the code below:\n",
- "\n",
- "- The function `confusion_matrix` takes two arguments `dataset` and `model`. This is a `VMDataset` and `VMModel` object respectively.\n",
- " - `VMDataset` objects allow you to access the dataset's true (target) values by accessing the `.y` attribute.\n",
- " - `VMDataset` objects allow you to access the predictions for a given model by accessing the `.y_pred()` method.\n",
- "- The function docstring provides a description of what the test does. This will be displayed along with the result in this notebook as well as in the ValidMind Platform.\n",
- "- The function body calculates the confusion matrix using the `sklearn.metrics.confusion_matrix` function as we just did above.\n",
- "- The function then returns the `ConfusionMatrixDisplay.figure_` object - this is important as the ValidMind Library expects the output of the custom test to be a plot or a table.\n",
- "- The `@vm.test` decorator is doing the work of creating a wrapper around the function that will allow it to be run by the ValidMind Library. It also registers the test so it can be found by the ID `my_custom_tests.ConfusionMatrix` (see the section below on how test IDs work in ValidMind and why this format is important)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 54,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "@vm.test(\"my_custom_tests.ConfusionMatrix\")\n",
- "def confusion_matrix(dataset, model):\n",
- " \"\"\"The confusion matrix is a table that is often used to describe the performance of a classification model on a set of data for which the true values are known.\n",
- "\n",
- " The confusion matrix is a 2x2 table that contains 4 values:\n",
- "\n",
- " - True Positive (TP): the number of correct positive predictions\n",
- " - True Negative (TN): the number of correct negative predictions\n",
- " - False Positive (FP): the number of incorrect positive predictions\n",
- " - False Negative (FN): the number of incorrect negative predictions\n",
- "\n",
- " The confusion matrix can be used to assess the holistic performance of a classification model by showing the accuracy, precision, recall, and F1 score of the model on a single figure.\n",
- " \"\"\"\n",
- " y_true = dataset.y\n",
- " y_pred = dataset.y_pred(model=model)\n",
- "\n",
- " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n",
- "\n",
- " cm_display = metrics.ConfusionMatrixDisplay(\n",
- " confusion_matrix=confusion_matrix, display_labels=[False, True]\n",
- " )\n",
- " cm_display.plot()\n",
- "\n",
- " plt.close() # close the plot to avoid displaying it\n",
- "\n",
- " return cm_display.figure_ # return the figure object itself"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "You can now run the newly created custom test on both the training and test datasets using the `run_test()` function:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "# Training dataset\n",
- "result = vm.tests.run_test(\n",
- " \"my_custom_tests.ConfusionMatrix:training_dataset\",\n",
- " inputs={\"model\": vm_model, \"dataset\": vm_train_ds},\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "# Test dataset\n",
- "result = vm.tests.run_test(\n",
- " \"my_custom_tests.ConfusionMatrix:test_dataset\",\n",
- " inputs={\"model\": vm_model, \"dataset\": vm_test_ds},\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Add parameters to custom tests\n",
- "\n",
- "Custom tests can take parameters just like any other function. Let's modify the `confusion_matrix` function to take an additional parameter `normalize` that will allow you to normalize the confusion matrix.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 57,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "@vm.test(\"my_custom_tests.ConfusionMatrix\")\n",
- "def confusion_matrix(dataset, model, normalize=False):\n",
- " \"\"\"The confusion matrix is a table that is often used to describe the performance of a classification model on a set of data for which the true values are known.\n",
- "\n",
- " The confusion matrix is a 2x2 table that contains 4 values:\n",
- "\n",
- " - True Positive (TP): the number of correct positive predictions\n",
- " - True Negative (TN): the number of correct negative predictions\n",
- " - False Positive (FP): the number of incorrect positive predictions\n",
- " - False Negative (FN): the number of incorrect negative predictions\n",
- "\n",
- " The confusion matrix can be used to assess the holistic performance of a classification model by showing the accuracy, precision, recall, and F1 score of the model on a single figure.\n",
- " \"\"\"\n",
- " y_true = dataset.y\n",
- " y_pred = dataset.y_pred(model=model)\n",
- "\n",
- " if normalize:\n",
- " confusion_matrix = metrics.confusion_matrix(y_true, y_pred, normalize=\"all\")\n",
- " else:\n",
- " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n",
- "\n",
- " cm_display = metrics.ConfusionMatrixDisplay(\n",
- " confusion_matrix=confusion_matrix, display_labels=[False, True]\n",
- " )\n",
- " cm_display.plot()\n",
- "\n",
- " plt.close() # close the plot to avoid displaying it\n",
- "\n",
- " return cm_display.figure_ # return the figure object itself"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Pass parameters to custom tests\n",
- "\n",
- "You can pass parameters to custom tests by providing a dictionary of parameters to the `run_test()` function. The parameters will override any default parameters set in the custom test definition. Note that `dataset` and `model` are still passed as `inputs`. Since these are `VMDataset` or `VMModel` inputs, they have a special meaning. When declaring a `dataset`, `model`, `datasets` or `models` argument in a custom test function, the ValidMind Library will expect these get passed as `inputs` to `run_test()` (or `run_documentation_tests()` instead).\n",
- "\n",
- "Re-running the confusion matrix with `normalize=True` looks like this:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "# Test dataset with normalize=True\n",
- "result = vm.tests.run_test(\n",
- " \"my_custom_tests.ConfusionMatrix:test_dataset_normalized\",\n",
- " inputs={\"model\": vm_model, \"dataset\": vm_test_ds},\n",
- " params={\"normalize\": True},\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Log the confusion matrix results\n",
- "\n",
- "As you saw in the pearson correlation example, you can log any result to the ValidMind Platform with the `.log()` method of the result object. This will allow you to add the result to the documentation.\n",
- "\n",
- "You can now do the same for the confusion matrix results.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 59,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "result.log()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Using external test providers\n",
- "\n",
- "Creating inline custom tests with a function is a great way to customize your model documentation. However, sometimes you may want to reuse the same set of tests across multiple models and share them with developers in your organization. In this case, you can create a custom test provider that will allow you to load custom tests from a local folder or a git repository.\n",
- "\n",
- "In this section you will learn how to declare a local filesystem test provider that allows loading tests from a local folder following these high level steps:\n",
- "\n",
- "1. Create a folder of custom tests from existing, inline tests (tests that exists in your active Jupyter notebook)\n",
- "2. Save an inline test to a file\n",
- "3. Define and register a `LocalTestProvider` that points to that folder\n",
- "4. Run test provider tests\n",
- "5. Add the test results to your documentation\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "#### Create a folder of custom tests from existing inline tests\n",
- "\n",
- "Here you will create a new folder that will contain reusable, custom tests. The following code snippet will create a new `my_tests` directory in the current working directory if it doesn't exist.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 60,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "tests_folder = \"my_tests\"\n",
- "\n",
- "import os\n",
- "\n",
- "# create tests folder\n",
- "os.makedirs(tests_folder, exist_ok=True)\n",
- "\n",
- "# remove existing tests\n",
- "for f in os.listdir(tests_folder):\n",
- " # remove files and pycache\n",
- " if f.endswith(\".py\") or f == \"__pycache__\":\n",
- " os.system(f\"rm -rf {tests_folder}/{f}\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "After running the command above, you should see a new directory next to this notebook file:\n",
- "\n",
- "\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "#### Save an inline test to a file\n",
- "\n",
- "The `@vm.test` decorator that was used above to register these as one-off custom tests also adds a convenience method to the function object that allows you to simply call `.save()` to save it to a file. This will save the function to a Python file to a path you specify. In this case, you can pass the variable `tests_folder` to save it to the custom tests folder we created.\n",
- "\n",
- "Normally, this will get you started by creating the file and saving the function code with the correct name. But it won't automatically add any import or other functions/variables outside of the function that are needed for the test to run. The `save()` method allows you to pass an optional `imports` argument that will ensure the necessary imports are added to the file.\n",
- "\n",
- "For the `confusion_matrix` test, note the imports that are required for the function to run properly:\n",
- "\n",
- "```python\n",
- "import matplotlib.pyplot as plt\n",
- "from sklearn import metrics\n",
- "```\n",
- "\n",
- "You can pass these imports to the `save()` method to ensure they are included in the file with the following command:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "confusion_matrix.save(\n",
- " tests_folder,\n",
- " imports=[\"import matplotlib.pyplot as plt\", \"from sklearn import metrics\"],\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "##### What happened?\n",
- "\n",
- "The `save()` method saved the `confusion_matrix` function to a file named `ConfusionMatrix.py` in the `my_tests` folder. Note that the new file provides some context on the origin of the test, which is useful for traceability.\n",
- "\n",
- "```\n",
- "# Saved from __main__.confusion_matrix\n",
- "# Original Test ID: my_custom_tests.ConfusionMatrix\n",
- "# New Test ID: .ConfusionMatrix\n",
- "```\n",
- "\n",
- "Additionally, the new test function has been stripped off its decorator, as it now resides in a file that will be loaded by the test provider:\n",
- "\n",
- "```python\n",
- "def ConfusionMatrix(dataset, model, normalize=False):\n",
- "```\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "#### Define and register a `LocalTestProvider` that points to that folder\n",
- "\n",
- "With the `my_tests` folder now having a sample custom test, you can now initialize a test provider that will tell the ValidMind Library where to find these tests. ValidMind offers out-of-the-box test providers for local tests (i.e. tests in a folder) or a Github provider for tests in a Github repository. You can also create your own test provider by creating a class that has a `load_test` method that takes a test ID and returns the test function matching that ID.\n",
- "\n",
- "The most important attribute for a test provider is its `namespace`. This is a string that will be used to prefix test IDs in model documentation. This allows you to have multiple test providers with tests that can even share the same ID, but are distinguished by their namespace.\n",
- "\n",
- "An extended introduction to test providers can be found in [this](../code_samples/custom_tests/integrate_external_test_providers.ipynb) notebook.\n",
- "\n",
- "\n",
- "\n",
- "### Initializing a local test provider\n",
- "\n",
- "For most use-cases, the local test provider should be sufficient. This test provider allows you load custom tests from a designated directory. Let's go ahead and see how we can do this with our custom tests.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 62,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "from validmind.tests import LocalTestProvider\n",
- "\n",
- "# initialize the test provider with the tests folder we created earlier\n",
- "my_test_provider = LocalTestProvider(tests_folder)\n",
- "\n",
- "vm.tests.register_test_provider(\n",
- " namespace=\"my_test_provider\",\n",
- " test_provider=my_test_provider,\n",
- ")\n",
- "# `my_test_provider.load_test()` will be called for any test ID that starts with `my_test_provider`\n",
- "# e.g. `my_test_provider.ConfusionMatrix` will look for a function named `ConfusionMatrix` in `my_tests/ConfusionMatrix.py` file"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "#### Run test provider tests\n",
- "\n",
- "Now that you have set up the test provider, you can run any test that's located in the tests folder by using the `run_test()` method as with any other test. For tests that reside in a test provider directory, the test ID will be the `namespace` specified when registering the provider, followed by the path to the test file relative to the tests folder. For example, the Confusion Matrix test we created earlier will have the test ID `my_test_provider.ConfusionMatrix`. You could organize the tests in subfolders, say `classification` and `regression`, and the test ID for the Confusion Matrix test would then be `my_test_provider.classification.ConfusionMatrix`.\n",
- "\n",
- "Let's go ahead and re-run the confusion matrix test by using the test ID `my_test_provider.ConfusionMatrix`. This should load the test from the test provider and run it as before.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "result = vm.tests.run_test(\n",
- " \"my_test_provider.ConfusionMatrix\",\n",
- " inputs={\"model\": vm_model, \"dataset\": vm_test_ds},\n",
- " params={\"normalize\": True},\n",
- ")\n",
- "\n",
- "result.log()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "#### Add the test results to your documentation\n",
- "\n",
- "You have already seen how to add individual results to the model documentation using the ValidMind Platform. Let's repeat the process and add the confusion matrix to the `Model Development` -> `Model Evaluation` section of the documentation. The \"add test driven block\" dialog should now show the new test result coming from the test provider:\n",
- "\n",
- "\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "## 4. Finalize testing and documentation\n",
- "\n",
- "In this section we cover how to finalize the testing and documentation of your model by focusing on:\n",
- "\n",
- "1. Using `run_documentation_tests()` to ensure custom test results are included in your documentation\n",
- "2. Viewing and updating the configuration for the entire model documentation template\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Use `run_documentation_tests()` to ensure custom test results are included in your documentation\n",
- "\n",
- "After adding test driven blocks to your model documentation, changes should persist and become available every time you call `vm.preview_template()`. However, you need to reload the connection to the ValidMind Platform if you have added test driven blocks when the connection was already established.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 64,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "vm.reload()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now, run `preview_template()` and verify that the new confusion matrix test you added is included in the proper section.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "vm.preview_template()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Since the test ID is now registered in the document you can now run tests for an entire section and all additional custom tests should be loaded without issues. Let's run all tests in the `model_evaluation` section of the documentation. Note that we have been running the sample custom confusion matrix with `normalize=True` to demonstrate the ability to provide custom parameters.\n",
- "\n",
- "In the `Run the model evaluation tests` section above you learned how to assign inputs to individual tests with `run_documentation_tests()`. Assigning parametesr is similar, you only need to provide assign a `params` dictionary to a given test ID, `my_test_provider.ConfusionMatrix` in this case.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "test_config = {\n",
- " \"validmind.model_validation.sklearn.ClassifierPerformance:in_sample\": {\n",
- " \"inputs\": {\n",
- " \"dataset\": vm_train_ds,\n",
- " \"model\": vm_model,\n",
- " },\n",
- " },\n",
- " \"my_test_provider.ConfusionMatrix\": {\n",
- " \"params\": {\"normalize\": True},\n",
- " },\n",
- "}\n",
- "results = vm.run_documentation_tests(\n",
- " section=[\"model_evaluation\"],\n",
- " inputs={\n",
- " \"dataset\": vm_test_ds, # Any test that requires a single dataset will use vm_test_ds\n",
- " \"model\": vm_model,\n",
- " \"datasets\": (\n",
- " vm_train_ds,\n",
- " vm_test_ds,\n",
- " ), # Any test that requires multiple datasets will use vm_train_ds and vm_test_ds\n",
- " },\n",
- " config=test_config,\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "### Viewing and updating the configuration for the entire model documentation template\n",
- "\n",
- "The ValidMind Library provides a utility function called `vm.get_test_suite().get_default_config()` that allows you to render the default configuration for the entire documentation template. This configuration will contain all the test IDs and their default parameters. You can then modify this configuration as needed and pass it to `run_documentation_tests()` to run all tests in the documentation template if needed. You also have the option to continue running tests for one section at a time, `get_default_config()` still provides a useful reference for providing default parametes to every test.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "import json\n",
- "\n",
- "model_test_suite = vm.get_test_suite()\n",
- "config = model_test_suite.get_default_config()\n",
- "print(\"Suite Config: \\n\", json.dumps(config, indent=2))"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "#### Update the config\n",
- "\n",
- "Note that the default config does not assign any inputs to a test, this is expected. You can assign inputs to individual tests as needed, depending on the datasets and models you want to pass to individual tests. The `config` dictionary, as a mapping of test IDs to test configurations, allows you to do this.\n",
- "\n",
- "For this particular documentation template (binary classification), the ValidMind Library provides a sample configuration that can be used to populate the entire model documentation using the following inputs as placeholders:\n",
- "\n",
- "- A `raw_dataset` raw dataset\n",
- "- A `train_dataset` training dataset\n",
- "- A `test_dataset` test dataset\n",
- "- A trained `model` instance\n",
- "\n",
- "As part of updating the `config` you will need to ensure the correct `input_id`s are used in the final config passed to `run_documentation_tests()`.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "from validmind.datasets.classification import customer_churn\n",
- "from validmind.utils import preview_test_config\n",
- "\n",
- "test_config = customer_churn.get_demo_test_config()\n",
- "preview_test_config(test_config)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Using this sample configuration, let's finish populating model documentation by running all tests for the `model_development` section of the documentation. Recall that the training and test datasets in our exercise have the following `input_id` values:\n",
- "\n",
- "- `train_dataset_final` for the training dataset\n",
- "- `test_dataset_final` for the test dataset\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "metadata": {}
- },
- "outputs": [],
- "source": [
- "config = {\n",
- " \"validmind.model_validation.ModelMetadata\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\"},\n",
- " },\n",
- " \"validmind.data_validation.DatasetSplit\": {\n",
- " \"inputs\": {\"datasets\": [\"train_dataset_final\", \"test_dataset_final\"]},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.PopulationStabilityIndex\": {\n",
- " \"inputs\": {\n",
- " \"model\": \"log_reg_model_v1\",\n",
- " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n",
- " },\n",
- " \"params\": {\"num_bins\": 10, \"mode\": \"fixed\"},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.ConfusionMatrix\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n",
- " },\n",
- " \"my_test_provider.ConfusionMatrix\": {\n",
- " \"inputs\": {\"dataset\": \"test_dataset_final\", \"model\": \"log_reg_model_v1\"},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.ClassifierPerformance:in_sample\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"train_dataset_final\"}\n",
- " },\n",
- " \"validmind.model_validation.sklearn.ClassifierPerformance:out_of_sample\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"}\n",
- " },\n",
- " \"validmind.model_validation.sklearn.PrecisionRecallCurve\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.ROCCurve\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.TrainingTestDegradation\": {\n",
- " \"inputs\": {\n",
- " \"model\": \"log_reg_model_v1\",\n",
- " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n",
- " },\n",
- " \"params\": {\n",
- " \"metrics\": [\"accuracy\", \"precision\", \"recall\", \"f1\"],\n",
- " \"max_threshold\": 0.1,\n",
- " },\n",
- " },\n",
- " \"validmind.model_validation.sklearn.MinimumAccuracy\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n",
- " \"params\": {\"min_threshold\": 0.7},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.MinimumF1Score\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n",
- " \"params\": {\"min_threshold\": 0.5},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.MinimumROCAUCScore\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n",
- " \"params\": {\"min_threshold\": 0.5},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.PermutationFeatureImportance\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.SHAPGlobalImportance\": {\n",
- " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n",
- " \"params\": {\"kernel_explainer_samples\": 10},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.WeakspotsDiagnosis\": {\n",
- " \"inputs\": {\n",
- " \"model\": \"log_reg_model_v1\",\n",
- " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n",
- " },\n",
- " \"params\": {\n",
- " \"thresholds\": {\"accuracy\": 0.75, \"precision\": 0.5, \"recall\": 0.5, \"f1\": 0.7}\n",
- " },\n",
- " },\n",
- " \"validmind.model_validation.sklearn.OverfitDiagnosis\": {\n",
- " \"inputs\": {\n",
- " \"model\": \"log_reg_model_v1\",\n",
- " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n",
- " },\n",
- " \"params\": {\"cut_off_percentage\": 4},\n",
- " },\n",
- " \"validmind.model_validation.sklearn.RobustnessDiagnosis\": {\n",
- " \"inputs\": {\n",
- " \"model\": \"log_reg_model_v1\",\n",
- " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n",
- " },\n",
- " \"params\": {\n",
- " \"scaling_factor_std_dev_list\": [0.0, 0.1, 0.2, 0.3, 0.4, 0.5],\n",
- " \"accuracy_decay_threshold\": 4,\n",
- " },\n",
- " },\n",
- "}\n",
- "\n",
- "\n",
- "full_suite = vm.run_documentation_tests(\n",
- " section=\"model_development\",\n",
- " config=config,\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "## Where to go from here\n",
- "\n",
- "In this notebook you have learned the end-to-end process to document a model with the ValidMind Library, running through some very common scenarios in a typical model development setting:\n",
- "\n",
- "- Running out-of-the-box tests\n",
- "- Documenting your model by adding evidence to model documentation\n",
- "- Extending the capabilities of the ValidMind Library by implementing custom tests\n",
- "- Ensuring that the documentation is complete by running all tests in the documentation template\n",
- "\n",
- "As a next step, you can explore the following notebooks to get a deeper understanding on how the ValidMind Library allows you generate model documentation for any use case:\n",
- "\n",
- "\n",
- "\n",
- "### Use cases\n",
- "\n",
- "- [Application scorecard demo](../code_samples/credit_risk/application_scorecard_demo.ipynb)\n",
- "- [Linear regression documentation demo](../code_samples/regression/quickstart_regression_full_suite.ipynb)\n",
- "- [LLM model documentation demo](../code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb)\n",
- "\n",
- "\n",
- "\n",
- "### More how-to guides and code samples\n",
- "\n",
- "- [Explore available tests in detail](../how_to/explore_tests.ipynb)\n",
- "- [In-depth guide for implementing custom tests](../code_samples/custom_tests/implement_custom_tests.ipynb)\n",
- "- [In-depth guide to external test providers](../code_samples/custom_tests/integrate_external_test_providers.ipynb)\n",
- "- [Configuring dataset features](../how_to/configure_dataset_features.ipynb)\n",
- "- [Introduction to unit and composite metrics](../how_to/run_unit_metrics.ipynb)\n",
- "\n",
- "\n",
- "\n",
- "### Discover more learning resources\n",
- "\n",
- "All notebook samples can be found in the following directories of the ValidMind Library GitHub repository:\n",
- "\n",
- "- [Code samples](https://github.com/validmind/validmind-library/tree/main/notebooks/code_samples)\n",
- "- [How-to guides](https://github.com/validmind/validmind-library/tree/main/notebooks/how_to)\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "\n",
- "## Upgrade ValidMind\n",
- "\n",
- "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n",
- "\n",
- "Retrieve the information for the currently installed version of ValidMind:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "%pip show validmind"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n",
- "\n",
- "```bash\n",
- "%pip install --upgrade validmind\n",
- "```"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "You may need to restart your kernel after running the upgrade package for changes to be applied."
+ "Training is FREE — Register now!
"
]
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "ValidMind Library",
"language": "python",
- "name": "python3"
+ "name": "validmind"
},
"language_info": {
"codemirror_mode": {
@@ -1798,7 +35,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.5"
+ "version": "3.10.13"
}
},
"nbformat": 4,
diff --git a/notebooks/tutorials/model_development/101-set_up_validmind.ipynb b/notebooks/tutorials/model_development/1-set_up_validmind.ipynb
similarity index 76%
rename from notebooks/tutorials/model_development/101-set_up_validmind.ipynb
rename to notebooks/tutorials/model_development/1-set_up_validmind.ipynb
index 462097c55..46a002a83 100644
--- a/notebooks/tutorials/model_development/101-set_up_validmind.ipynb
+++ b/notebooks/tutorials/model_development/1-set_up_validmind.ipynb
@@ -2,10 +2,10 @@
"cells": [
{
"cell_type": "markdown",
- "id": "4a4933a6",
+ "id": "b6fa2ac0",
"metadata": {},
"source": [
- "# ValidMind for model development — 101 Set up the ValidMind Library\n",
+ "# ValidMind for model development 1 — Set up the ValidMind Library\n",
"\n",
"Learn how to use ValidMind for your end-to-end model documentation process based on common model development scenarios with our series of four introductory notebooks. This first notebook walks you through the initial setup of the ValidMind Library.\n",
"\n",
@@ -14,26 +14,28 @@
},
{
"cell_type": "markdown",
- "id": "eb8f781f",
+ "id": "fe2e0eca",
"metadata": {},
"source": [
"::: {.content-hidden when-format=\"html\"}\n",
"## Contents \n",
- "- [About ValidMind](#toc1_) \n",
- " - [Before you begin](#toc1_1_) \n",
- " - [New to ValidMind?](#toc1_2_) \n",
- " - [Key concepts](#toc1_3_) \n",
- "- [Initializing the ValidMind Library](#toc2_) \n",
- " - [Install the ValidMind Library](#toc2_1_) \n",
- " - [Initialize the ValidMind Library](#toc2_2_) \n",
- " - [Get your code snippet](#toc2_2_1_) \n",
- "- [Getting to know ValidMind](#toc3_) \n",
- " - [Preview the documentation template](#toc3_1_) \n",
- " - [Explore available tests](#toc3_2_) \n",
- "- [Upgrade ValidMind](#toc4_) \n",
- "- [In summary](#toc5_) \n",
- "- [Next steps](#toc6_) \n",
- " - [Start the model development process](#toc6_1_) \n",
+ "- [Introduction](#toc1_) \n",
+ "- [About ValidMind](#toc2_) \n",
+ " - [Before you begin](#toc2_1_) \n",
+ " - [New to ValidMind?](#toc2_2_) \n",
+ " - [Key concepts](#toc2_3_) \n",
+ "- [Initializing the ValidMind Library](#toc3_) \n",
+ " - [Install the ValidMind Library](#toc3_1_) \n",
+ " - [Initialize the ValidMind Library](#toc3_2_) \n",
+ " - [Get your code snippet](#toc3_2_1_) \n",
+ "- [Getting to know ValidMind](#toc4_) \n",
+ " - [Preview the documentation template](#toc4_1_) \n",
+ " - [View model documentation in the ValidMind Platform](#toc4_1_1_) \n",
+ " - [Explore available tests](#toc4_2_) \n",
+ "- [Upgrade ValidMind](#toc5_) \n",
+ "- [In summary](#toc6_) \n",
+ "- [Next steps](#toc7_) \n",
+ " - [Start the model development process](#toc7_1_) \n",
"\n",
":::\n",
"\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f1d4715f",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Introduction\n",
+ "\n",
+ "Model validation aims to independently assess the compliance of *champion models* created by model developers with regulatory guidance by conducting thorough testing and analysis, potentially including the use of challenger models to benchmark performance. Assessments, presented in the form of a validation report, typically include *model findings* and recommendations to address those issues.\n",
+ "\n",
+ "A *binary classification model* is a type of predictive model used in churn analysis to identify customers who are likely to leave a service or subscription by analyzing various behavioral, transactional, and demographic factors.\n",
+ "\n",
+ "- This model helps businesses take proactive measures to retain at-risk customers by offering personalized incentives, improving customer service, or adjusting pricing strategies.\n",
+ "- Effective validation of a churn prediction model ensures that businesses can accurately identify potential churners, optimize retention efforts, and enhance overall customer satisfaction while minimizing revenue loss."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "14c2d80d",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## About ValidMind\n",
+ "\n",
+ "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n",
+ "\n",
+ "You use the ValidMind Library to automate comparison and other validation tests, and then use the ValidMind Platform to submit compliance assessments of champion models via comprehensive validation reports. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model developers."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "151a4ca5",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Before you begin\n",
+ "\n",
+ "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n",
+ "\n",
+ "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "089c960e",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### New to ValidMind?\n",
+ "\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
+ "\n",
+ "
For access to all features available in this notebook, create a free ValidMind account.\n",
+ " \n",
+ "Signing up is FREE — Register with ValidMind
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5f307177",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Key concepts\n",
+ "\n",
+ "**Validation report**: A comprehensive and structured assessment of a model’s development and performance, focusing on verifying its integrity, appropriateness, and alignment with its intended use. It includes analyses of model assumptions, data quality, performance metrics, outcomes of testing procedures, and risk considerations. The validation report supports transparency, regulatory compliance, and informed decision-making by documenting the validator’s independent review and conclusions.\n",
+ "\n",
+ "**Validation report template**: Serves as a standardized framework for conducting and documenting model validation activities. It outlines the required sections, recommended analyses, and expected validation tests, ensuring consistency and completeness across validation reports. The template helps guide validators through a systematic review process while promoting comparability and traceability of validation outcomes.\n",
+ "\n",
+ "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets.\n",
+ "\n",
+ "**Metrics**: A subset of tests that do not have thresholds. In the context of this notebook, metrics and tests can be thought of as interchangeable concepts.\n",
+ "\n",
+ "**Custom metrics**: Custom metrics are functions that you define to evaluate your model or dataset. These functions can be registered with the ValidMind Library to be used in the ValidMind Platform.\n",
+ "\n",
+ "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n",
+ "\n",
+ " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n",
+ " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n",
+ " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom metric.\n",
+ " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom metric. (Learn more: [Run tests with multiple datasets](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html))\n",
+ "\n",
+ "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a metric, customize its behavior, or provide additional context.\n",
+ "\n",
+ "**Outputs**: Custom metrics can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c42665b8",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Setting up"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0faed42c",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Register a sample model\n",
+ "\n",
+ "In a usual model lifecycle, a champion model will have been independently registered in your model inventory and submitted to you for validation by your model development team as part of the effective challenge process. (**Learn more:** [Submit for approval](https://docs.validmind.ai/guide/model-documentation/submit-for-approval.html))\n",
+ "\n",
+ "For this series of notebooks, we'll have you register a dummy model in the ValidMind Platform inventory and assign yourself as the validator to familiarize you with the ValidMind interface and circumvent the need for an existing model:\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n",
+ "\n",
+ "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n",
+ "\n",
+ " For example, to register a model for use with this notebook, select:\n",
+ "\n",
+ " - Documentation template: `Binary classification`\n",
+ " - Use case: `Marketing/Sales - Attrition/Churn Management`\n",
+ "\n",
+ " You can fill in other options according to your preference."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0c350e0d",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Assign validator credentials\n",
+ "\n",
+ "In order to log tests as a validator instead of as a developer, on the model details page that appears after you've successfully registered your sample model:\n",
+ "\n",
+ "1. Remove yourself as a developer: \n",
+ "\n",
+ " - Click on the **DEVELOPERS** tile.\n",
+ " - Click the **x** next to your name to remove yourself from that model's role.\n",
+ " - Click **Save** to apply your changes to that role.\n",
+ "\n",
+ "2. Add yourself as a validator: \n",
+ "\n",
+ " - Click on the **VALIDATORS** tile.\n",
+ " - Select your name from the drop-down menu.\n",
+ " - Click **Save** to apply your changes to that role."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Install the ValidMind Library\n",
+ "\n",
+ "
\n",
+ "\n",
+ "To install the library:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "931d8f7f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip install -q validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5ec7fcb7",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the ValidMind Library\n",
+ "\n",
+ "ValidMind generates a unique _code snippet_ for each registered model to connect with your validation environment. You initialize the ValidMind Library with this code snippet, which ensures that your test results are uploaded to the correct model when you run the notebook."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Get your code snippet\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n",
+ "\n",
+ "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
+ "\n",
+ "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d5d87e2d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load your model identifier credentials from an `.env` file\n",
+ "\n",
+ "%load_ext dotenv\n",
+ "%dotenv .env\n",
+ "\n",
+ "# Or replace with your code snippet\n",
+ "\n",
+ "import validmind as vm\n",
+ "\n",
+ "vm.init(\n",
+ " # api_host=\"...\",\n",
+ " # api_key=\"...\",\n",
+ " # api_secret=\"...\",\n",
+ " # model=\"...\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b4b5a00f",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Getting to know ValidMind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Preview the validation report template\n",
+ "\n",
+ "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for model validation. A template predefines sections for your validation report and provides a general outline to follow, making the validation process much easier.\n",
+ "\n",
+ "You will attach evidence to this template in the form of risk assessment notes, findings, and test results later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "13d34bbb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.preview_template()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a2e86bc8",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### View validation report in the ValidMind Platform\n",
+ "\n",
+ "Next, let's head to the ValidMind Platform to see the template in action:\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n",
+ "\n",
+ "3. Click on the **Validation Report** for your model and note:\n",
+ "\n",
+ " - [x] The risk assessment compliance summary at the top of the report (screenshot below)\n",
+ " - [x] How the structure of the validation report reflects the previewed template\n",
+ "\n",
+ " \n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Explore available tests\n",
+ "\n",
+ "Next, let's explore the list of all available tests in the ValidMind Library with [the `vm.tests.list_tests()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) — we'll later narrow down the tests we want to run from this list when we learn to run tests."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "de6abc2a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.list_tests()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Upgrade ValidMind\n",
+ "\n",
+ "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n",
+ "\n",
+ "Retrieve the information for the currently installed version of ValidMind:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "10272aa9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip show validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "upgrade-version-d64591ca-3073-4b3e-9586-d3577adda203",
+ "metadata": {},
+ "source": [
+ "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n",
+ "\n",
+ "```bash\n",
+ "%pip install --upgrade validmind\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "44657dea",
+ "metadata": {},
+ "source": [
+ "You may need to restart your kernel after running the upgrade package for changes to be applied."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "39f45f58",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## In summary\n",
+ "\n",
+ "In this first notebook, you learned how to:\n",
+ "\n",
+ "- [x] Register a model within the ValidMind Platform and assign yourself as the validator\n",
+ "- [x] Install and initialize the ValidMind Library\n",
+ "- [x] Preview the validation report template for your model\n",
+ "- [x] Explore the available tests offered by the ValidMind Library\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Next steps\n",
+ "\n",
+ "\n",
+ "\n",
+ "### Start the model validation process\n",
+ "\n",
+ "Now that the ValidMind Library is connected to your model in the ValidMind Library with the correct template applied, we can go ahead and start the model validation process: **[2 — Start the model validation process](2-start_validation_process.ipynb)**"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "ValidMind Library",
+ "language": "python",
+ "name": "validmind"
+ },
+ "language_info": {
+ "name": "python",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/tutorials/model_validation/2-start_validation_process.ipynb b/notebooks/tutorials/model_validation/2-start_validation_process.ipynb
new file mode 100644
index 000000000..5493f1f9c
--- /dev/null
+++ b/notebooks/tutorials/model_validation/2-start_validation_process.ipynb
@@ -0,0 +1,873 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# ValidMind for model validation 2 — Start the model validation process\n",
+ "\n",
+ "Learn how to use ValidMind for your end-to-end model validation process with our series of four introductory notebooks. In this second notebook, independently verify the data quality tests performed on the dataset used to train the champion model.\n",
+ "\n",
+ "You'll learn how to run relevant validation tests with ValidMind, log the results of those tests to the ValidMind Platform, and insert your logged test results as evidence into your validation report. You'll become familiar with the tests available in ValidMind, as well as how to run them. Running tests during model validation is crucial to the effective challenge process, as we want to independently evaluate the evidence and assessments provided by the model development team.\n",
+ "\n",
+ "While running our tests in this notebook, we'll focus on:\n",
+ "\n",
+ "- Ensuring that data used for training and testing the model is of appropriate data quality\n",
+ "- Ensuring that the raw data has been preprocessed appropriately and that the resulting final datasets reflects this\n",
+ "\n",
+ "**For a full list of out-of-the-box tests,** refer to our [Test descriptions](https://docs.validmind.ai/developer/model-testing/test-descriptions.html) or try the interactive [Test sandbox](https://docs.validmind.ai/developer/model-testing/test-sandbox.html)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "::: {.content-hidden when-format=\"html\"}\n",
+ "## Contents \n",
+ "- [Prerequisites](#toc1_) \n",
+ "- [Setting up](#toc2_) \n",
+ " - [Initialize the ValidMind Library](#toc2_1_) \n",
+ "- [Load the sample dataset](#toc3_) \n",
+ "- [Verifying data quality adjustments](#toc4_) \n",
+ " - [Identify qualitative tests](#toc4_1_) \n",
+ " - [Initialize the ValidMind datasets](#toc4_2_) \n",
+ " - [Run data quality tests](#toc4_3_) \n",
+ " - [Run tabular data tests](#toc4_3_1_) \n",
+ " - [Remove highly correlated features](#toc4_4_) \n",
+ "- [Documenting test results](#toc5_) \n",
+ " - [Configure and run comparison tests](#toc5_1_) \n",
+ " - [Log tests with a unique identifiers](#toc5_2_) \n",
+ " - [Add test results to reporting](#toc5_3_) \n",
+ "- [Split the preprocessed dataset](#toc6_) \n",
+ " - [Initialize the split datasets](#toc6_1_) \n",
+ "- [In summary](#toc7_) \n",
+ "- [Next steps](#toc8_) \n",
+ " - [Develop potential challenger models](#toc8_1_) \n",
+ "\n",
+ ":::\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Prerequisites\n",
+ "\n",
+ "In order to independently assess the quality of your datasets with notebook, you'll need to first have:\n",
+ "\n",
+ "- [x] Registered a model within the ValidMind Platform and granted yourself access to the model as a validator\n",
+ "- [x] Installed the ValidMind Library in your local environment, allowing you to access all its features\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Setting up"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the ValidMind Library\n",
+ "\n",
+ "First, let's connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n",
+ "\n",
+ "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
+ "\n",
+ "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Make sure the ValidMind Library is installed\n",
+ "\n",
+ "%pip install -q validmind\n",
+ "\n",
+ "# Load your model identifier credentials from an `.env` file\n",
+ "\n",
+ "%load_ext dotenv\n",
+ "%dotenv .env\n",
+ "\n",
+ "# Or replace with your code snippet\n",
+ "\n",
+ "import validmind as vm\n",
+ "\n",
+ "vm.init(\n",
+ " # api_host=\"...\",\n",
+ " # api_key=\"...\",\n",
+ " # api_secret=\"...\",\n",
+ " # model=\"...\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Load the sample dataset\n",
+ "\n",
+ "Let's first import the public [Bank Customer Churn Prediction](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction) dataset from Kaggle, which was used to develop the dummy champion model.\n",
+ "\n",
+ "We'll use this dataset to review steps that should have been conducted during the initial development and documentation of the model to ensure that the model was built correctly. By independently performing steps taken by the model development team, we can confirm whether the model was built using appropriate and properly processed data.\n",
+ "\n",
+ "In our below example, note that:\n",
+ "\n",
+ "- The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n",
+ "- The ValidMind Library provides a wrapper to automatically load the dataset as a Pandas [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from validmind.datasets.classification import customer_churn as demo_dataset\n",
+ "\n",
+ "print(\n",
+ " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n",
+ ")\n",
+ "\n",
+ "raw_df = demo_dataset.load_data()\n",
+ "raw_df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Verifying data quality adjustments\n",
+ "\n",
+ "Let's say that thanks to the documentation submitted by the model development team ([Learn more ...](https://docs.validmind.ai/developer/validmind-library.html#for-model-development)), we know that the sample dataset was first modified before being used to train the champion model. After performing some data quality assessments on the raw dataset, it was determined that the dataset required rebalancing, and highly correlated features were also removed."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Identify qualitative tests\n",
+ "\n",
+ "During model validation, we use the same data processing logic and training procedure to confirm that the model's results can be reproduced independently, so let's start by doing some data quality assessments by running a few individual tests just like the development team did.\n",
+ "\n",
+ "Use the [`vm.tests.list_tests()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) introduced by the first notebook in this series in combination with [`vm.tests.list_tags()`](https://docs.validmind.ai/validmind/validmind/tests.html#list_tags) and [`vm.tests.list_tasks()`](https://docs.validmind.ai/validmind/validmind/tests.html#list_tasks) to find which prebuilt tests are relevant for data quality assessment:\n",
+ "\n",
+ "- **`tasks`** represent the kind of modeling task associated with a test. Here we'll focus on `classification` tasks.\n",
+ "- **`tags`** are free-form descriptions providing more details about the test, for example, what category the test falls into. Here we'll focus on the `data_quality` tag."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get the list of available task types\n",
+ "sorted(vm.tests.list_tasks())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get the list of available tags\n",
+ "sorted(vm.tests.list_tags())"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "You can pass `tags` and `tasks` as parameters to the `vm.tests.list_tests()` function to filter the tests based on the tags and task types.\n",
+ "\n",
+ "For example, to find tests related to tabular data quality for classification models, you can call `list_tests()` like this:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.list_tests(task=\"classification\", tags=[\"tabular_data\", \"data_quality\"])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "
Want to learn more about navigating ValidMind tests?\n",
+ " \n",
+ "Refer to our notebook outlining the utilities available for viewing and understanding available ValidMind tests: Explore tests
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the ValidMind datasets\n",
+ "\n",
+ "With the individual tests we want to run identified, the next step is to connect your data with a ValidMind `Dataset` object. **This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind,** but you only need to do it once per dataset.\n",
+ "\n",
+ "Initialize a ValidMind dataset object using the [`init_dataset` function](https://docs.validmind.ai/validmind/validmind.html#init_dataset) from the ValidMind (`vm`) module. For this example, we'll pass in the following arguments:\n",
+ "\n",
+ "- **`dataset`** — The raw dataset that you want to provide as input to tests.\n",
+ "- **`input_id`** — A unique identifier that allows tracking what inputs are used when running each individual test.\n",
+ "- **`target_column`** — A required argument if tests require access to true values. This is the name of the target column in the dataset."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# vm_raw_dataset is now a VMDataset object that you can pass to any ValidMind test\n",
+ "vm_raw_dataset = vm.init_dataset(\n",
+ " dataset=raw_df,\n",
+ " input_id=\"raw_dataset\",\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Run data quality tests\n",
+ "\n",
+ "Now that we know how to initialize a ValidMind `dataset` object, we're ready to run some tests!\n",
+ "\n",
+ "You run individual tests by calling [the `run_test` function](https://docs.validmind.ai/validmind/validmind/tests.html#run_test) provided by the `validmind.tests` module. For the examples below, we'll pass in the following arguments:\n",
+ "\n",
+ "- **`test_id`** — The ID of the test to run, as seen in the `ID` column when you run `list_tests`. \n",
+ "- **`params`** — A dictionary of parameters for the test. These will override any `default_params` set in the test definition. "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Run tabular data tests\n",
+ "\n",
+ "The inputs expected by a test can also be found in the test definition — let's take [`validmind.data_validation.DescriptiveStatistics`](https://docs.validmind.ai/tests/data_validation/DescriptiveStatistics.html) as an example.\n",
+ "\n",
+ "Note that the output of the [`describe_test()` function](https://docs.validmind.ai/validmind/validmind/tests.html#describe_test) below shows that this test expects a `dataset` as input:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.describe_test(\"validmind.data_validation.DescriptiveStatistics\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now, let's run a few tests to assess the quality of the dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result2 = vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.ClassImbalance\",\n",
+ " inputs={\"dataset\": vm_raw_dataset},\n",
+ " params={\"min_percent_threshold\": 30},\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The output above shows that [the class imbalance test](https://docs.validmind.ai/tests/data_validation/ClassImbalance.html) did not pass according to the value we set for `min_percent_threshold` — great, this matches what was reported by the model development team.\n",
+ "\n",
+ "To address this issue, we'll re-run the test on some processed data. In this case let's apply a very simple rebalancing technique to the dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "raw_copy_df = raw_df.sample(frac=1) # Create a copy of the raw dataset\n",
+ "\n",
+ "# Create a balanced dataset with the same number of exited and not exited customers\n",
+ "exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 1]\n",
+ "not_exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 0].sample(n=exited_df.shape[0])\n",
+ "\n",
+ "balanced_raw_df = pd.concat([exited_df, not_exited_df])\n",
+ "balanced_raw_df = balanced_raw_df.sample(frac=1, random_state=42)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "With this new balanced dataset, you can re-run the individual test to see if it now passes the class imbalance test requirement.\n",
+ "\n",
+ "As this is technically a different dataset, **remember to first initialize a new ValidMind `Dataset` object** to pass in as input as required by `run_test()`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Register new data and now 'balanced_raw_dataset' is the new dataset object of interest\n",
+ "vm_balanced_raw_dataset = vm.init_dataset(\n",
+ " dataset=balanced_raw_df,\n",
+ " input_id=\"balanced_raw_dataset\",\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Pass the initialized `balanced_raw_dataset` as input into the test run\n",
+ "result = vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.ClassImbalance\",\n",
+ " inputs={\"dataset\": vm_balanced_raw_dataset},\n",
+ " params={\"min_percent_threshold\": 30},\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Remove highly correlated features\n",
+ "\n",
+ "Next, let's also remove highly correlated features from our dataset as outlined by the development team. Removing highly correlated features helps make the model simpler, more stable, and easier to understand.\n",
+ "\n",
+ "You can utilize the output from a ValidMind test for further use — in this below example, to retrieve the list of features with the highest correlation coefficients and use them to reduce the final list of features for modeling.\n",
+ "\n",
+ "First, we'll run [`validmind.data_validation.HighPearsonCorrelation`](https://docs.validmind.ai/tests/data_validation/HighPearsonCorrelation.html) with the `balanced_raw_dataset` we initialized previously as input as is for comparison with later runs:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "corr_result = vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n",
+ " params={\"max_threshold\": 0.3},\n",
+ " inputs={\"dataset\": vm_balanced_raw_dataset},\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The output above shows that the test did not pass according to the value we set for `max_threshold` — as reported and expected.\n",
+ "\n",
+ "`corr_result` is an object of type `TestResult`. We can inspect the result object to see what the test has produced:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(type(corr_result))\n",
+ "print(\"Result ID: \", corr_result.result_id)\n",
+ "print(\"Params: \", corr_result.params)\n",
+ "print(\"Passed: \", corr_result.passed)\n",
+ "print(\"Tables: \", corr_result.tables)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's remove the highly correlated features and create a new VM `dataset` object.\n",
+ "\n",
+ "We'll begin by checking out the table in the result and extracting a list of features that failed the test:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract table from `corr_result.tables`\n",
+ "features_df = corr_result.tables[0].data\n",
+ "features_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract list of features that failed the test\n",
+ "high_correlation_features = features_df[features_df[\"Pass/Fail\"] == \"Fail\"][\"Columns\"].tolist()\n",
+ "high_correlation_features"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Next, extract the feature names from the list of strings (example: `(Age, Exited)` > `Age`):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "high_correlation_features = [feature.split(\",\")[0].strip(\"()\") for feature in high_correlation_features]\n",
+ "high_correlation_features"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now, it's time to re-initialize the dataset with the highly correlated features removed.\n",
+ "\n",
+ "**Note the use of a different `input_id`.** This allows tracking the inputs used when running each individual test."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Remove the highly correlated features from the dataset\n",
+ "balanced_raw_no_age_df = balanced_raw_df.drop(columns=high_correlation_features)\n",
+ "\n",
+ "# Re-initialize the dataset object\n",
+ "vm_raw_dataset_preprocessed = vm.init_dataset(\n",
+ " dataset=balanced_raw_no_age_df,\n",
+ " input_id=\"raw_dataset_preprocessed\",\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Re-running the test with the reduced feature set should pass the test:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "corr_result = vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n",
+ " params={\"max_threshold\": 0.3},\n",
+ " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "You can also plot the correlation matrix to visualize the new correlation between features:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "corr_result = vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.PearsonCorrelationMatrix\",\n",
+ " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Documenting test results\n",
+ "\n",
+ "Now that we've done some analysis on two different datasets, we can use ValidMind to easily document why certain things were done to our raw data with testing to support it. As we learned above, every test result returned by the `run_test()` function has a `.log()` method that can be used to send the test results to the ValidMind Platform.\n",
+ "\n",
+ "When logging validation test results to the platform, you'll need to manually add those results to the desired section of the validation report. To demonstrate how to add test results to your validation report, we'll log our data quality tests and insert the results via the ValidMind Platform."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Configure and run comparison tests\n",
+ "\n",
+ "Below, we'll perform comparison tests between the original raw dataset (`raw_dataset`) and the final preprocessed (`raw_dataset_preprocessed`) dataset, again logging the results to the ValidMind Platform. \n",
+ "\n",
+ "We can specify all the tests we'd ike to run in a dictionary called `test_config`, and we'll pass in the following arguments for each test:\n",
+ "\n",
+ " - **`params`:** Individual test parameters.\n",
+ " - **`input_grid`:** Individual test inputs to compare. In this case, we'll input our two datasets for comparison.\n",
+ "\n",
+ "**Note here that the `input_grid` expects the `input_id` of the dataset as the value rather than the variable name we specified:**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Individual test config with inputs specified\n",
+ "test_config = {\n",
+ " \"validmind.data_validation.ClassImbalance\": {\n",
+ " \"input_grid\": {\"dataset\": [\"raw_dataset\", \"raw_dataset_preprocessed\"]},\n",
+ " \"params\": {\"min_percent_threshold\": 30}\n",
+ " },\n",
+ " \"validmind.data_validation.HighPearsonCorrelation\": {\n",
+ " \"input_grid\": {\"dataset\": [\"raw_dataset\", \"raw_dataset_preprocessed\"]},\n",
+ " \"params\": {\"max_threshold\": 0.3}\n",
+ " },\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Then batch run and log our tests in `test_config`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for t in test_config:\n",
+ " print(t)\n",
+ " try:\n",
+ " # Check if test has input_grid\n",
+ " if 'input_grid' in test_config[t]:\n",
+ " # For tests with input_grid, pass the input_grid configuration\n",
+ " if 'params' in test_config[t]:\n",
+ " vm.tests.run_test(t, input_grid=test_config[t]['input_grid'], params=test_config[t]['params']).log()\n",
+ " else:\n",
+ " vm.tests.run_test(t, input_grid=test_config[t]['input_grid']).log()\n",
+ " else:\n",
+ " # Original logic for regular inputs\n",
+ " if 'params' in test_config[t]:\n",
+ " vm.tests.run_test(t, inputs=test_config[t]['inputs'], params=test_config[t]['params']).log()\n",
+ " else:\n",
+ " vm.tests.run_test(t, inputs=test_config[t]['inputs']).log()\n",
+ " except Exception as e:\n",
+ " print(f\"Error running test {t}: {str(e)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "
Note the output returned indicating that a test-driven block doesn't currently exist in your model's documentation for some test IDs. \n",
+ " \n",
+ "That's expected, as when we run validations tests the results logged need to be manually added to your report as part of your compliance assessment process within the ValidMind Platform.
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Log tests with a unique identifiers\n",
+ "\n",
+ "Next, we'll use the previously initialized `vm_balanced_raw_dataset` (that still has a highly correlated `Age` column) as input to run an individual test, then log the result to the ValidMind Platform.\n",
+ "\n",
+ "When running individual tests, **you can use a custom `result_id` to tag the individual result with a unique identifier:**\n",
+ "\n",
+ "- This `result_id` can be appended to `test_id` with a `:` separator.\n",
+ "- The `balanced_raw_dataset` result identifier will correspond to the `balanced_raw_dataset` input, the dataset that still has the `Age` column."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.HighPearsonCorrelation:balanced_raw_dataset\",\n",
+ " params={\"max_threshold\": 0.3},\n",
+ " inputs={\"dataset\": vm_balanced_raw_dataset},\n",
+ ")\n",
+ "result.log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Add test results to reporting\n",
+ "\n",
+ "With some test results logged, let's head to the model we connected to at the beginning of this notebook and learn how to insert a test result into our validation report ([Need more help?](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#link-validator-evidence)).\n",
+ "\n",
+ "While the example below focuses on a specific test result, you can follow the same general procedure for your other results:\n",
+ "\n",
+ "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n",
+ "\n",
+ "2. In the left sidebar that appears for your model, click **Validation Report**.\n",
+ "\n",
+ "3. Locate the Data Preparation section and click on **2.2.1. Data Quality** to expand that section.\n",
+ "\n",
+ "4. Under the Class Imbalance Assessment section, locate Validator Evidence then click **Link Evidence to Report**:\n",
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "5. Select the Class Imbalance test results we logged: **ValidMind Data Validation Class Imbalance** \n",
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "6. Click **Update Linked Evidence** to add the test results to the validation report.\n",
+ "\n",
+ " Confirm that the results for the Class Imbalance test you inserted has been correctly inserted into section **2.2.1. Data Quality** of the report:\n",
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "7. Note that these test results are flagged as **Requires Attention** — as they include comparative results from our initial raw dataset.\n",
+ "\n",
+ " Click **See evidence details** to review the LLM-generated description that summarizes the test results, that confirm that our final preprocessed dataset actually passes our test:\n",
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "\n",
+ "
Here in this text editor, you can make qualitative edits to the draft that ValidMind generated to finalize the test results.\n",
+ " \n",
+ "Learn more: Work with content blocks
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Split the preprocessed dataset\n",
+ "\n",
+ "With our raw dataset rebalanced with highly correlated features removed, let's now **spilt our dataset into train and test** in preparation for model evaluation testing.\n",
+ "\n",
+ "To start, let's grab the first few rows from the `balanced_raw_no_age_df` dataset we initialized earlier:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "balanced_raw_no_age_df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Before training the model, we need to encode the categorical features in the dataset:\n",
+ "\n",
+ "- Use the `OneHotEncoder` class from the `sklearn.preprocessing` module to encode the categorical features.\n",
+ "- The categorical features in the dataset are `Geography` and `Gender`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "balanced_raw_no_age_df = pd.get_dummies(\n",
+ " balanced_raw_no_age_df, columns=[\"Geography\", \"Gender\"], drop_first=True\n",
+ ")\n",
+ "balanced_raw_no_age_df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Splitting our dataset into training and testing is essential for proper validation testing, as this helps assess how well the model generalizes to unseen data:\n",
+ "\n",
+ "- We start by dividing our `balanced_raw_no_age_df` dataset into training and test subsets using `train_test_split`, with 80% of the data allocated to training (`train_df`) and 20% to testing (`test_df`).\n",
+ "- From each subset, we separate the features (all columns except \"Exited\") into `X_train` and `X_test`, and the target column (\"Exited\") into `y_train` and `y_test`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn.model_selection import train_test_split\n",
+ "\n",
+ "train_df, test_df = train_test_split(balanced_raw_no_age_df, test_size=0.20)\n",
+ "\n",
+ "X_train = train_df.drop(\"Exited\", axis=1)\n",
+ "y_train = train_df[\"Exited\"]\n",
+ "X_test = test_df.drop(\"Exited\", axis=1)\n",
+ "y_test = test_df[\"Exited\"]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the split datasets\n",
+ "\n",
+ "Next, let's initialize the training and testing datasets so they are available for use:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_train_ds = vm.init_dataset(\n",
+ " input_id=\"train_dataset_final\",\n",
+ " dataset=train_df,\n",
+ " target_column=\"Exited\",\n",
+ ")\n",
+ "\n",
+ "vm_test_ds = vm.init_dataset(\n",
+ " input_id=\"test_dataset_final\",\n",
+ " dataset=test_df,\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## In summary\n",
+ "\n",
+ "In this second notebook, you learned how to:\n",
+ "\n",
+ "- [x] Import a sample dataset\n",
+ "- [x] Identify which tests you might want to run with ValidMind\n",
+ "- [x] Initialize ValidMind datasets\n",
+ "- [x] Run individual tests\n",
+ "- [x] Utilize the output from tests you’ve run\n",
+ "- [x] Log test results as evidence to the ValidMind Platform\n",
+ "- [x] Insert test results into your validation report"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Next steps\n",
+ "\n",
+ "\n",
+ "\n",
+ "### Develop potential challenger models\n",
+ "\n",
+ "Now that you're familiar with the basics of using the ValidMind Library, let's use it to develop a challenger model: **[3 — Developing a potential challenger model](3-developing_challenger_model.ipynb)**"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "ValidMind Library",
+ "language": "python",
+ "name": "validmind"
+ },
+ "language_info": {
+ "name": "python",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb b/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb
new file mode 100644
index 000000000..b0d226012
--- /dev/null
+++ b/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb
@@ -0,0 +1,871 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# ValidMind for model validation 3 — Developing a potential challenger model\n",
+ "\n",
+ "Learn how to use ValidMind for your end-to-end model validation process with our series of four introductory notebooks. In this third notebook, develop a potential challenger model and then pass your model and its predictions to ValidMind.\n",
+ "\n",
+ "A *challenger model* is an alternate model that attempt to outperform the champion model, ensuring that the best performing fit-for-purpose model is always considered for deployment. Challenger models also help avoid over-reliance on a single model, and allow testing of new features, algorithms, or data sources without disrupting the production lifecycle."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "::: {.content-hidden when-format=\"html\"}\n",
+ "## Contents \n",
+ "- [Prerequisites](#toc1_) \n",
+ "- [Setting up](#toc2_) \n",
+ " - [Initialize the ValidMind Library](#toc2_1_) \n",
+ " - [Import the sample dataset](#toc2_2_) \n",
+ " - [Preprocess the dataset](#toc2_2_1_) \n",
+ " - [Split the preprocessed dataset](#toc2_3_) \n",
+ "- [Import the champion model](#toc3_) \n",
+ "- [Training a potential challenger model](#toc4_) \n",
+ " - [Random forest classification model](#toc4_1_) \n",
+ "- [Initializing the model objects](#toc5_) \n",
+ " - [Initialize the model objects](#toc5_1_) \n",
+ " - [Assign predictions](#toc5_2_) \n",
+ "- [Running model validation tests](#toc6_) \n",
+ " - [Run model performance tests](#toc6_1_) \n",
+ " - [Evaluate performance of the champion model](#toc6_1_1_) \n",
+ " - [Log a model finding](#toc6_1_2_) \n",
+ " - [Evaluate performance of challenger model](#toc6_1_3_) \n",
+ " - [Run diagnostic tests](#toc6_2_) \n",
+ " - [Run feature importance tests](#toc6_3_) \n",
+ "- [In summary](#toc7_) \n",
+ "- [Next steps](#toc8_) \n",
+ " - [Finalize validation and reporting](#toc8_1_) \n",
+ "\n",
+ ":::\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Prerequisites\n",
+ "\n",
+ "In order to develop potential challenger models with this notebook, you'll need to first have:\n",
+ "\n",
+ "- [x] Registered a model within the ValidMind Platform and granted yourself access to the model as a validator\n",
+ "- [x] Installed the ValidMind Library in your local environment, allowing you to access all its features\n",
+ "- [x] Learned how to import and initialize datasets for use with ValidMind\n",
+ "- [x] Understood the basics of how to run and log tests with ValidMind\n",
+ "- [x] Run data quality tests on the datasets used to train the champion model, and logged the results of those tests to ValidMind\n",
+ "- [x] Inserted your logged test results into your validation report\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Setting up\n",
+ "\n",
+ "This section should be quite familiar to you — as we performed the same actions in the previous notebook, **[2 — Start the model validation process](2-start_validation_process.ipynb)**."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the ValidMind Library\n",
+ "\n",
+ "As usual, let's first connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n",
+ "\n",
+ "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
+ "\n",
+ "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Make sure the ValidMind Library is installed\n",
+ "\n",
+ "%pip install -q validmind\n",
+ "\n",
+ "# Load your model identifier credentials from an `.env` file\n",
+ "\n",
+ "%load_ext dotenv\n",
+ "%dotenv .env\n",
+ "\n",
+ "# Or replace with your code snippet\n",
+ "\n",
+ "import validmind as vm\n",
+ "\n",
+ "vm.init(\n",
+ " # api_host=\"...\",\n",
+ " # api_key=\"...\",\n",
+ " # api_secret=\"...\",\n",
+ " # model=\"...\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Import the sample dataset\n",
+ "\n",
+ "Next, we'll load in the sample [Bank Customer Churn Prediction](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction) dataset used to develop the champion model that we will independently preprocess:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load the sample dataset\n",
+ "from validmind.datasets.classification import customer_churn as demo_dataset\n",
+ "\n",
+ "print(\n",
+ " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n",
+ ")\n",
+ "\n",
+ "raw_df = demo_dataset.load_data()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Preprocess the dataset\n",
+ "\n",
+ "We’ll apply a simple rebalancing technique to the dataset before continuing:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "raw_copy_df = raw_df.sample(frac=1) # Create a copy of the raw dataset\n",
+ "\n",
+ "# Create a balanced dataset with the same number of exited and not exited customers\n",
+ "exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 1]\n",
+ "not_exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 0].sample(n=exited_df.shape[0])\n",
+ "\n",
+ "balanced_raw_df = pd.concat([exited_df, not_exited_df])\n",
+ "balanced_raw_df = balanced_raw_df.sample(frac=1, random_state=42)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let’s also quickly remove highly correlated features from the dataset using the output from a ValidMind test.\n",
+ "\n",
+ "As you know, before we can run tests you’ll need to initialize a ValidMind dataset object with the [`init_dataset` function](https://docs.validmind.ai/validmind/validmind.html#init_dataset):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Register new data and now 'balanced_raw_dataset' is the new dataset object of interest\n",
+ "vm_balanced_raw_dataset = vm.init_dataset(\n",
+ " dataset=balanced_raw_df,\n",
+ " input_id=\"balanced_raw_dataset\",\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "With our balanced dataset initialized, we can then run our test and utilize the output to help us identify the features we want to remove:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Run HighPearsonCorrelation test with our balanced dataset as input and return a result object\n",
+ "corr_result = vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n",
+ " params={\"max_threshold\": 0.3},\n",
+ " inputs={\"dataset\": vm_balanced_raw_dataset},\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# From result object, extract table from `corr_result.tables`\n",
+ "features_df = corr_result.tables[0].data\n",
+ "features_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract list of features that failed the test\n",
+ "high_correlation_features = features_df[features_df[\"Pass/Fail\"] == \"Fail\"][\"Columns\"].tolist()\n",
+ "high_correlation_features"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract feature names from the list of strings\n",
+ "high_correlation_features = [feature.split(\",\")[0].strip(\"()\") for feature in high_correlation_features]\n",
+ "high_correlation_features"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We can then re-initialize the dataset with a different `input_id` and the highly correlated features removed and re-run the test for confirmation:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Remove the highly correlated features from the dataset\n",
+ "balanced_raw_no_age_df = balanced_raw_df.drop(columns=high_correlation_features)\n",
+ "\n",
+ "# Re-initialize the dataset object\n",
+ "vm_raw_dataset_preprocessed = vm.init_dataset(\n",
+ " dataset=balanced_raw_no_age_df,\n",
+ " input_id=\"raw_dataset_preprocessed\",\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Re-run the test with the reduced feature set\n",
+ "corr_result = vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n",
+ " params={\"max_threshold\": 0.3},\n",
+ " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Split the preprocessed dataset\n",
+ "\n",
+ "With our raw dataset rebalanced with highly correlated features removed, let's now **spilt our dataset into train and test** in preparation for model evaluation testing:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Encode categorical features in the dataset\n",
+ "balanced_raw_no_age_df = pd.get_dummies(\n",
+ " balanced_raw_no_age_df, columns=[\"Geography\", \"Gender\"], drop_first=True\n",
+ ")\n",
+ "balanced_raw_no_age_df.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn.model_selection import train_test_split\n",
+ "\n",
+ "# Split the dataset into train and test\n",
+ "train_df, test_df = train_test_split(balanced_raw_no_age_df, test_size=0.20)\n",
+ "\n",
+ "X_train = train_df.drop(\"Exited\", axis=1)\n",
+ "y_train = train_df[\"Exited\"]\n",
+ "X_test = test_df.drop(\"Exited\", axis=1)\n",
+ "y_test = test_df[\"Exited\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize the split datasets\n",
+ "vm_train_ds = vm.init_dataset(\n",
+ " input_id=\"train_dataset_final\",\n",
+ " dataset=train_df,\n",
+ " target_column=\"Exited\",\n",
+ ")\n",
+ "\n",
+ "vm_test_ds = vm.init_dataset(\n",
+ " input_id=\"test_dataset_final\",\n",
+ " dataset=test_df,\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Import the champion model\n",
+ "\n",
+ "With our raw dataset assessed and preprocessed, let's go ahead and import the champion model submitted by the model development team in the format of a `.pkl` file: **[lr_model_champion.pkl](lr_model_champion.pkl)**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Import the champion model\n",
+ "import pickle as pkl\n",
+ "\n",
+ "with open(\"lr_model_champion.pkl\", \"rb\") as f:\n",
+ " log_reg = pkl.load(f)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Training a potential challenger model\n",
+ "\n",
+ "We're curious how an alternate model compares to our champion model, so let's train a challenger model as a basis for our testing.\n",
+ "\n",
+ "Our champion *logistic regression model* is a simpler, parametric model that assumes a linear relationship between the independent variables and the log-odds of the outcome. While logistic regression may not capture complex patterns as effectively, it offers a high degree of interpretability and is easier to explain to stakeholders. However, model risk is not calculated in isolation from a single factor, but rather in consideration with trade-offs in predictive performance, ease of interpretability, and overall alignment with business objectives."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Random forest classification model\n",
+ "\n",
+ "A *random forest classification model* is an ensemble machine learning algorithm that uses multiple decision trees to classify data. In ensemble learning, multiple models are combined to improve prediction accuracy and robustness.\n",
+ "\n",
+ "Random forest classification models generally have higher accuracy because they capture complex, non-linear relationships, but as a result they lack transparency in their predictions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Import the Random Forest Classification model\n",
+ "from sklearn.ensemble import RandomForestClassifier\n",
+ "\n",
+ "# Create the model instance with 50 decision trees\n",
+ "rf_model = RandomForestClassifier(\n",
+ " n_estimators=50,\n",
+ " random_state=42,\n",
+ ")\n",
+ "\n",
+ "# Train the model\n",
+ "rf_model.fit(X_train, y_train)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Initializing the model objects"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the model objects\n",
+ "\n",
+ "In addition to the initialized datasets, you'll also need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data for each of our two models.\n",
+ "\n",
+ "You simply initialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize the champion logistic regression model\n",
+ "vm_log_model = vm.init_model(\n",
+ " log_reg,\n",
+ " input_id=\"log_model_champion\",\n",
+ ")\n",
+ "\n",
+ "# Initialize the challenger random forest classification model\n",
+ "vm_rf_model = vm.init_model(\n",
+ " rf_model,\n",
+ " input_id=\"rf_model\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Assign predictions\n",
+ "\n",
+ "With our models registered, we'll move on to assigning both the predictive probabilities coming directly from each model's predictions, and the binary prediction after applying the cutoff threshold described in the Compute binary predictions step above.\n",
+ "\n",
+ "- The [`assign_predictions()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#assign_predictions) from the `Dataset` object can link existing predictions to any number of models.\n",
+ "- This method links the model's class prediction values and probabilities to our `vm_train_ds` and `vm_test_ds` datasets.\n",
+ "\n",
+ "If no prediction values are passed, the method will compute predictions automatically:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Champion — Logistic regression model\n",
+ "vm_train_ds.assign_predictions(model=vm_log_model)\n",
+ "vm_test_ds.assign_predictions(model=vm_log_model)\n",
+ "\n",
+ "# Challenger — Random forest classification model\n",
+ "vm_train_ds.assign_predictions(model=vm_rf_model)\n",
+ "vm_test_ds.assign_predictions(model=vm_rf_model)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Running model validation tests\n",
+ "\n",
+ "With everything ready for us, let's run the rest of our validation tests. We'll focus on comprehensive testing around model performance of both the champion and challenger models going forward as we've already verified the data quality of the datasets used to train the champion model."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Run model performance tests\n",
+ "\n",
+ "Let's run some performance tests, beginning with independent testing of our champion logistic regression model, then moving on to our potential challenger model.\n",
+ "\n",
+ "Use [`vm.tests.list_tests()`](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to identify all the model performance tests for classification:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "vm.tests.list_tests(tags=[\"model_performance\"], task=\"classification\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We'll isolate the specific tests we want to run in `mpt`:\n",
+ "\n",
+ "- [`ClassifierPerformance`](https://docs.validmind.ai/tests/model_validation/sklearn/ClassifierPerformance.html)\n",
+ "- [`ConfusionMatrix`](https://docs.validmind.ai/tests/model_validation/sklearn/ConfusionMatrix.html)\n",
+ "- [`MinimumAccuracy`](https://docs.validmind.ai/tests/model_validation/sklearn/MinimumAccuracy.html)\n",
+ "- [`MinimumF1Score`](https://docs.validmind.ai/tests/model_validation/sklearn/MinimumF1Score.html)\n",
+ "- [`ROCCurve`](https://docs.validmind.ai/tests/model_validation/sklearn/ROCCurve.html)\n",
+ "\n",
+ "As we learned in the previous notebook [2 — Start the model validation process](2-start_validation_process.ipynb), you can use a custom `result_id` to tag the individual result with a unique identifier by appending this `result_id` to the `test_id` with a `:` separator. We'll append an identifier for our champion model here:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mpt = [\n",
+ " \"validmind.model_validation.sklearn.ClassifierPerformance:logreg_champion\",\n",
+ " \"validmind.model_validation.sklearn.ConfusionMatrix:logreg_champion\",\n",
+ " \"validmind.model_validation.sklearn.MinimumAccuracy:logreg_champion\",\n",
+ " \"validmind.model_validation.sklearn.MinimumF1Score:logreg_champion\",\n",
+ " \"validmind.model_validation.sklearn.ROCCurve:logreg_champion\"\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Evaluate performance of the champion model\n",
+ "\n",
+ "Now, let's run and log our batch of model performance tests using our testing dataset (`vm_test_ds`) for our champion model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for test in mpt:\n",
+ " vm.tests.run_test(\n",
+ " test,\n",
+ " inputs={\n",
+ " \"dataset\": vm_test_ds, \"model\" : vm_log_model,\n",
+ " },\n",
+ " ).log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "
Note the output returned indicating that a test-driven block doesn't currently exist in your model's documentation for some test IDs. \n",
+ " \n",
+ "That's expected, as when we run validations tests the results logged need to be manually added to your report as part of your compliance assessment process within the ValidMind Platform.
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Log a model finding\n",
+ "\n",
+ "As we can observe from the output above, our champion model doesn't pass the `MinimumAccuracy` based on the default thresholds of the out-of-the-box test, so let's log a model finding in the ValidMind Platform ([Need more help?](https://docs.validmind.ai/guide/model-validation/add-manage-model-findings.html)):\n",
+ "\n",
+ "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n",
+ "\n",
+ "2. In the left sidebar that appears for your model, click **Validation Report**.\n",
+ "\n",
+ "3. Locate the Data Preparation section and click on **2.2.2. Model Performance** to expand that section.\n",
+ "\n",
+ "4. Under the Model Performance Metrics section, locate Findings then click **Link Finding to Report**:\n",
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "5. Click **+ Create New Finding** to add a finding.\n",
+ "\n",
+ "6. Enter in the details for your finding, for example:\n",
+ "\n",
+ " - **TITLE** — Champion Logistic Regression Model Fails Minimum Accuracy Threshold\n",
+ " - **RISK AREA** — Model Performance\n",
+ " - **DOCUMENTATION SECTION** — 3.2. Model Evaluation\n",
+ " - **DESCRIPTION** — The logistic regression champion model was subjected to a Minimum Accuracy test to determine whether its predictive accuracy meets the predefined performance threshold of 0.7. The model achieved an accuracy score of 0.6136, which falls below the required minimum. As a result, the test produced a Fail outcome.\n",
+ "\n",
+ "7. Click **Save**.\n",
+ "\n",
+ "8. Select the finding you just added to link to your validation report:\n",
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "9. Click **Update Linked Findings** to insert your finding.\n",
+ "\n",
+ "10. Confirm that finding you inserted has been correctly inserted into section **2.2.2. Model Performance** of the report:\n",
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "11. Click on the finding to expand the finding, where you can adjust details such as severity, owner, due date, status, etc. as well as include proposed remediation plans or supporting documentation as attachments."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Evaluate performance of challenger model\n",
+ "\n",
+ "We've now conducted similar tests as the model development team for our champion model, with the aim of verifying their test results.\n",
+ "\n",
+ "Next, let's see how our challenger models compare. We'll use the same batch of tests here as we did in `mpt`, but append a different `result_id` to indicate that these results should be associated with our challenger model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mpt_chall = [\n",
+ " \"validmind.model_validation.sklearn.ClassifierPerformance:champion_vs_challenger\",\n",
+ " \"validmind.model_validation.sklearn.ConfusionMatrix:champion_vs_challenger\",\n",
+ " \"validmind.model_validation.sklearn.MinimumAccuracy:champion_vs_challenger\",\n",
+ " \"validmind.model_validation.sklearn.MinimumF1Score:champion_vs_challenger\",\n",
+ " \"validmind.model_validation.sklearn.ROCCurve:champion_vs_challenger\"\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We'll run each test once for each model with the same `vm_test_ds` dataset to compare them:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for test in mpt_chall:\n",
+ " vm.tests.run_test(\n",
+ " test,\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds], \"model\" : [vm_log_model,vm_rf_model]\n",
+ " }\n",
+ " ).log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "
Based on the performance metrics, our challenger random forest classification model passes the MinimumAccuracy where our champion did not.\n",
+ " \n",
+ "In your validation report, support your recommendation in your finding's Proposed Remediation Plan to investigate the usage of our challenger model by inserting the performance tests we logged with this notebook into the appropriate section.
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Run diagnostic tests\n",
+ "\n",
+ "Next we want to inspect the robustness and stability testing comparison between our champion and challenger model.\n",
+ "\n",
+ "Use `list_tests()` to identify all the model diagnosis tests for classification:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.list_tests(tags=[\"model_diagnosis\"], task=\"classification\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's see if models suffer from any *overfit* potentials and also where there are potential sub-segments of issues with the [`OverfitDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/OverfitDiagnosis.html). \n",
+ "\n",
+ "Overfitting occurs when a model learns the training data too well, capturing not only the true pattern but noise and random fluctuations resulting in excellent performance on the training dataset but poor generalization to new, unseen data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.run_test(\n",
+ " test_id=\"validmind.model_validation.sklearn.OverfitDiagnosis:champion_vs_challenger\",\n",
+ " input_grid={\n",
+ " \"datasets\": [[vm_train_ds,vm_test_ds]],\n",
+ " \"model\" : [vm_log_model,vm_rf_model]\n",
+ " }\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's also conduct *robustness* and *stability* testing of the two models with the [`RobustnessDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/RobustnessDiagnosis.html).\n",
+ "\n",
+ "Robustness refers to a model's ability to maintain consistent performance, and stability refers to a model's ability to produce consistent outputs over time across different data subsets."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.run_test(\n",
+ " test_id=\"validmind.model_validation.sklearn.RobustnessDiagnosis:Champion_vs_LogRegression\",\n",
+ " input_grid={\n",
+ " \"datasets\": [[vm_train_ds,vm_test_ds]],\n",
+ " \"model\" : [vm_log_model,vm_rf_model]\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Run feature importance tests\n",
+ "\n",
+ "We also want to verify the relative influence of different input features on our models' predictions, as well as inspect the differences between our champion and challenger model to see if a certain model offers more understandable or logical importance scores for features.\n",
+ "\n",
+ "Use `list_tests()` to identify all the feature importance tests for classification:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Store the feature importance tests\n",
+ "FI = vm.tests.list_tests(tags=[\"feature_importance\"], task=\"classification\",pretty=False)\n",
+ "FI"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Run and log our feature importance tests for both models for the testing dataset\n",
+ "for test in FI:\n",
+ " vm.tests.run_test(\n",
+ " \"\".join((test,':champion_vs_challenger')),\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds], \"model\" : [vm_log_model,vm_rf_model]\n",
+ " },\n",
+ " ).log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## In summary\n",
+ "\n",
+ "In this third notebook, you learned how to:\n",
+ "\n",
+ "- [x] Initialize ValidMind model objects\n",
+ "- [x] Assign predictions and probabilities to your ValidMind model objects\n",
+ "- [x] Use tests from ValidMind to evaluate the potential of models, including comparative tests between champion and challenger models\n",
+ "- [x] Log a model finding in the ValidMind Platform"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Next steps\n",
+ "\n",
+ "\n",
+ "\n",
+ "### Finalize validation and reporting\n",
+ "\n",
+ "Now that you're familiar with the basics of using the ValidMind Library to run and log validation tests, let's learn how to implement some custom tests and wrap up our validation: **[4 — Finalize validation and reporting](4-finalize_validation_reporting.ipynb)**"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "ValidMind Library",
+ "language": "python",
+ "name": "validmind"
+ },
+ "language_info": {
+ "name": "python",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb b/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb
new file mode 100644
index 000000000..1e5561c51
--- /dev/null
+++ b/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb
@@ -0,0 +1,1207 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# ValidMind for model validation 4 — Finalize testing and reporting\n",
+ "\n",
+ "Learn how to use ValidMind for your end-to-end model validation process with our series of four introductory notebooks. In this last notebook, finalize the compliance assessment process and have a complete validation report ready for review.\n",
+ "\n",
+ "This notebook will walk you through how to supplement ValidMind tests with your own custom tests and include them as additional evidence in your validation report. A custom test is any function that takes a set of inputs and parameters as arguments and returns one or more outputs:\n",
+ "\n",
+ "- The function can be as simple or as complex as you need it to be — it can use external libraries, make API calls, or do anything else that you can do in Python.\n",
+ "- The only requirement is that the function signature and return values can be \"understood\" and handled by the ValidMind Library. As such, custom tests offer added flexibility by extending the default tests provided by ValidMind, enabling you to document any type of model or use case.\n",
+ "\n",
+ "**For a more in-depth introduction to custom tests,** refer to our [Implement custom tests](../../code_samples/custom_tests/implement_custom_tests.ipynb) notebook."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "::: {.content-hidden when-format=\"html\"}\n",
+ "## Contents \n",
+ "- [Prerequisites](#toc1_) \n",
+ "- [Setting up](#toc2_) \n",
+ " - [Initialize the ValidMind Library](#toc2_1_) \n",
+ " - [Import the sample dataset](#toc2_2_) \n",
+ " - [Split the preprocessed dataset](#toc2_3_) \n",
+ " - [Import the champion model](#toc2_4_) \n",
+ " - [Train potential challenger model](#toc2_5_) \n",
+ " - [Initialize the model objects](#toc2_6_) \n",
+ "- [Implementing custom tests](#toc3_) \n",
+ " - [Implement a custom inline test](#toc3_1_) \n",
+ " - [Create a confusion matrix plot](#toc3_1_1_) \n",
+ " - [Add parameters to custom tests](#toc3_1_2_) \n",
+ " - [Pass parameters to custom tests](#toc3_1_3_) \n",
+ " - [Use external test providers](#toc3_2_) \n",
+ " - [Create custom tests folder](#toc3_2_1_) \n",
+ " - [Save an inline test](#toc3_2_2_) \n",
+ " - [Register a local test provider](#toc3_2_3_) \n",
+ "- [Verify test runs](#toc4_) \n",
+ "- [In summary](#toc5_) \n",
+ "- [Next steps](#toc6_) \n",
+ " - [Work with your validation report](#toc6_1_) \n",
+ " - [Learn more](#toc6_2_) \n",
+ " - [More how-to guides and code samples](#toc6_2_1_) \n",
+ " - [Discover more learning resources](#toc6_2_2_) \n",
+ "\n",
+ ":::\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Prerequisites\n",
+ "\n",
+ "In order to finalize validation and reporting, you'll need to first have:\n",
+ "\n",
+ "- [x] Registered a model within the ValidMind Platform and granted yourself access to the model as a validator\n",
+ "- [x] Installed the ValidMind Library in your local environment, allowing you to access all its features\n",
+ "- [x] Learned how to import and initialize datasets and models for use with ValidMind\n",
+ "- [x] Understood the basics of how to identify and run validation tests\n",
+ "- [x] Run validation tests for your champion and challenger models, and logged the results of those tests to the ValidMind Platform\n",
+ "- [x] Inserted your logged test results into your validation report\n",
+ "- [x] Added some preliminary findings to your validation report\n",
+ "\n",
+ "
\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Setting up\n",
+ "\n",
+ "This section should be very familiar to you now — as we performed the same actions in the previous two notebooks in this series."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the ValidMind Library\n",
+ "\n",
+ "As usual, let's first connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n",
+ "\n",
+ "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
+ "\n",
+ "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Make sure the ValidMind Library is installed\n",
+ "\n",
+ "%pip install -q validmind\n",
+ "\n",
+ "# Load your model identifier credentials from an `.env` file\n",
+ "\n",
+ "%load_ext dotenv\n",
+ "%dotenv .env\n",
+ "\n",
+ "# Or replace with your code snippet\n",
+ "\n",
+ "import validmind as vm\n",
+ "\n",
+ "vm.init(\n",
+ " # api_host=\"...\",\n",
+ " # api_key=\"...\",\n",
+ " # api_secret=\"...\",\n",
+ " # model=\"...\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Import the sample dataset\n",
+ "\n",
+ "Next, we'll load in the same sample [Bank Customer Churn Prediction](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction) dataset used to develop the champion model that we will independently preprocess:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load the sample dataset\n",
+ "from validmind.datasets.classification import customer_churn as demo_dataset\n",
+ "\n",
+ "print(\n",
+ " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n",
+ ")\n",
+ "\n",
+ "raw_df = demo_dataset.load_data()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize the raw dataset for use in ValidMind tests\n",
+ "vm_raw_dataset = vm.init_dataset(\n",
+ " dataset=raw_df,\n",
+ " input_id=\"raw_dataset\",\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "raw_copy_df = raw_df.sample(frac=1) # Create a copy of the raw dataset\n",
+ "\n",
+ "# Create a balanced dataset with the same number of exited and not exited customers\n",
+ "exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 1]\n",
+ "not_exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 0].sample(n=exited_df.shape[0])\n",
+ "\n",
+ "balanced_raw_df = pd.concat([exited_df, not_exited_df])\n",
+ "balanced_raw_df = balanced_raw_df.sample(frac=1, random_state=42)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let’s also quickly remove highly correlated features from the dataset using the output from a ValidMind test:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Register new data and now 'balanced_raw_dataset' is the new dataset object of interest\n",
+ "vm_balanced_raw_dataset = vm.init_dataset(\n",
+ " dataset=balanced_raw_df,\n",
+ " input_id=\"balanced_raw_dataset\",\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Run HighPearsonCorrelation test with our balanced dataset as input and return a result object\n",
+ "corr_result = vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n",
+ " params={\"max_threshold\": 0.3},\n",
+ " inputs={\"dataset\": vm_balanced_raw_dataset},\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# From result object, extract table from `corr_result.tables`\n",
+ "features_df = corr_result.tables[0].data\n",
+ "features_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract list of features that failed the test\n",
+ "high_correlation_features = features_df[features_df[\"Pass/Fail\"] == \"Fail\"][\"Columns\"].tolist()\n",
+ "high_correlation_features"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract feature names from the list of strings\n",
+ "high_correlation_features = [feature.split(\",\")[0].strip(\"()\") for feature in high_correlation_features]\n",
+ "high_correlation_features"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Remove the highly correlated features from the dataset\n",
+ "balanced_raw_no_age_df = balanced_raw_df.drop(columns=high_correlation_features)\n",
+ "\n",
+ "# Re-initialize the dataset object\n",
+ "vm_raw_dataset_preprocessed = vm.init_dataset(\n",
+ " dataset=balanced_raw_no_age_df,\n",
+ " input_id=\"raw_dataset_preprocessed\",\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Re-run the test with the reduced feature set\n",
+ "corr_result = vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n",
+ " params={\"max_threshold\": 0.3},\n",
+ " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Split the preprocessed dataset\n",
+ "\n",
+ "With our raw dataset rebalanced with highly correlated features removed, let's now **spilt our dataset into train and test** in preparation for model evaluation testing:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Encode categorical features in the dataset\n",
+ "balanced_raw_no_age_df = pd.get_dummies(\n",
+ " balanced_raw_no_age_df, columns=[\"Geography\", \"Gender\"], drop_first=True\n",
+ ")\n",
+ "balanced_raw_no_age_df.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn.model_selection import train_test_split\n",
+ "\n",
+ "# Split the dataset into train and test\n",
+ "train_df, test_df = train_test_split(balanced_raw_no_age_df, test_size=0.20)\n",
+ "\n",
+ "X_train = train_df.drop(\"Exited\", axis=1)\n",
+ "y_train = train_df[\"Exited\"]\n",
+ "X_test = test_df.drop(\"Exited\", axis=1)\n",
+ "y_test = test_df[\"Exited\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize the split datasets\n",
+ "vm_train_ds = vm.init_dataset(\n",
+ " input_id=\"train_dataset_final\",\n",
+ " dataset=train_df,\n",
+ " target_column=\"Exited\",\n",
+ ")\n",
+ "\n",
+ "vm_test_ds = vm.init_dataset(\n",
+ " input_id=\"test_dataset_final\",\n",
+ " dataset=test_df,\n",
+ " target_column=\"Exited\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Import the champion model\n",
+ "\n",
+ "With our raw dataset assessed and preprocessed, let's go ahead and import the champion model submitted by the model development team in the format of a `.pkl` file: **[lr_model_champion.pkl](lr_model_champion.pkl)**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Import the champion model\n",
+ "import pickle as pkl\n",
+ "\n",
+ "with open(\"lr_model_champion.pkl\", \"rb\") as f:\n",
+ " log_reg = pkl.load(f)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Train potential challenger model\n",
+ "\n",
+ "We'll also train our random forest classification challenger model to see how it compares:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Import the Random Forest Classification model\n",
+ "from sklearn.ensemble import RandomForestClassifier\n",
+ "\n",
+ "# Create the model instance with 50 decision trees\n",
+ "rf_model = RandomForestClassifier(\n",
+ " n_estimators=50,\n",
+ " random_state=42,\n",
+ ")\n",
+ "\n",
+ "# Train the model\n",
+ "rf_model.fit(X_train, y_train)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Initialize the model objects\n",
+ "\n",
+ "In addition to the initialized datasets, you'll also need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data for each of our two models:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize the champion logistic regression model\n",
+ "vm_log_model = vm.init_model(\n",
+ " log_reg,\n",
+ " input_id=\"log_model_champion\",\n",
+ ")\n",
+ "\n",
+ "# Initialize the challenger random forest classification model\n",
+ "vm_rf_model = vm.init_model(\n",
+ " rf_model,\n",
+ " input_id=\"rf_model\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Assign predictions to Champion — Logistic regression model\n",
+ "vm_train_ds.assign_predictions(model=vm_log_model)\n",
+ "vm_test_ds.assign_predictions(model=vm_log_model)\n",
+ "\n",
+ "# Assign predictions to Challenger — Random forest classification model\n",
+ "vm_train_ds.assign_predictions(model=vm_rf_model)\n",
+ "vm_test_ds.assign_predictions(model=vm_rf_model)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Implementing custom tests\n",
+ "\n",
+ "Thanks to the model documentation ([Learn more ...](https://docs.validmind.ai/developer/validmind-library.html#for-model-development)), we know that the model development team implemented a custom test to further evaluate the performance of the champion model.\n",
+ "\n",
+ "In a usual model validation situation, you would load a saved custom test provided by the model development team. In the following section, we'll have you implement the same custom test and make it available for reuse, to familiarize you with the processes.\n",
+ "\n",
+ "
Want to learn more about custom tests?\n",
+ " \n",
+ "Refer to our in-depth introduction to custom tests: Implement custom tests
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Implement a custom inline test\n",
+ "\n",
+ "Let's implement the same custom *inline test* that calculates the confusion matrix for a binary classification model that the model development team used in their performance evaluations.\n",
+ "\n",
+ "- An inline test refers to a test written and executed within the same environment as the code being tested — in this case, right in this Jupyter Notebook — without requiring a separate test file or framework.\n",
+ "- You'll note that the custom test function is just a regular Python function that can include and require any Python library as you see fit."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Create a confusion matrix plot\n",
+ "\n",
+ "Let's first create a confusion matrix plot using the `confusion_matrix` function from the `sklearn.metrics` module:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn import metrics\n",
+ "\n",
+ "# Get the predicted classes\n",
+ "y_pred = log_reg.predict(vm_test_ds.x)\n",
+ "\n",
+ "confusion_matrix = metrics.confusion_matrix(y_test, y_pred)\n",
+ "\n",
+ "cm_display = metrics.ConfusionMatrixDisplay(\n",
+ " confusion_matrix=confusion_matrix, display_labels=[False, True]\n",
+ ")\n",
+ "cm_display.plot()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Next, create a [`@vm.test` wrapper](https://docs.validmind.ai/validmind/validmind.html#test) that will allow you to create a reusable test. **Note the following changes in the code below:**\n",
+ "\n",
+ "- The function `confusion_matrix` takes two arguments `dataset` and `model`. This is a `VMDataset` and `VMModel` object respectively.\n",
+ " - `VMDataset` objects allow you to access the dataset's true (target) values by accessing the `.y` attribute.\n",
+ " - `VMDataset` objects allow you to access the predictions for a given model by accessing the `.y_pred()` method.\n",
+ "- The function docstring provides a description of what the test does. This will be displayed along with the result in this notebook as well as in the ValidMind Platform.\n",
+ "- The function body calculates the confusion matrix using the `sklearn.metrics.confusion_matrix` function as we just did above.\n",
+ "- The function then returns the `ConfusionMatrixDisplay.figure_` object — this is important as the ValidMind Library expects the output of the custom test to be a plot or a table.\n",
+ "- The `@vm.test` decorator is doing the work of creating a wrapper around the function that will allow it to be run by the ValidMind Library. It also registers the test so it can be found by the ID `my_custom_tests.ConfusionMatrix`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "@vm.test(\"my_custom_tests.ConfusionMatrix\")\n",
+ "def confusion_matrix(dataset, model):\n",
+ " \"\"\"The confusion matrix is a table that is often used to describe the performance of a classification model on a set of data for which the true values are known.\n",
+ "\n",
+ " The confusion matrix is a 2x2 table that contains 4 values:\n",
+ "\n",
+ " - True Positive (TP): the number of correct positive predictions\n",
+ " - True Negative (TN): the number of correct negative predictions\n",
+ " - False Positive (FP): the number of incorrect positive predictions\n",
+ " - False Negative (FN): the number of incorrect negative predictions\n",
+ "\n",
+ " The confusion matrix can be used to assess the holistic performance of a classification model by showing the accuracy, precision, recall, and F1 score of the model on a single figure.\n",
+ " \"\"\"\n",
+ " y_true = dataset.y\n",
+ " y_pred = dataset.y_pred(model=model)\n",
+ "\n",
+ " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n",
+ "\n",
+ " cm_display = metrics.ConfusionMatrixDisplay(\n",
+ " confusion_matrix=confusion_matrix, display_labels=[False, True]\n",
+ " )\n",
+ " cm_display.plot()\n",
+ "\n",
+ " plt.close() # close the plot to avoid displaying it\n",
+ "\n",
+ " return cm_display.figure_ # return the figure object itself"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "You can now run the newly created custom test on both the training and test datasets for both models using the [`run_test()` function](https://docs.validmind.ai/validmind/validmind/tests.html#run_test):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Champion train and test\n",
+ "vm.tests.run_test(\n",
+ " test_id=\"my_custom_tests.ConfusionMatrix:champion\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_train_ds,vm_test_ds],\n",
+ " \"model\" : [vm_log_model]\n",
+ " }\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Challenger train and test\n",
+ "vm.tests.run_test(\n",
+ " test_id=\"my_custom_tests.ConfusionMatrix:challenger\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_train_ds,vm_test_ds],\n",
+ " \"model\" : [vm_rf_model]\n",
+ " }\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "
Note the output returned indicating that a test-driven block doesn't currently exist in your model's documentation for some test IDs. \n",
+ " \n",
+ "That's expected, as when we run validations tests the results logged need to be manually added to your report as part of your compliance assessment process within the ValidMind Platform.
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Add parameters to custom tests\n",
+ "\n",
+ "Custom tests can take parameters just like any other function. To demonstrate, let's modify the `confusion_matrix` function to take an additional parameter `normalize` that will allow you to normalize the confusion matrix:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "@vm.test(\"my_custom_tests.ConfusionMatrix\")\n",
+ "def confusion_matrix(dataset, model, normalize=False):\n",
+ " \"\"\"The confusion matrix is a table that is often used to describe the performance of a classification model on a set of data for which the true values are known.\n",
+ "\n",
+ " The confusion matrix is a 2x2 table that contains 4 values:\n",
+ "\n",
+ " - True Positive (TP): the number of correct positive predictions\n",
+ " - True Negative (TN): the number of correct negative predictions\n",
+ " - False Positive (FP): the number of incorrect positive predictions\n",
+ " - False Negative (FN): the number of incorrect negative predictions\n",
+ "\n",
+ " The confusion matrix can be used to assess the holistic performance of a classification model by showing the accuracy, precision, recall, and F1 score of the model on a single figure.\n",
+ " \"\"\"\n",
+ " y_true = dataset.y\n",
+ " y_pred = dataset.y_pred(model=model)\n",
+ "\n",
+ " if normalize:\n",
+ " confusion_matrix = metrics.confusion_matrix(y_true, y_pred, normalize=\"all\")\n",
+ " else:\n",
+ " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n",
+ "\n",
+ " cm_display = metrics.ConfusionMatrixDisplay(\n",
+ " confusion_matrix=confusion_matrix, display_labels=[False, True]\n",
+ " )\n",
+ " cm_display.plot()\n",
+ "\n",
+ " plt.close() # close the plot to avoid displaying it\n",
+ "\n",
+ " return cm_display.figure_ # return the figure object itself"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Pass parameters to custom tests\n",
+ "\n",
+ "You can pass parameters to custom tests by providing a dictionary of parameters to the `run_test()` function.\n",
+ "\n",
+ "- The parameters will override any default parameters set in the custom test definition. Note that `dataset` and `model` are still passed as `inputs`.\n",
+ "- Since these are `VMDataset` or `VMModel` inputs, they have a special meaning.\n",
+ "\n",
+ "Re-running and logging the custom confusion matrix with `normalize=True` for both models and our testing dataset looks like this:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Champion with test dataset and normalize=True\n",
+ "vm.tests.run_test(\n",
+ " test_id=\"my_custom_tests.ConfusionMatrix:test_normalized_champion\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\" : [vm_log_model]\n",
+ " },\n",
+ " params={\"normalize\": True}\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Challenger with test dataset and normalize=True\n",
+ "vm.tests.run_test(\n",
+ " test_id=\"my_custom_tests.ConfusionMatrix:test_normalized_challenger\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\" : [vm_rf_model]\n",
+ " },\n",
+ " params={\"normalize\": True}\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Use external test providers\n",
+ "\n",
+ "Sometimes you may want to reuse the same set of custom tests across multiple models and share them with others in your organization, like the model development team would have done with you in this example workflow featured in this series of notebooks. In this case, you can create an external custom *test provider* that will allow you to load custom tests from a local folder or a Git repository.\n",
+ "\n",
+ "In this section you will learn how to declare a local filesystem test provider that allows loading tests from a local folder following these high level steps:\n",
+ "\n",
+ "1. Create a folder of custom tests from existing inline tests (tests that exist in your active Jupyter Notebook)\n",
+ "2. Save an inline test to a file\n",
+ "3. Define and register a [`LocalTestProvider`](https://docs.validmind.ai/validmind/validmind/tests.html#LocalTestProvider) that points to that folder\n",
+ "4. Run test provider tests\n",
+ "5. Add the test results to your documentation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Create custom tests folder\n",
+ "\n",
+ "Let's start by creating a new folder that will contain reusable custom tests from your existing inline tests.\n",
+ "\n",
+ "The following code snippet will create a new `my_tests` directory in the current working directory if it doesn't exist:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tests_folder = \"my_tests\"\n",
+ "\n",
+ "import os\n",
+ "\n",
+ "# create tests folder\n",
+ "os.makedirs(tests_folder, exist_ok=True)\n",
+ "\n",
+ "# remove existing tests\n",
+ "for f in os.listdir(tests_folder):\n",
+ " # remove files and pycache\n",
+ " if f.endswith(\".py\") or f == \"__pycache__\":\n",
+ " os.system(f\"rm -rf {tests_folder}/{f}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "After running the command above, confirm that a new `my_tests` directory was created successfully. For example:\n",
+ "\n",
+ "```\n",
+ "~/notebooks/tutorials/model_validation/my_tests/\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Save an inline test\n",
+ "\n",
+ "The `@vm.test` decorator we used in **Implement a custom inline test** above to register one-off custom tests also includes a convenience method on the function object that allows you to simply call `.save()` to save the test to a Python file at a specified path.\n",
+ "\n",
+ "While `save()` will get you started by creating the file and saving the function code with the correct name, it won't automatically include any imports, or other functions or variables, outside of the functions that are needed for the test to run. To solve this, pass in an optional `imports` argument ensuring necessary imports are added to the file.\n",
+ "\n",
+ "The `confusion_matrix` test requires the following additional imports:\n",
+ "\n",
+ "```python\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn import metrics\n",
+ "```\n",
+ "\n",
+ "Let's pass these imports to the `save()` method to ensure they are included in the file with the following command:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "confusion_matrix.save(\n",
+ " # Save it to the custom tests folder we created\n",
+ " tests_folder,\n",
+ " imports=[\"import matplotlib.pyplot as plt\", \"from sklearn import metrics\"],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "- [x] Confirm that the `save()` method saved the `confusion_matrix` function to a file named `ConfusionMatrix.py` in the `my_tests` folder.\n",
+ "- [x] Note that the new file provides some context on the origin of the test, which is useful for traceability:\n",
+ "\n",
+ " ```\n",
+ " # Saved from __main__.confusion_matrix\n",
+ " # Original Test ID: my_custom_tests.ConfusionMatrix\n",
+ " # New Test ID: .ConfusionMatrix\n",
+ " ```\n",
+ "\n",
+ "- [x] Additionally, the new test function has been stripped off its decorator, as it now resides in a file that will be loaded by the test provider:\n",
+ "\n",
+ " ```python\n",
+ " def ConfusionMatrix(dataset, model, normalize=False):\n",
+ " ```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Register a local test provider\n",
+ "\n",
+ "Now that your `my_tests` folder has a sample custom test, let's initialize a test provider that will tell the ValidMind Library where to find your custom tests:\n",
+ "\n",
+ "- ValidMind offers out-of-the-box test providers for local tests (tests in a folder) or a Github provider for tests in a Github repository.\n",
+ "- You can also create your own test provider by creating a class that has a [`load_test` method](https://docs.validmind.ai/validmind/validmind/tests.html#load_test) that takes a test ID and returns the test function matching that ID.\n",
+ "\n",
+ "
Want to learn more about test providers?\n",
+ " \n",
+ "An extended introduction to test providers can be found in: Integrate external test providers
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##### Initialize a local test provider\n",
+ "\n",
+ "For most use cases, using a `LocalTestProvider` that allows you to load custom tests from a designated directory should be sufficient.\n",
+ "\n",
+ "**The most important attribute for a test provider is its `namespace`.** This is a string that will be used to prefix test IDs in model documentation. This allows you to have multiple test providers with tests that can even share the same ID, but are distinguished by their namespace.\n",
+ "\n",
+ "Let's go ahead and load the custom tests from our `my_tests` directory:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from validmind.tests import LocalTestProvider\n",
+ "\n",
+ "# initialize the test provider with the tests folder we created earlier\n",
+ "my_test_provider = LocalTestProvider(tests_folder)\n",
+ "\n",
+ "vm.tests.register_test_provider(\n",
+ " namespace=\"my_test_provider\",\n",
+ " test_provider=my_test_provider,\n",
+ ")\n",
+ "# `my_test_provider.load_test()` will be called for any test ID that starts with `my_test_provider`\n",
+ "# e.g. `my_test_provider.ConfusionMatrix` will look for a function named `ConfusionMatrix` in `my_tests/ConfusionMatrix.py` file"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##### Run test provider tests\n",
+ "\n",
+ "Now that we've set up the test provider, we can run any test that's located in the tests folder by using the `run_test()` method as with any other test:\n",
+ "\n",
+ "- For tests that reside in a test provider directory, the test ID will be the `namespace` specified when registering the provider, followed by the path to the test file relative to the tests folder.\n",
+ "- For example, the Confusion Matrix test we created earlier will have the test ID `my_test_provider.ConfusionMatrix`. You could organize the tests in subfolders, say `classification` and `regression`, and the test ID for the Confusion Matrix test would then be `my_test_provider.classification.ConfusionMatrix`.\n",
+ "\n",
+ "Let's go ahead and re-run the confusion matrix test with our testing dataset for our two models by using the test ID `my_test_provider.ConfusionMatrix`. This should load the test from the test provider and run it as before."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Champion with test dataset and test provider custom test\n",
+ "vm.tests.run_test(\n",
+ " test_id=\"my_test_provider.ConfusionMatrix:champion\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\" : [vm_log_model]\n",
+ " }\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Challenger with test dataset and test provider custom test\n",
+ "vm.tests.run_test(\n",
+ " test_id=\"my_test_provider.ConfusionMatrix:challenger\",\n",
+ " input_grid={\n",
+ " \"dataset\": [vm_test_ds],\n",
+ " \"model\" : [vm_rf_model]\n",
+ " }\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Verify test runs\n",
+ "\n",
+ "Our final task is to verify that all the tests provided by the model development team were run and reported accurately. Note the appended `result_ids` to delineate which dataset we ran the test with for the relevant tests.\n",
+ "\n",
+ "Here, we'll specify all the tests we'd like to independently rerun in a dictionary called `test_config`. **Note here that `inputs` and `input_grid` expect the `input_id` of the dataset or model as the value rather than the variable name we specified**:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_config = {\n",
+ " # Run with the raw dataset\n",
+ " 'validmind.data_validation.DatasetDescription:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'}\n",
+ " },\n",
+ " 'validmind.data_validation.DescriptiveStatistics:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'}\n",
+ " },\n",
+ " 'validmind.data_validation.MissingValues:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'min_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.ClassImbalance:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'min_percent_threshold': 10}\n",
+ " },\n",
+ " 'validmind.data_validation.Duplicates:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'min_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.HighCardinality:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {\n",
+ " 'num_threshold': 100,\n",
+ " 'percent_threshold': 0.1,\n",
+ " 'threshold_type': 'percent'\n",
+ " }\n",
+ " },\n",
+ " 'validmind.data_validation.Skewness:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'max_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.UniqueRows:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'min_percent_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.TooManyZeroValues:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'max_percent_threshold': 0.03}\n",
+ " },\n",
+ " 'validmind.data_validation.IQROutliersTable:raw_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset'},\n",
+ " 'params': {'threshold': 5}\n",
+ " },\n",
+ " # Run with the preprocessed dataset\n",
+ " 'validmind.data_validation.DescriptiveStatistics:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset_preprocessed'}\n",
+ " },\n",
+ " 'validmind.data_validation.TabularDescriptionTables:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset_preprocessed'}\n",
+ " },\n",
+ " 'validmind.data_validation.MissingValues:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset_preprocessed'},\n",
+ " 'params': {'min_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.TabularNumericalHistograms:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset_preprocessed'}\n",
+ " },\n",
+ " 'validmind.data_validation.TabularCategoricalBarPlots:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset_preprocessed'}\n",
+ " },\n",
+ " 'validmind.data_validation.TargetRateBarPlots:preprocessed_data': {\n",
+ " 'inputs': {'dataset': 'raw_dataset_preprocessed'},\n",
+ " 'params': {'default_column': 'loan_status'}\n",
+ " },\n",
+ " # Run with the training and test datasets\n",
+ " 'validmind.data_validation.DescriptiveStatistics:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']}\n",
+ " },\n",
+ " 'validmind.data_validation.TabularDescriptionTables:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']}\n",
+ " },\n",
+ " 'validmind.data_validation.ClassImbalance:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']},\n",
+ " 'params': {'min_percent_threshold': 10}\n",
+ " },\n",
+ " 'validmind.data_validation.UniqueRows:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']},\n",
+ " 'params': {'min_percent_threshold': 1}\n",
+ " },\n",
+ " 'validmind.data_validation.TabularNumericalHistograms:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']}\n",
+ " },\n",
+ " 'validmind.data_validation.MutualInformation:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']},\n",
+ " 'params': {'min_threshold': 0.01}\n",
+ " },\n",
+ " 'validmind.data_validation.PearsonCorrelationMatrix:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']}\n",
+ " },\n",
+ " 'validmind.data_validation.HighPearsonCorrelation:development_data': {\n",
+ " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']},\n",
+ " 'params': {'max_threshold': 0.3, 'top_n_correlations': 10}\n",
+ " },\n",
+ " 'validmind.model_validation.ModelMetadata': {\n",
+ " 'input_grid': {'model': ['log_model_champion', 'rf_model']}\n",
+ " },\n",
+ " 'validmind.model_validation.sklearn.ModelParameters': {\n",
+ " 'input_grid': {'model': ['log_model_champion', 'rf_model']}\n",
+ " },\n",
+ " 'validmind.model_validation.sklearn.ROCCurve': {\n",
+ " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final'], 'model': ['log_model_champion']}\n",
+ " },\n",
+ " 'validmind.model_validation.sklearn.MinimumROCAUCScore': {\n",
+ " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final'], 'model': ['log_model_champion']},\n",
+ " 'params': {'min_threshold': 0.5}\n",
+ " }\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Then batch run and log our tests in `test_config`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for t in test_config:\n",
+ " print(t)\n",
+ " try:\n",
+ " # Check if test has input_grid\n",
+ " if 'input_grid' in test_config[t]:\n",
+ " # For tests with input_grid, pass the input_grid configuration\n",
+ " if 'params' in test_config[t]:\n",
+ " vm.tests.run_test(t, input_grid=test_config[t]['input_grid'], params=test_config[t]['params']).log()\n",
+ " else:\n",
+ " vm.tests.run_test(t, input_grid=test_config[t]['input_grid']).log()\n",
+ " else:\n",
+ " # Original logic for regular inputs\n",
+ " if 'params' in test_config[t]:\n",
+ " vm.tests.run_test(t, inputs=test_config[t]['inputs'], params=test_config[t]['params']).log()\n",
+ " else:\n",
+ " vm.tests.run_test(t, inputs=test_config[t]['inputs']).log()\n",
+ " except Exception as e:\n",
+ " print(f\"Error running test {t}: {str(e)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## In summary\n",
+ "\n",
+ "In this final notebook, you learned how to:\n",
+ "\n",
+ "- [x] Implement a custom inline test\n",
+ "- [x] Run and log your custom inline tests\n",
+ "- [x] Use external custom test providers\n",
+ "- [x] Run and log tests from your custom test providers\n",
+ "- [x] Re-run tests provided by your model development team to verify that they were run and reported accurately\n",
+ "\n",
+ "With our ValidMind for model validation series of notebooks, you learned how to validate a model end-to-end with the ValidMind Library by running through some common scenarios in a typical model validation setting:\n",
+ "\n",
+ "- Verifying the data quality steps performed by the model development team\n",
+ "- Independently replicating the champion model's results and conducting additional tests to assess performance, stability, and robustness\n",
+ "- Setting up test inputs and a challenger model for comparative analysis\n",
+ "- Running validation tests, analyzing results, and logging findings to ValidMind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## Next steps"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Work with your validation report\n",
+ "\n",
+ "Now that you've logged all your test results and verified the work done by the model development team, head to the ValidMind Platform to wrap up your validation report. Continue to work on your validation report by:\n",
+ "\n",
+ "- **Inserting additional test results:** Click **Link Evidence to Report** under any section of 2. Validation in your validation report. (Learn more: [Link evidence to reports](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#link-evidence-to-reports))\n",
+ "\n",
+ "- **Making qualitative edits to your test descriptions:** Expand any linked evidence under Validator Evidence and click **See evidence details** to review and edit the ValidMind-generated test descriptions for quality and accuracy.\n",
+ "\n",
+ "- **Adding more findings:** Click **Link Finding to Report** in any validation report section, then click **+ Create New Finding**. (Learn more: [Add and manage model findings](https://docs.validmind.ai/guide/model-validation/add-manage-model-findings.html))\n",
+ "\n",
+ "- **Adding risk assessment notes:** Click under **Risk Assessment Notes** in any validation report section to access the text editor and content editing toolbar, including an option to generate a draft with AI. Edit your ValidMind-generated test descriptions (Learn more: [Work with content blocks](https://docs.validmind.ai/guide/model-documentation/work-with-content-blocks.html#content-editing-toolbar))\n",
+ "\n",
+ "- **Assessing compliance:** Under the Guideline for any validation report section, click **ASSESSMENT** and select the compliance status from the drop-down menu. (Learn more: [Provide compliance assessments](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#provide-compliance-assessments))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### Learn more\n",
+ "\n",
+ "Now that you're familiar with the basics, you can explore the following notebooks to get a deeper understanding on how the ValidMind Library assists you in streamlining model validation:"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### More how-to guides and code samples\n",
+ "\n",
+ "- [Explore available tests in detail](../../how_to/explore_tests.ipynb)\n",
+ "- [In-depth guide on running dataset based tests](../../how_to/run_tests/1_run_dataset_based_tests.ipynb)\n",
+ "- [In-depth guide for running comparison tests](../../how_to/run_tests/2_run_comparison_tests.ipynb)\n",
+ "- [In-depth guide for implementing custom tests](../../code_samples/custom_tests/implement_custom_tests.ipynb)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "#### Discover more learning resources\n",
+ "\n",
+ "All notebook samples can be found in the following directories of the ValidMind Library GitHub repository:\n",
+ "\n",
+ "- [Code samples](https://github.com/validmind/validmind-library/tree/main/notebooks/code_samples)\n",
+ "- [How-to guides](https://github.com/validmind/validmind-library/tree/main/notebooks/how_to)\n",
+ "\n",
+ "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "ValidMind Library",
+ "language": "python",
+ "name": "validmind"
+ },
+ "language_info": {
+ "name": "python",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/tutorials/model_validation/class-imbalance-results-detail.png b/notebooks/tutorials/model_validation/class-imbalance-results-detail.png
new file mode 100644
index 000000000..00251ecaa
Binary files /dev/null and b/notebooks/tutorials/model_validation/class-imbalance-results-detail.png differ
diff --git a/notebooks/tutorials/model_validation/compliance-summary.png b/notebooks/tutorials/model_validation/compliance-summary.png
new file mode 100644
index 000000000..a8abd2dd3
Binary files /dev/null and b/notebooks/tutorials/model_validation/compliance-summary.png differ
diff --git a/notebooks/tutorials/model_validation/inserted-class-imbalance-results.png b/notebooks/tutorials/model_validation/inserted-class-imbalance-results.png
new file mode 100644
index 000000000..2efea1a09
Binary files /dev/null and b/notebooks/tutorials/model_validation/inserted-class-imbalance-results.png differ
diff --git a/notebooks/tutorials/model_validation/inserted-finding.png b/notebooks/tutorials/model_validation/inserted-finding.png
new file mode 100644
index 000000000..bb7843653
Binary files /dev/null and b/notebooks/tutorials/model_validation/inserted-finding.png differ
diff --git a/notebooks/tutorials/model_validation/inserted-minimum-f1-scores.png b/notebooks/tutorials/model_validation/inserted-minimum-f1-scores.png
new file mode 100644
index 000000000..52ae43c71
Binary files /dev/null and b/notebooks/tutorials/model_validation/inserted-minimum-f1-scores.png differ
diff --git a/notebooks/tutorials/model_validation/link-finding.png b/notebooks/tutorials/model_validation/link-finding.png
new file mode 100644
index 000000000..dd1c8a266
Binary files /dev/null and b/notebooks/tutorials/model_validation/link-finding.png differ
diff --git a/notebooks/tutorials/model_validation/link-validator-evidence.png b/notebooks/tutorials/model_validation/link-validator-evidence.png
new file mode 100644
index 000000000..7403cad67
Binary files /dev/null and b/notebooks/tutorials/model_validation/link-validator-evidence.png differ
diff --git a/notebooks/tutorials/model_validation/link-validator-evidence_OLD.png b/notebooks/tutorials/model_validation/link-validator-evidence_OLD.png
new file mode 100644
index 000000000..58823e8f2
Binary files /dev/null and b/notebooks/tutorials/model_validation/link-validator-evidence_OLD.png differ
diff --git a/notebooks/tutorials/model_validation/lr_model_champion.pkl b/notebooks/tutorials/model_validation/lr_model_champion.pkl
new file mode 100644
index 000000000..9b81662b2
Binary files /dev/null and b/notebooks/tutorials/model_validation/lr_model_champion.pkl differ
diff --git a/notebooks/tutorials/model_validation/select-finding.png b/notebooks/tutorials/model_validation/select-finding.png
new file mode 100644
index 000000000..ba35661d5
Binary files /dev/null and b/notebooks/tutorials/model_validation/select-finding.png differ
diff --git a/notebooks/tutorials/model_validation/selecting-class-imbalance-results.png b/notebooks/tutorials/model_validation/selecting-class-imbalance-results.png
new file mode 100644
index 000000000..cf8687422
Binary files /dev/null and b/notebooks/tutorials/model_validation/selecting-class-imbalance-results.png differ
diff --git a/notebooks/tutorials/model_validation/selecting-minimum-f1-scores.png b/notebooks/tutorials/model_validation/selecting-minimum-f1-scores.png
new file mode 100644
index 000000000..60ae6b960
Binary files /dev/null and b/notebooks/tutorials/model_validation/selecting-minimum-f1-scores.png differ
diff --git a/poetry.lock b/poetry.lock
index 81266120e..8452217ac 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
[[package]]
name = "aiodns"
@@ -6,6 +6,8 @@ version = "3.2.0"
description = "Simple DNS resolver for asyncio"
optional = false
python-versions = "*"
+groups = ["main"]
+markers = "sys_platform == \"linux\" or sys_platform == \"darwin\""
files = [
{file = "aiodns-3.2.0-py3-none-any.whl", hash = "sha256:e443c0c27b07da3174a109fd9e736d69058d808f144d3c9d56dbd1776964c5f5"},
{file = "aiodns-3.2.0.tar.gz", hash = "sha256:62869b23409349c21b072883ec8998316b234c9a9e36675756e8e317e8768f72"},
@@ -20,6 +22,7 @@ version = "2.4.4"
description = "Happy Eyeballs for asyncio"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "aiohappyeyeballs-2.4.4-py3-none-any.whl", hash = "sha256:a980909d50efcd44795c4afeca523296716d50cd756ddca6af8c65b996e27de8"},
{file = "aiohappyeyeballs-2.4.4.tar.gz", hash = "sha256:5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745"},
@@ -31,6 +34,7 @@ version = "3.10.11"
description = "Async http client/server framework (asyncio)"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "aiohttp-3.10.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5077b1a5f40ffa3ba1f40d537d3bec4383988ee51fbba6b74aa8fb1bc466599e"},
{file = "aiohttp-3.10.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8d6a14a4d93b5b3c2891fca94fa9d41b2322a68194422bef0dd5ec1e57d7d298"},
@@ -138,7 +142,7 @@ multidict = ">=4.5,<7.0"
yarl = ">=1.12.0,<2.0"
[package.extras]
-speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"]
+speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (>=3.2.0) ; sys_platform == \"linux\" or sys_platform == \"darwin\"", "brotlicffi ; platform_python_implementation != \"CPython\""]
[[package]]
name = "aiosignal"
@@ -146,6 +150,7 @@ version = "1.3.1"
description = "aiosignal: a list of registered asynchronous callbacks"
optional = false
python-versions = ">=3.7"
+groups = ["main"]
files = [
{file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"},
{file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"},
@@ -160,6 +165,7 @@ version = "0.7.13"
description = "A configurable sidebar-enabled Sphinx theme"
optional = false
python-versions = ">=3.6"
+groups = ["dev"]
files = [
{file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"},
{file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"},
@@ -171,6 +177,7 @@ version = "0.7.0"
description = "Reusable constraint types to use with typing.Annotated"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"},
{file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
@@ -185,6 +192,7 @@ version = "1.1.8"
description = "ANSI colors for Python"
optional = false
python-versions = "*"
+groups = ["dev"]
files = [
{file = "ansicolors-1.1.8-py2.py3-none-any.whl", hash = "sha256:00d2dde5a675579325902536738dd27e4fac1fd68f773fe36c21044eb559e187"},
{file = "ansicolors-1.1.8.zip", hash = "sha256:99f94f5e3348a0bcd43c82e5fc4414013ccc19d70bd939ad71e0133ce9c372e0"},
@@ -196,6 +204,7 @@ version = "4.5.2"
description = "High level compatibility layer for multiple asynchronous event loop implementations"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "anyio-4.5.2-py3-none-any.whl", hash = "sha256:c011ee36bc1e8ba40e5a81cb9df91925c218fe9b778554e0b56a21e1b5d4716f"},
{file = "anyio-4.5.2.tar.gz", hash = "sha256:23009af4ed04ce05991845451e11ef02fc7c5ed29179ac9a420e5ad0ac7ddc5b"},
@@ -209,7 +218,7 @@ typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""}
[package.extras]
doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
-test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"]
+test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21.0b1) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\""]
trio = ["trio (>=0.26.1)"]
[[package]]
@@ -218,6 +227,7 @@ version = "0.9.15"
description = "custom jupyter widgets made easy"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "anywidget-0.9.15-py3-none-any.whl", hash = "sha256:fd7876332e47f380e0428f552f26b7227f5694d4e0a257bbc23354d9b9e9a73c"},
{file = "anywidget-0.9.15.tar.gz", hash = "sha256:1891c11897aaf7cff8809f996413f618f97d786b6097f7e46266423969a726a0"},
@@ -237,6 +247,8 @@ version = "1.4.4"
description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
optional = true
python-versions = "*"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
@@ -248,10 +260,12 @@ version = "0.1.4"
description = "Disable App Nap on macOS >= 10.9"
optional = false
python-versions = ">=3.6"
+groups = ["main", "dev"]
files = [
{file = "appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c"},
{file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"},
]
+markers = {main = "sys_platform == \"darwin\"", dev = "sys_platform == \"darwin\" or platform_system == \"Darwin\""}
[[package]]
name = "arch"
@@ -259,6 +273,7 @@ version = "5.6.0"
description = "ARCH for Python"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "arch-5.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:478d049fb18e022670952792ebaa6b66acff77580c0f691497b706ce9192e941"},
{file = "arch-5.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f0da8a70e56759b469eeef52153e0a6eaf2e384c4f48427189433ea12fc8886a"},
@@ -298,6 +313,7 @@ version = "23.1.0"
description = "Argon2 for Python"
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "argon2_cffi-23.1.0-py3-none-any.whl", hash = "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"},
{file = "argon2_cffi-23.1.0.tar.gz", hash = "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08"},
@@ -318,6 +334,7 @@ version = "21.2.0"
description = "Low-level CFFI bindings for Argon2"
optional = false
python-versions = ">=3.6"
+groups = ["dev"]
files = [
{file = "argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3"},
{file = "argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367"},
@@ -355,6 +372,7 @@ version = "1.3.0"
description = "Better dates & times for Python"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "arrow-1.3.0-py3-none-any.whl", hash = "sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80"},
{file = "arrow-1.3.0.tar.gz", hash = "sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85"},
@@ -374,6 +392,7 @@ version = "3.0.0"
description = "Annotate AST trees with source code positions"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2"},
{file = "asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7"},
@@ -389,6 +408,8 @@ version = "1.6.3"
description = "An AST unparser for Python"
optional = false
python-versions = "*"
+groups = ["dev"]
+markers = "python_version == \"3.8\""
files = [
{file = "astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8"},
{file = "astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872"},
@@ -404,6 +425,7 @@ version = "2.0.4"
description = "Simple LRU cache for asyncio"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "async-lru-2.0.4.tar.gz", hash = "sha256:b8a59a5df60805ff63220b2a0c5b5393da5521b113cd5465a44eb037d81a5627"},
{file = "async_lru-2.0.4-py3-none-any.whl", hash = "sha256:ff02944ce3c288c5be660c42dbcca0742b32c3b279d6dceda655190240b99224"},
@@ -418,6 +440,8 @@ version = "4.0.3"
description = "Timeout context manager for asyncio programs"
optional = false
python-versions = ">=3.7"
+groups = ["main"]
+markers = "python_version == \"3.10\" or python_version == \"3.9\" or python_version == \"3.8\""
files = [
{file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"},
{file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"},
@@ -425,22 +449,23 @@ files = [
[[package]]
name = "attrs"
-version = "25.1.0"
+version = "25.2.0"
description = "Classes Without Boilerplate"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
- {file = "attrs-25.1.0-py3-none-any.whl", hash = "sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a"},
- {file = "attrs-25.1.0.tar.gz", hash = "sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e"},
+ {file = "attrs-25.2.0-py3-none-any.whl", hash = "sha256:611344ff0a5fed735d86d7784610c84f8126b95e549bcad9ff61b4242f2d386b"},
+ {file = "attrs-25.2.0.tar.gz", hash = "sha256:18a06db706db43ac232cce80443fcd9f2500702059ecf53489e3c5a3f417acaf"},
]
[package.extras]
-benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
-tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"]
+benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
+cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
+dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
+docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"]
+tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
+tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""]
[[package]]
name = "babel"
@@ -448,6 +473,7 @@ version = "2.17.0"
description = "Internationalization utilities"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"},
{file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"},
@@ -457,7 +483,7 @@ files = [
pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""}
[package.extras]
-dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"]
+dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""]
[[package]]
name = "backcall"
@@ -465,6 +491,7 @@ version = "0.2.0"
description = "Specifications for callback functions passed in to an API"
optional = false
python-versions = "*"
+groups = ["main", "dev"]
files = [
{file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"},
{file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"},
@@ -476,6 +503,8 @@ version = "1.2.0"
description = "Backport of CPython tarfile module"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
+markers = "platform_system != \"Windows\" or platform_python_implementation == \"PyPy\""
files = [
{file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"},
{file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"},
@@ -491,6 +520,7 @@ version = "4.13.3"
description = "Screen-scraping library"
optional = false
python-versions = ">=3.7.0"
+groups = ["main", "dev"]
files = [
{file = "beautifulsoup4-4.13.3-py3-none-any.whl", hash = "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16"},
{file = "beautifulsoup4-4.13.3.tar.gz", hash = "sha256:1bd32405dacc920b42b83ba01644747ed77456a65760e285fbc47633ceddaf8b"},
@@ -513,6 +543,7 @@ version = "0.3.13"
description = "PyTorch implementation of BERT score"
optional = false
python-versions = ">=3.6"
+groups = ["main"]
files = [
{file = "bert_score-0.3.13-py3-none-any.whl", hash = "sha256:bbbb4c7fcdaa46d7681aff49f37f96faa09ed74e1b150e659bdc6b58a66989b9"},
{file = "bert_score-0.3.13.tar.gz", hash = "sha256:8ffe5838eac8cdd988b8b1a896af7f49071188c8c011a1ed160d71a9899a2ba4"},
@@ -534,6 +565,7 @@ version = "22.12.0"
description = "The uncompromising code formatter."
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"},
{file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"},
@@ -569,6 +601,7 @@ version = "6.1.0"
description = "An easy safelist-based HTML-sanitizing tool."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "bleach-6.1.0-py3-none-any.whl", hash = "sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6"},
{file = "bleach-6.1.0.tar.gz", hash = "sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe"},
@@ -588,6 +621,8 @@ version = "1.1.0"
description = "Python bindings for the Brotli compression library"
optional = false
python-versions = "*"
+groups = ["main"]
+markers = "platform_python_implementation == \"CPython\""
files = [
{file = "Brotli-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752"},
{file = "Brotli-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9"},
@@ -599,6 +634,10 @@ files = [
{file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d"},
{file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0"},
{file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e"},
+ {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c"},
+ {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1"},
+ {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2"},
+ {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec"},
{file = "Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2"},
{file = "Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128"},
{file = "Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc"},
@@ -611,8 +650,14 @@ files = [
{file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9"},
{file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265"},
{file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8"},
+ {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f"},
+ {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757"},
+ {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0"},
+ {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b"},
{file = "Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50"},
{file = "Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1"},
+ {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28"},
+ {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f"},
{file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409"},
{file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2"},
{file = "Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451"},
@@ -623,8 +668,24 @@ files = [
{file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180"},
{file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248"},
{file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966"},
+ {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9"},
+ {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb"},
+ {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111"},
+ {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839"},
{file = "Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0"},
{file = "Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951"},
+ {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5"},
+ {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8"},
+ {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f"},
+ {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648"},
+ {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0"},
+ {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089"},
+ {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368"},
+ {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c"},
+ {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284"},
+ {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7"},
+ {file = "Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0"},
+ {file = "Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b"},
{file = "Brotli-1.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1"},
{file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d"},
{file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b"},
@@ -634,6 +695,10 @@ files = [
{file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2"},
{file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354"},
{file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2"},
+ {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:aea440a510e14e818e67bfc4027880e2fb500c2ccb20ab21c7a7c8b5b4703d75"},
+ {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:6974f52a02321b36847cd19d1b8e381bf39939c21efd6ee2fc13a28b0d99348c"},
+ {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:a7e53012d2853a07a4a79c00643832161a910674a893d296c9f1259859a289d2"},
+ {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:d7702622a8b40c49bffb46e1e3ba2e81268d5c04a34f460978c6b5517a34dd52"},
{file = "Brotli-1.1.0-cp36-cp36m-win32.whl", hash = "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460"},
{file = "Brotli-1.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579"},
{file = "Brotli-1.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c"},
@@ -645,6 +710,10 @@ files = [
{file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74"},
{file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b"},
{file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438"},
+ {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:cb1dac1770878ade83f2ccdf7d25e494f05c9165f5246b46a621cc849341dc01"},
+ {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:3ee8a80d67a4334482d9712b8e83ca6b1d9bc7e351931252ebef5d8f7335a547"},
+ {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5e55da2c8724191e5b557f8e18943b1b4839b8efc3ef60d65985bcf6f587dd38"},
+ {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:d342778ef319e1026af243ed0a07c97acf3bad33b9f29e7ae6a1f68fd083e90c"},
{file = "Brotli-1.1.0-cp37-cp37m-win32.whl", hash = "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95"},
{file = "Brotli-1.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68"},
{file = "Brotli-1.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3"},
@@ -657,6 +726,10 @@ files = [
{file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a"},
{file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:141bd4d93984070e097521ed07e2575b46f817d08f9fa42b16b9b5f27b5ac088"},
{file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596"},
+ {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d2b35ca2c7f81d173d2fadc2f4f31e88cc5f7a39ae5b6db5513cf3383b0e0ec7"},
+ {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:af6fa6817889314555aede9a919612b23739395ce767fe7fcbea9a80bf140fe5"},
+ {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:2feb1d960f760a575dbc5ab3b1c00504b24caaf6986e2dc2b01c09c87866a943"},
+ {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4410f84b33374409552ac9b6903507cdb31cd30d2501fc5ca13d18f73548444a"},
{file = "Brotli-1.1.0-cp38-cp38-win32.whl", hash = "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b"},
{file = "Brotli-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0"},
{file = "Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a"},
@@ -669,6 +742,10 @@ files = [
{file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c"},
{file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d"},
{file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59"},
+ {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419"},
+ {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2"},
+ {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f"},
+ {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb"},
{file = "Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64"},
{file = "Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467"},
{file = "Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724"},
@@ -680,6 +757,8 @@ version = "1.1.0.0"
description = "Python CFFI bindings to the Brotli library"
optional = false
python-versions = ">=3.7"
+groups = ["main"]
+markers = "platform_python_implementation != \"CPython\""
files = [
{file = "brotlicffi-1.1.0.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9b7ae6bd1a3f0df532b6d67ff674099a96d22bc0948955cb338488c31bfb8851"},
{file = "brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19ffc919fa4fc6ace69286e0a23b3789b4219058313cf9b45625016bf7ff996b"},
@@ -719,6 +798,7 @@ version = "1.2.7"
description = "CatBoost Python Package"
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "catboost-1.2.7-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:12cd01533912f3b2b6cf4d1be7e7305f0870c109f5eb9f9a5dd48a5c07649e77"},
{file = "catboost-1.2.7-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:bc5611329fe843cff65196032517647b2d009d46da9f02bd30d92dca26e4c013"},
@@ -765,6 +845,7 @@ version = "2025.1.31"
description = "Python package for providing Mozilla's CA Bundle."
optional = false
python-versions = ">=3.6"
+groups = ["main", "dev"]
files = [
{file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"},
{file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"},
@@ -776,6 +857,7 @@ version = "1.17.1"
description = "Foreign Function Interface for Python calling C code."
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"},
{file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"},
@@ -845,6 +927,7 @@ files = [
{file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"},
{file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"},
]
+markers = {main = "platform_python_implementation != \"CPython\" or sys_platform == \"linux\" or sys_platform == \"darwin\""}
[package.dependencies]
pycparser = "*"
@@ -855,6 +938,7 @@ version = "3.4.0"
description = "Validate configuration and produce human readable error messages."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"},
{file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"},
@@ -866,6 +950,7 @@ version = "3.4.1"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "charset_normalizer-3.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de"},
{file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176"},
@@ -967,6 +1052,7 @@ version = "8.1.8"
description = "Composable command line interface toolkit"
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"},
{file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"},
@@ -981,6 +1067,7 @@ version = "3.1.1"
description = "Pickler class to extend the standard pickle.Pickler functionality"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "cloudpickle-3.1.1-py3-none-any.whl", hash = "sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e"},
{file = "cloudpickle-3.1.1.tar.gz", hash = "sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64"},
@@ -992,10 +1079,12 @@ version = "0.4.6"
description = "Cross-platform colored terminal text."
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+groups = ["main", "dev"]
files = [
{file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
]
+markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\""}
[[package]]
name = "comm"
@@ -1003,6 +1092,7 @@ version = "0.2.2"
description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc."
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "comm-0.2.2-py3-none-any.whl", hash = "sha256:e6fb86cb70ff661ee8c9c14e7d36d6de3b4066f1441be4063df9c5009f0a64d3"},
{file = "comm-0.2.2.tar.gz", hash = "sha256:3fd7a84065306e07bea1773df6eb8282de51ba82f77c72f9c85716ab11fe980e"},
@@ -1020,6 +1110,7 @@ version = "1.1.1"
description = "Python library for calculating contours of 2D quadrilateral grids"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "contourpy-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:46e24f5412c948d81736509377e255f6040e94216bf1a9b5ea1eaa9d29f6ec1b"},
{file = "contourpy-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e48694d6a9c5a26ee85b10130c77a011a4fedf50a7279fa0bdaf44bafb4299d"},
@@ -1091,6 +1182,8 @@ version = "43.0.3"
description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
+markers = "sys_platform == \"linux\""
files = [
{file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"},
{file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"},
@@ -1140,6 +1233,7 @@ version = "0.12.1"
description = "Composable style cycles"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30"},
{file = "cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"},
@@ -1155,6 +1249,7 @@ version = "0.29.37"
description = "The Cython compiler for writing C extensions for the Python language."
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+groups = ["dev"]
files = [
{file = "Cython-0.29.37-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f2d621fe4cb50007446742134a890500b34e3f50abaf7993baaca02634af7e15"},
{file = "Cython-0.29.37-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:d94caf90ae9cb56116ca6d54cdcbccd3c4df6b0cb7233922b2233ee7fe81d05b"},
@@ -1200,26 +1295,14 @@ files = [
{file = "Cython-0.29.37.tar.gz", hash = "sha256:f813d4a6dd94adee5d4ff266191d1d95bf6d4164a4facc535422c021b2504cfb"},
]
-[[package]]
-name = "dacite"
-version = "1.9.2"
-description = "Simple creation of data classes from dictionaries."
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "dacite-1.9.2-py3-none-any.whl", hash = "sha256:053f7c3f5128ca2e9aceb66892b1a3c8936d02c686e707bee96e19deef4bc4a0"},
- {file = "dacite-1.9.2.tar.gz", hash = "sha256:6ccc3b299727c7aa17582f0021f6ae14d5de47c7227932c47fec4cdfefd26f09"},
-]
-
-[package.extras]
-dev = ["black", "coveralls", "mypy", "pre-commit", "pylint", "pytest (>=5)", "pytest-benchmark", "pytest-cov"]
-
[[package]]
name = "dataclasses-json"
version = "0.6.7"
description = "Easily serialize dataclasses to and from JSON."
optional = true
python-versions = "<4.0,>=3.7"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a"},
{file = "dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0"},
@@ -1235,6 +1318,7 @@ version = "2.21.0"
description = "HuggingFace community-driven open-source library of datasets"
optional = false
python-versions = ">=3.8.0"
+groups = ["main"]
files = [
{file = "datasets-2.21.0-py3-none-any.whl", hash = "sha256:25e4e097110ce28824b746a107727ada94024cba11db8bc588d468414692b65a"},
{file = "datasets-2.21.0.tar.gz", hash = "sha256:998f85a8460f1bd982e5bd058f8a0808eef424249e3df1e8cdd594ccd0dc8ba2"},
@@ -1258,9 +1342,9 @@ xxhash = "*"
[package.extras]
apache-beam = ["apache-beam (>=2.26.0)"]
-audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0)"]
+audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""]
benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"]
-dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch", "torch (>=2.0.0)", "transformers", "transformers (>=4.42.0)", "typing-extensions (>=4.6.1)", "zstandard"]
+dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "transformers", "transformers (>=4.42.0)", "typing-extensions (>=4.6.1)", "zstandard"]
docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"]
jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"]
metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk (<3.8.2)", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"]
@@ -1268,8 +1352,8 @@ quality = ["ruff (>=0.3.0)"]
s3 = ["s3fs"]
tensorflow = ["tensorflow (>=2.6.0)"]
tensorflow-gpu = ["tensorflow (>=2.6.0)"]
-tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch (>=2.0.0)", "transformers (>=4.42.0)", "typing-extensions (>=4.6.1)", "zstandard"]
-tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "typing-extensions (>=4.6.1)", "zstandard"]
+tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "transformers (>=4.42.0)", "typing-extensions (>=4.6.1)", "zstandard"]
+tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "typing-extensions (>=4.6.1)", "zstandard"]
torch = ["torch"]
vision = ["Pillow (>=9.4.0)"]
@@ -1279,6 +1363,7 @@ version = "1.8.13"
description = "An implementation of the Debug Adapter Protocol for Python"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "debugpy-1.8.13-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:06859f68e817966723ffe046b896b1bd75c665996a77313370336ee9e1de3e90"},
{file = "debugpy-1.8.13-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb56c2db69fb8df3168bc857d7b7d2494fed295dfdbde9a45f27b4b152f37520"},
@@ -1314,6 +1399,7 @@ version = "5.2.1"
description = "Decorators for Humans"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a"},
{file = "decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360"},
@@ -1325,6 +1411,7 @@ version = "0.7.1"
description = "XML bomb protection for Python stdlib modules"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+groups = ["dev"]
files = [
{file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"},
{file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"},
@@ -1336,6 +1423,7 @@ version = "0.3.8"
description = "serialize all of Python"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"},
{file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"},
@@ -1351,6 +1439,7 @@ version = "0.3.9"
description = "Distribution utilities"
optional = false
python-versions = "*"
+groups = ["dev"]
files = [
{file = "distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87"},
{file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"},
@@ -1362,18 +1451,35 @@ version = "1.9.0"
description = "Distro - an OS platform information API"
optional = false
python-versions = ">=3.6"
+groups = ["main"]
files = [
{file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"},
{file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"},
]
+[[package]]
+name = "docstring-parser"
+version = "0.16"
+description = "Parse Python docstrings in reST, Google and Numpydoc format"
+optional = false
+python-versions = ">=3.6,<4.0"
+groups = ["dev"]
+files = [
+ {file = "docstring_parser-0.16-py3-none-any.whl", hash = "sha256:bf0a1387354d3691d102edef7ec124f219ef639982d096e26e3b60aeffa90637"},
+ {file = "docstring_parser-0.16.tar.gz", hash = "sha256:538beabd0af1e2db0146b6bd3caa526c35a34d61af9fd2887f3a8a27a739aa6e"},
+]
+
[[package]]
name = "docutils"
version = "0.18.1"
description = "Docutils -- Python Documentation Utilities"
optional = false
-python-versions = "*"
-files = []
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+groups = ["dev"]
+files = [
+ {file = "docutils-0.18.1-py2.py3-none-any.whl", hash = "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c"},
+ {file = "docutils-0.18.1.tar.gz", hash = "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"},
+]
[[package]]
name = "entrypoints"
@@ -1381,6 +1487,7 @@ version = "0.4"
description = "Discover and load entry points from installed packages."
optional = false
python-versions = ">=3.6"
+groups = ["dev"]
files = [
{file = "entrypoints-0.4-py3-none-any.whl", hash = "sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f"},
{file = "entrypoints-0.4.tar.gz", hash = "sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4"},
@@ -1392,6 +1499,7 @@ version = "0.4.3"
description = "HuggingFace community-driven open-source library of evaluation"
optional = false
python-versions = ">=3.8.0"
+groups = ["main"]
files = [
{file = "evaluate-0.4.3-py3-none-any.whl", hash = "sha256:47d8770bdea76e2c2ed0d40189273027d1a41ccea861bcc7ba12d30ec5d1e517"},
{file = "evaluate-0.4.3.tar.gz", hash = "sha256:3a5700cf83aabee9549264e1e5666f116367c61dbd4d38352015e859a5e2098d"},
@@ -1427,6 +1535,8 @@ version = "1.2.2"
description = "Backport of PEP 654 (exception groups)"
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
+markers = "python_version == \"3.10\" or python_version == \"3.9\" or python_version == \"3.8\""
files = [
{file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"},
{file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"},
@@ -1441,13 +1551,14 @@ version = "2.2.0"
description = "Get the currently executing AST node of a frame, and other information"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa"},
{file = "executing-2.2.0.tar.gz", hash = "sha256:5d108c028108fe2551d1a7b2e8b713341e2cb4fc0aa7dcf966fa4327a5226755"},
]
[package.extras]
-tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"]
+tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich ; python_version >= \"3.11\""]
[[package]]
name = "fastjsonschema"
@@ -1455,6 +1566,7 @@ version = "2.21.1"
description = "Fastest Python implementation of JSON schema"
optional = false
python-versions = "*"
+groups = ["dev"]
files = [
{file = "fastjsonschema-2.21.1-py3-none-any.whl", hash = "sha256:c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667"},
{file = "fastjsonschema-2.21.1.tar.gz", hash = "sha256:794d4f0a58f848961ba16af7b9c85a3e88cd360df008c59aac6fc5ae9323b5d4"},
@@ -1469,6 +1581,7 @@ version = "3.16.1"
description = "A platform independent file lock."
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"},
{file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"},
@@ -1477,7 +1590,7 @@ files = [
[package.extras]
docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"]
testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"]
-typing = ["typing-extensions (>=4.12.2)"]
+typing = ["typing-extensions (>=4.12.2) ; python_version < \"3.11\""]
[[package]]
name = "flake8"
@@ -1485,6 +1598,7 @@ version = "4.0.1"
description = "the modular source code checker: pep8 pyflakes and co"
optional = false
python-versions = ">=3.6"
+groups = ["dev"]
files = [
{file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"},
{file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"},
@@ -1501,6 +1615,7 @@ version = "4.56.0"
description = "Tools to manipulate font files"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "fonttools-4.56.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:331954d002dbf5e704c7f3756028e21db07097c19722569983ba4d74df014000"},
{file = "fonttools-4.56.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8d1613abd5af2f93c05867b3a3759a56e8bf97eb79b1da76b2bc10892f96ff16"},
@@ -1555,18 +1670,18 @@ files = [
]
[package.extras]
-all = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "fs (>=2.2.0,<3)", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres", "pycairo", "scipy", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0)", "xattr", "zopfli (>=0.1.4)"]
+all = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "fs (>=2.2.0,<3)", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\"", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0) ; python_version <= \"3.12\"", "xattr ; sys_platform == \"darwin\"", "zopfli (>=0.1.4)"]
graphite = ["lz4 (>=1.7.4.2)"]
-interpolatable = ["munkres", "pycairo", "scipy"]
+interpolatable = ["munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\""]
lxml = ["lxml (>=4.0)"]
pathops = ["skia-pathops (>=0.5.0)"]
plot = ["matplotlib"]
repacker = ["uharfbuzz (>=0.23.0)"]
symfont = ["sympy"]
-type1 = ["xattr"]
+type1 = ["xattr ; sys_platform == \"darwin\""]
ufo = ["fs (>=2.2.0,<3)"]
-unicode = ["unicodedata2 (>=15.1.0)"]
-woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"]
+unicode = ["unicodedata2 (>=15.1.0) ; python_version <= \"3.12\""]
+woff = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "zopfli (>=0.1.4)"]
[[package]]
name = "fqdn"
@@ -1574,6 +1689,7 @@ version = "1.5.1"
description = "Validates fully-qualified domain names against RFC 1123, so that they are acceptable to modern bowsers"
optional = false
python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4, <4"
+groups = ["dev"]
files = [
{file = "fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014"},
{file = "fqdn-1.5.1.tar.gz", hash = "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f"},
@@ -1585,6 +1701,7 @@ version = "2.4.6"
description = "A simple immutable dictionary"
optional = false
python-versions = ">=3.6"
+groups = ["main"]
files = [
{file = "frozendict-2.4.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3a05c0a50cab96b4bb0ea25aa752efbfceed5ccb24c007612bc63e51299336f"},
{file = "frozendict-2.4.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f5b94d5b07c00986f9e37a38dd83c13f5fe3bf3f1ccc8e88edea8fe15d6cd88c"},
@@ -1633,6 +1750,7 @@ version = "1.5.0"
description = "A list-like structure which implements collections.abc.MutableSequence"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5b6a66c18b5b9dd261ca98dffcb826a525334b2f29e7caa54e182255c5f6a65a"},
{file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1b3eb7b05ea246510b43a7e53ed1653e55c2121019a97e60cad7efb881a97bb"},
@@ -1734,6 +1852,7 @@ version = "2024.6.1"
description = "File-system specification"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e"},
{file = "fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"},
@@ -1776,6 +1895,7 @@ version = "0.20.3"
description = "Simple Python interface for Graphviz"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "graphviz-0.20.3-py3-none-any.whl", hash = "sha256:81f848f2904515d8cd359cc611faba817598d2feaac4027b266aa3eda7b3dde5"},
{file = "graphviz-0.20.3.zip", hash = "sha256:09d6bc81e6a9fa392e7ba52135a9d49f1ed62526f96499325930e87ca1b5925d"},
@@ -1792,6 +1912,8 @@ version = "3.1.1"
description = "Lightweight in-process concurrent programming"
optional = true
python-versions = ">=3.7"
+groups = ["main"]
+markers = "(platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\") and (extra == \"all\" or extra == \"llm\")"
files = [
{file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"},
{file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"},
@@ -1872,12 +1994,29 @@ files = [
docs = ["Sphinx", "furo"]
test = ["objgraph", "psutil"]
+[[package]]
+name = "griffe"
+version = "1.4.0"
+description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API."
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+ {file = "griffe-1.4.0-py3-none-any.whl", hash = "sha256:e589de8b8c137e99a46ec45f9598fc0ac5b6868ce824b24db09c02d117b89bc5"},
+ {file = "griffe-1.4.0.tar.gz", hash = "sha256:8fccc585896d13f1221035d32c50dec65830c87d23f9adb9b1e6f3d63574f7f5"},
+]
+
+[package.dependencies]
+astunparse = {version = ">=1.6", markers = "python_version < \"3.9\""}
+colorama = ">=0.4"
+
[[package]]
name = "h11"
version = "0.14.0"
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
{file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
@@ -1889,26 +2028,18 @@ version = "2024.2.26"
description = "Turn HTML into equivalent Markdown-structured text."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "html2text-2024.2.26.tar.gz", hash = "sha256:05f8e367d15aaabc96415376776cdd11afd5127a77fce6e36afc60c563ca2c32"},
]
-[[package]]
-name = "htmlmin"
-version = "0.1.12"
-description = "An HTML Minifier"
-optional = false
-python-versions = "*"
-files = [
- {file = "htmlmin-0.1.12.tar.gz", hash = "sha256:50c1ef4630374a5d723900096a961cff426dff46b48f34d194a81bbe14eca178"},
-]
-
[[package]]
name = "httpcore"
version = "1.0.7"
description = "A minimal low-level HTTP client."
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"},
{file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"},
@@ -1930,6 +2061,7 @@ version = "0.28.1"
description = "The next generation HTTP client."
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"},
{file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"},
@@ -1942,7 +2074,7 @@ httpcore = "==1.*"
idna = "*"
[package.extras]
-brotli = ["brotli", "brotlicffi"]
+brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""]
cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
http2 = ["h2 (>=3,<5)"]
socks = ["socksio (==1.*)"]
@@ -1950,13 +2082,14 @@ zstd = ["zstandard (>=0.18.0)"]
[[package]]
name = "huggingface-hub"
-version = "0.29.2"
+version = "0.29.3"
description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"
optional = false
python-versions = ">=3.8.0"
+groups = ["main"]
files = [
- {file = "huggingface_hub-0.29.2-py3-none-any.whl", hash = "sha256:c56f20fca09ef19da84dcde2b76379ecdaddf390b083f59f166715584953307d"},
- {file = "huggingface_hub-0.29.2.tar.gz", hash = "sha256:590b29c0dcbd0ee4b7b023714dc1ad8563fe4a68a91463438b74e980d28afaf3"},
+ {file = "huggingface_hub-0.29.3-py3-none-any.whl", hash = "sha256:0b25710932ac649c08cdbefa6c6ccb8e88eef82927cacdb048efb726429453aa"},
+ {file = "huggingface_hub-0.29.3.tar.gz", hash = "sha256:64519a25716e0ba382ba2d3fb3ca082e7c7eb4a2fc634d200e8380006e0760e5"},
]
[package.dependencies]
@@ -1988,6 +2121,7 @@ version = "2.6.1"
description = "File identification library for Python"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "identify-2.6.1-py2.py3-none-any.whl", hash = "sha256:53863bcac7caf8d2ed85bd20312ea5dcfc22226800f6d6881f232d861db5a8f0"},
{file = "identify-2.6.1.tar.gz", hash = "sha256:91478c5fb7c3aac5ff7bf9b4344f803843dc586832d5f110d672b19aa1984c98"},
@@ -2002,6 +2136,7 @@ version = "3.10"
description = "Internationalized Domain Names in Applications (IDNA)"
optional = false
python-versions = ">=3.6"
+groups = ["main", "dev"]
files = [
{file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"},
{file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"},
@@ -2010,29 +2145,13 @@ files = [
[package.extras]
all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"]
-[[package]]
-name = "imagehash"
-version = "4.3.1"
-description = "Image Hashing library"
-optional = false
-python-versions = "*"
-files = [
- {file = "ImageHash-4.3.1-py2.py3-none-any.whl", hash = "sha256:5ad9a5cde14fe255745a8245677293ac0d67f09c330986a351f34b614ba62fb5"},
- {file = "ImageHash-4.3.1.tar.gz", hash = "sha256:7038d1b7f9e0585beb3dd8c0a956f02b95a346c0b5f24a9e8cc03ebadaf0aa70"},
-]
-
-[package.dependencies]
-numpy = "*"
-pillow = "*"
-PyWavelets = "*"
-scipy = "*"
-
[[package]]
name = "imagesize"
version = "1.4.1"
description = "Getting image size from png/jpeg/jpeg2000/gif file"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+groups = ["dev"]
files = [
{file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"},
{file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"},
@@ -2044,21 +2163,23 @@ version = "8.5.0"
description = "Read metadata from Python packages"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"},
{file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"},
]
+markers = {main = "python_version == \"3.8\""}
[package.dependencies]
zipp = ">=3.20"
[package.extras]
-check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"]
+check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""]
cover = ["pytest-cov"]
doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
enabler = ["pytest-enabler (>=2.2)"]
perf = ["ipython"]
-test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"]
+test = ["flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"]
type = ["pytest-mypy"]
[[package]]
@@ -2067,16 +2188,18 @@ version = "6.4.5"
description = "Read resources from Python packages"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "importlib_resources-6.4.5-py3-none-any.whl", hash = "sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717"},
{file = "importlib_resources-6.4.5.tar.gz", hash = "sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065"},
]
+markers = {main = "python_version == \"3.8\" or python_version == \"3.9\"", dev = "python_version == \"3.8\""}
[package.dependencies]
zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""}
[package.extras]
-check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"]
+check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""]
cover = ["pytest-cov"]
doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
enabler = ["pytest-enabler (>=2.2)"]
@@ -2089,6 +2212,7 @@ version = "6.29.5"
description = "IPython Kernel for Jupyter"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "ipykernel-6.29.5-py3-none-any.whl", hash = "sha256:afdb66ba5aa354b09b91379bac28ae4afebbb30e8b39510c9690afb7a10421b5"},
{file = "ipykernel-6.29.5.tar.gz", hash = "sha256:f093a22c4a40f8828f8e330a9c297cb93dcab13bd9678ded6de8e5cf81c56215"},
@@ -2122,6 +2246,7 @@ version = "8.12.3"
description = "IPython: Productive Interactive Computing"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "ipython-8.12.3-py3-none-any.whl", hash = "sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c"},
{file = "ipython-8.12.3.tar.gz", hash = "sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363"},
@@ -2161,6 +2286,7 @@ version = "8.1.5"
description = "Jupyter interactive widgets"
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "ipywidgets-8.1.5-py3-none-any.whl", hash = "sha256:3290f526f87ae6e77655555baba4f36681c555b8bdbbff430b70e52c34c86245"},
{file = "ipywidgets-8.1.5.tar.gz", hash = "sha256:870e43b1a35656a80c18c9503bbf2d16802db1cb487eec6fab27d683381dde17"},
@@ -2182,6 +2308,7 @@ version = "20.11.0"
description = "Operations with ISO 8601 durations"
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042"},
{file = "isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9"},
@@ -2196,6 +2323,7 @@ version = "5.13.2"
description = "A Python utility / library to sort Python imports."
optional = false
python-versions = ">=3.8.0"
+groups = ["dev"]
files = [
{file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"},
{file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"},
@@ -2210,6 +2338,7 @@ version = "3.4.0"
description = "Utility functions for Python class constructs"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790"},
{file = "jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd"},
@@ -2228,6 +2357,7 @@ version = "6.0.1"
description = "Useful decorators and context managers"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jaraco.context-6.0.1-py3-none-any.whl", hash = "sha256:f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4"},
{file = "jaraco_context-6.0.1.tar.gz", hash = "sha256:9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3"},
@@ -2238,7 +2368,7 @@ files = [
[package.extras]
doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
-test = ["portend", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)"]
+test = ["portend", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""]
[[package]]
name = "jaraco-functools"
@@ -2246,6 +2376,7 @@ version = "4.1.0"
description = "Functools like those found in stdlib"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jaraco.functools-4.1.0-py3-none-any.whl", hash = "sha256:ad159f13428bc4acbf5541ad6dec511f91573b90fba04df61dafa2a1231cf649"},
{file = "jaraco_functools-4.1.0.tar.gz", hash = "sha256:70f7e0e2ae076498e212562325e805204fc092d7b4c17e0e86c959e249701a9d"},
@@ -2255,7 +2386,7 @@ files = [
more-itertools = "*"
[package.extras]
-check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"]
+check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""]
cover = ["pytest-cov"]
doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
enabler = ["pytest-enabler (>=2.2)"]
@@ -2268,6 +2399,7 @@ version = "0.19.2"
description = "An autocompletion tool for Python that can be used for text editors."
optional = false
python-versions = ">=3.6"
+groups = ["main", "dev"]
files = [
{file = "jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9"},
{file = "jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0"},
@@ -2287,13 +2419,15 @@ version = "0.9.0"
description = "Low-level, pure Python DBus protocol wrapper."
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
+markers = "sys_platform == \"linux\""
files = [
{file = "jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683"},
{file = "jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732"},
]
[package.extras]
-test = ["async-timeout", "pytest", "pytest-asyncio (>=0.17)", "pytest-trio", "testpath", "trio"]
+test = ["async-timeout ; python_version < \"3.11\"", "pytest", "pytest-asyncio (>=0.17)", "pytest-trio", "testpath", "trio"]
trio = ["trio"]
[[package]]
@@ -2302,6 +2436,7 @@ version = "3.1.6"
description = "A very fast and expressive template engine."
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"},
{file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"},
@@ -2315,87 +2450,88 @@ i18n = ["Babel (>=2.7)"]
[[package]]
name = "jiter"
-version = "0.8.2"
+version = "0.9.0"
description = "Fast iterable JSON parser."
optional = false
python-versions = ">=3.8"
-files = [
- {file = "jiter-0.8.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:ca8577f6a413abe29b079bc30f907894d7eb07a865c4df69475e868d73e71c7b"},
- {file = "jiter-0.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b25bd626bde7fb51534190c7e3cb97cee89ee76b76d7585580e22f34f5e3f393"},
- {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5c826a221851a8dc028eb6d7d6429ba03184fa3c7e83ae01cd6d3bd1d4bd17d"},
- {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d35c864c2dff13dfd79fb070fc4fc6235d7b9b359efe340e1261deb21b9fcb66"},
- {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f557c55bc2b7676e74d39d19bcb8775ca295c7a028246175d6a8b431e70835e5"},
- {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:580ccf358539153db147e40751a0b41688a5ceb275e6f3e93d91c9467f42b2e3"},
- {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af102d3372e917cffce49b521e4c32c497515119dc7bd8a75665e90a718bbf08"},
- {file = "jiter-0.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cadcc978f82397d515bb2683fc0d50103acff2a180552654bb92d6045dec2c49"},
- {file = "jiter-0.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ba5bdf56969cad2019d4e8ffd3f879b5fdc792624129741d3d83fc832fef8c7d"},
- {file = "jiter-0.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3b94a33a241bee9e34b8481cdcaa3d5c2116f575e0226e421bed3f7a6ea71cff"},
- {file = "jiter-0.8.2-cp310-cp310-win32.whl", hash = "sha256:6e5337bf454abddd91bd048ce0dca5134056fc99ca0205258766db35d0a2ea43"},
- {file = "jiter-0.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:4a9220497ca0cb1fe94e3f334f65b9b5102a0b8147646118f020d8ce1de70105"},
- {file = "jiter-0.8.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:2dd61c5afc88a4fda7d8b2cf03ae5947c6ac7516d32b7a15bf4b49569a5c076b"},
- {file = "jiter-0.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a6c710d657c8d1d2adbbb5c0b0c6bfcec28fd35bd6b5f016395f9ac43e878a15"},
- {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9584de0cd306072635fe4b89742bf26feae858a0683b399ad0c2509011b9dc0"},
- {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5a90a923338531b7970abb063cfc087eebae6ef8ec8139762007188f6bc69a9f"},
- {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21974d246ed0181558087cd9f76e84e8321091ebfb3a93d4c341479a736f099"},
- {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32475a42b2ea7b344069dc1e81445cfc00b9d0e3ca837f0523072432332e9f74"},
- {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b9931fd36ee513c26b5bf08c940b0ac875de175341cbdd4fa3be109f0492586"},
- {file = "jiter-0.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce0820f4a3a59ddced7fce696d86a096d5cc48d32a4183483a17671a61edfddc"},
- {file = "jiter-0.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8ffc86ae5e3e6a93765d49d1ab47b6075a9c978a2b3b80f0f32628f39caa0c88"},
- {file = "jiter-0.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5127dc1abd809431172bc3fbe8168d6b90556a30bb10acd5ded41c3cfd6f43b6"},
- {file = "jiter-0.8.2-cp311-cp311-win32.whl", hash = "sha256:66227a2c7b575720c1871c8800d3a0122bb8ee94edb43a5685aa9aceb2782d44"},
- {file = "jiter-0.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:cde031d8413842a1e7501e9129b8e676e62a657f8ec8166e18a70d94d4682855"},
- {file = "jiter-0.8.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e6ec2be506e7d6f9527dae9ff4b7f54e68ea44a0ef6b098256ddf895218a2f8f"},
- {file = "jiter-0.8.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76e324da7b5da060287c54f2fabd3db5f76468006c811831f051942bf68c9d44"},
- {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:180a8aea058f7535d1c84183c0362c710f4750bef66630c05f40c93c2b152a0f"},
- {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:025337859077b41548bdcbabe38698bcd93cfe10b06ff66617a48ff92c9aec60"},
- {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ecff0dc14f409599bbcafa7e470c00b80f17abc14d1405d38ab02e4b42e55b57"},
- {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ffd9fee7d0775ebaba131f7ca2e2d83839a62ad65e8e02fe2bd8fc975cedeb9e"},
- {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14601dcac4889e0a1c75ccf6a0e4baf70dbc75041e51bcf8d0e9274519df6887"},
- {file = "jiter-0.8.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:92249669925bc1c54fcd2ec73f70f2c1d6a817928480ee1c65af5f6b81cdf12d"},
- {file = "jiter-0.8.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e725edd0929fa79f8349ab4ec7f81c714df51dc4e991539a578e5018fa4a7152"},
- {file = "jiter-0.8.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bf55846c7b7a680eebaf9c3c48d630e1bf51bdf76c68a5f654b8524335b0ad29"},
- {file = "jiter-0.8.2-cp312-cp312-win32.whl", hash = "sha256:7efe4853ecd3d6110301665a5178b9856be7e2a9485f49d91aa4d737ad2ae49e"},
- {file = "jiter-0.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:83c0efd80b29695058d0fd2fa8a556490dbce9804eac3e281f373bbc99045f6c"},
- {file = "jiter-0.8.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:ca1f08b8e43dc3bd0594c992fb1fd2f7ce87f7bf0d44358198d6da8034afdf84"},
- {file = "jiter-0.8.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5672a86d55416ccd214c778efccf3266b84f87b89063b582167d803246354be4"},
- {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58dc9bc9767a1101f4e5e22db1b652161a225874d66f0e5cb8e2c7d1c438b587"},
- {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:37b2998606d6dadbb5ccda959a33d6a5e853252d921fec1792fc902351bb4e2c"},
- {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ab9a87f3784eb0e098f84a32670cfe4a79cb6512fd8f42ae3d0709f06405d18"},
- {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:79aec8172b9e3c6d05fd4b219d5de1ac616bd8da934107325a6c0d0e866a21b6"},
- {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:711e408732d4e9a0208008e5892c2966b485c783cd2d9a681f3eb147cf36c7ef"},
- {file = "jiter-0.8.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:653cf462db4e8c41995e33d865965e79641ef45369d8a11f54cd30888b7e6ff1"},
- {file = "jiter-0.8.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:9c63eaef32b7bebac8ebebf4dabebdbc6769a09c127294db6babee38e9f405b9"},
- {file = "jiter-0.8.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:eb21aaa9a200d0a80dacc7a81038d2e476ffe473ffdd9c91eb745d623561de05"},
- {file = "jiter-0.8.2-cp313-cp313-win32.whl", hash = "sha256:789361ed945d8d42850f919342a8665d2dc79e7e44ca1c97cc786966a21f627a"},
- {file = "jiter-0.8.2-cp313-cp313-win_amd64.whl", hash = "sha256:ab7f43235d71e03b941c1630f4b6e3055d46b6cb8728a17663eaac9d8e83a865"},
- {file = "jiter-0.8.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b426f72cd77da3fec300ed3bc990895e2dd6b49e3bfe6c438592a3ba660e41ca"},
- {file = "jiter-0.8.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2dd880785088ff2ad21ffee205e58a8c1ddabc63612444ae41e5e4b321b39c0"},
- {file = "jiter-0.8.2-cp313-cp313t-win_amd64.whl", hash = "sha256:3ac9f578c46f22405ff7f8b1f5848fb753cc4b8377fbec8470a7dc3997ca7566"},
- {file = "jiter-0.8.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:9e1fa156ee9454642adb7e7234a383884452532bc9d53d5af2d18d98ada1d79c"},
- {file = "jiter-0.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0cf5dfa9956d96ff2efb0f8e9c7d055904012c952539a774305aaaf3abdf3d6c"},
- {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e52bf98c7e727dd44f7c4acb980cb988448faeafed8433c867888268899b298b"},
- {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a2ecaa3c23e7a7cf86d00eda3390c232f4d533cd9ddea4b04f5d0644faf642c5"},
- {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:08d4c92bf480e19fc3f2717c9ce2aa31dceaa9163839a311424b6862252c943e"},
- {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99d9a1eded738299ba8e106c6779ce5c3893cffa0e32e4485d680588adae6db8"},
- {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d20be8b7f606df096e08b0b1b4a3c6f0515e8dac296881fe7461dfa0fb5ec817"},
- {file = "jiter-0.8.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d33f94615fcaf872f7fd8cd98ac3b429e435c77619777e8a449d9d27e01134d1"},
- {file = "jiter-0.8.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:317b25e98a35ffec5c67efe56a4e9970852632c810d35b34ecdd70cc0e47b3b6"},
- {file = "jiter-0.8.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fc9043259ee430ecd71d178fccabd8c332a3bf1e81e50cae43cc2b28d19e4cb7"},
- {file = "jiter-0.8.2-cp38-cp38-win32.whl", hash = "sha256:fc5adda618205bd4678b146612ce44c3cbfdee9697951f2c0ffdef1f26d72b63"},
- {file = "jiter-0.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:cd646c827b4f85ef4a78e4e58f4f5854fae0caf3db91b59f0d73731448a970c6"},
- {file = "jiter-0.8.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:e41e75344acef3fc59ba4765df29f107f309ca9e8eace5baacabd9217e52a5ee"},
- {file = "jiter-0.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f22b16b35d5c1df9dfd58843ab2cd25e6bf15191f5a236bed177afade507bfc"},
- {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7200b8f7619d36aa51c803fd52020a2dfbea36ffec1b5e22cab11fd34d95a6d"},
- {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:70bf4c43652cc294040dbb62256c83c8718370c8b93dd93d934b9a7bf6c4f53c"},
- {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f9d471356dc16f84ed48768b8ee79f29514295c7295cb41e1133ec0b2b8d637d"},
- {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:859e8eb3507894093d01929e12e267f83b1d5f6221099d3ec976f0c995cb6bd9"},
- {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaa58399c01db555346647a907b4ef6d4f584b123943be6ed5588c3f2359c9f4"},
- {file = "jiter-0.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8f2d5ed877f089862f4c7aacf3a542627c1496f972a34d0474ce85ee7d939c27"},
- {file = "jiter-0.8.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:03c9df035d4f8d647f8c210ddc2ae0728387275340668fb30d2421e17d9a0841"},
- {file = "jiter-0.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8bd2a824d08d8977bb2794ea2682f898ad3d8837932e3a74937e93d62ecbb637"},
- {file = "jiter-0.8.2-cp39-cp39-win32.whl", hash = "sha256:ca29b6371ebc40e496995c94b988a101b9fbbed48a51190a4461fcb0a68b4a36"},
- {file = "jiter-0.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:1c0dfbd1be3cbefc7510102370d86e35d1d53e5a93d48519688b1bf0f761160a"},
- {file = "jiter-0.8.2.tar.gz", hash = "sha256:cd73d3e740666d0e639f678adb176fad25c1bcbdae88d8d7b857e1783bb4212d"},
+groups = ["main"]
+files = [
+ {file = "jiter-0.9.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:816ec9b60fdfd1fec87da1d7ed46c66c44ffec37ab2ef7de5b147b2fce3fd5ad"},
+ {file = "jiter-0.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b1d3086f8a3ee0194ecf2008cf81286a5c3e540d977fa038ff23576c023c0ea"},
+ {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1339f839b91ae30b37c409bf16ccd3dc453e8b8c3ed4bd1d6a567193651a4a51"},
+ {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ffba79584b3b670fefae66ceb3a28822365d25b7bf811e030609a3d5b876f538"},
+ {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cfc7d0a8e899089d11f065e289cb5b2daf3d82fbe028f49b20d7b809193958d"},
+ {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e00a1a2bbfaaf237e13c3d1592356eab3e9015d7efd59359ac8b51eb56390a12"},
+ {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1d9870561eb26b11448854dce0ff27a9a27cb616b632468cafc938de25e9e51"},
+ {file = "jiter-0.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9872aeff3f21e437651df378cb75aeb7043e5297261222b6441a620218b58708"},
+ {file = "jiter-0.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1fd19112d1049bdd47f17bfbb44a2c0001061312dcf0e72765bfa8abd4aa30e5"},
+ {file = "jiter-0.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6ef5da104664e526836070e4a23b5f68dec1cc673b60bf1edb1bfbe8a55d0678"},
+ {file = "jiter-0.9.0-cp310-cp310-win32.whl", hash = "sha256:cb12e6d65ebbefe5518de819f3eda53b73187b7089040b2d17f5b39001ff31c4"},
+ {file = "jiter-0.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:c43ca669493626d8672be3b645dbb406ef25af3f4b6384cfd306da7eb2e70322"},
+ {file = "jiter-0.9.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6c4d99c71508912a7e556d631768dcdef43648a93660670986916b297f1c54af"},
+ {file = "jiter-0.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8f60fb8ce7df529812bf6c625635a19d27f30806885139e367af93f6e734ef58"},
+ {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51c4e1a4f8ea84d98b7b98912aa4290ac3d1eabfde8e3c34541fae30e9d1f08b"},
+ {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f4c677c424dc76684fea3e7285a7a2a7493424bea89ac441045e6a1fb1d7b3b"},
+ {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2221176dfec87f3470b21e6abca056e6b04ce9bff72315cb0b243ca9e835a4b5"},
+ {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3c7adb66f899ffa25e3c92bfcb593391ee1947dbdd6a9a970e0d7e713237d572"},
+ {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c98d27330fdfb77913c1097a7aab07f38ff2259048949f499c9901700789ac15"},
+ {file = "jiter-0.9.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eda3f8cc74df66892b1d06b5d41a71670c22d95a1ca2cbab73654745ce9d0419"},
+ {file = "jiter-0.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:dd5ab5ddc11418dce28343123644a100f487eaccf1de27a459ab36d6cca31043"},
+ {file = "jiter-0.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:42f8a68a69f047b310319ef8e2f52fdb2e7976fb3313ef27df495cf77bcad965"},
+ {file = "jiter-0.9.0-cp311-cp311-win32.whl", hash = "sha256:a25519efb78a42254d59326ee417d6f5161b06f5da827d94cf521fed961b1ff2"},
+ {file = "jiter-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:923b54afdd697dfd00d368b7ccad008cccfeb1efb4e621f32860c75e9f25edbd"},
+ {file = "jiter-0.9.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:7b46249cfd6c48da28f89eb0be3f52d6fdb40ab88e2c66804f546674e539ec11"},
+ {file = "jiter-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:609cf3c78852f1189894383cf0b0b977665f54cb38788e3e6b941fa6d982c00e"},
+ {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d726a3890a54561e55a9c5faea1f7655eda7f105bd165067575ace6e65f80bb2"},
+ {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2e89dc075c1fef8fa9be219e249f14040270dbc507df4215c324a1839522ea75"},
+ {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04e8ffa3c353b1bc4134f96f167a2082494351e42888dfcf06e944f2729cbe1d"},
+ {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:203f28a72a05ae0e129b3ed1f75f56bc419d5f91dfacd057519a8bd137b00c42"},
+ {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fca1a02ad60ec30bb230f65bc01f611c8608b02d269f998bc29cca8619a919dc"},
+ {file = "jiter-0.9.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:237e5cee4d5d2659aaf91bbf8ec45052cc217d9446070699441a91b386ae27dc"},
+ {file = "jiter-0.9.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:528b6b71745e7326eed73c53d4aa57e2a522242320b6f7d65b9c5af83cf49b6e"},
+ {file = "jiter-0.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9f48e86b57bc711eb5acdfd12b6cb580a59cc9a993f6e7dcb6d8b50522dcd50d"},
+ {file = "jiter-0.9.0-cp312-cp312-win32.whl", hash = "sha256:699edfde481e191d81f9cf6d2211debbfe4bd92f06410e7637dffb8dd5dfde06"},
+ {file = "jiter-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:099500d07b43f61d8bd780466d429c45a7b25411b334c60ca875fa775f68ccb0"},
+ {file = "jiter-0.9.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:2764891d3f3e8b18dce2cff24949153ee30c9239da7c00f032511091ba688ff7"},
+ {file = "jiter-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:387b22fbfd7a62418d5212b4638026d01723761c75c1c8232a8b8c37c2f1003b"},
+ {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d8da8629ccae3606c61d9184970423655fb4e33d03330bcdfe52d234d32f69"},
+ {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1be73d8982bdc278b7b9377426a4b44ceb5c7952073dd7488e4ae96b88e1103"},
+ {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2228eaaaa111ec54b9e89f7481bffb3972e9059301a878d085b2b449fbbde635"},
+ {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:11509bfecbc319459647d4ac3fd391d26fdf530dad00c13c4dadabf5b81f01a4"},
+ {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f22238da568be8bbd8e0650e12feeb2cfea15eda4f9fc271d3b362a4fa0604d"},
+ {file = "jiter-0.9.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:17f5d55eb856597607562257c8e36c42bc87f16bef52ef7129b7da11afc779f3"},
+ {file = "jiter-0.9.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6a99bed9fbb02f5bed416d137944419a69aa4c423e44189bc49718859ea83bc5"},
+ {file = "jiter-0.9.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e057adb0cd1bd39606100be0eafe742de2de88c79df632955b9ab53a086b3c8d"},
+ {file = "jiter-0.9.0-cp313-cp313-win32.whl", hash = "sha256:f7e6850991f3940f62d387ccfa54d1a92bd4bb9f89690b53aea36b4364bcab53"},
+ {file = "jiter-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:c8ae3bf27cd1ac5e6e8b7a27487bf3ab5f82318211ec2e1346a5b058756361f7"},
+ {file = "jiter-0.9.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f0b2827fb88dda2cbecbbc3e596ef08d69bda06c6f57930aec8e79505dc17001"},
+ {file = "jiter-0.9.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:062b756ceb1d40b0b28f326cba26cfd575a4918415b036464a52f08632731e5a"},
+ {file = "jiter-0.9.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6f7838bc467ab7e8ef9f387bd6de195c43bad82a569c1699cb822f6609dd4cdf"},
+ {file = "jiter-0.9.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4a2d16360d0642cd68236f931b85fe50288834c383492e4279d9f1792e309571"},
+ {file = "jiter-0.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e84ed1c9c9ec10bbb8c37f450077cbe3c0d4e8c2b19f0a49a60ac7ace73c7452"},
+ {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f3c848209ccd1bfa344a1240763975ca917de753c7875c77ec3034f4151d06c"},
+ {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7825f46e50646bee937e0f849d14ef3a417910966136f59cd1eb848b8b5bb3e4"},
+ {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d82a811928b26d1a6311a886b2566f68ccf2b23cf3bfed042e18686f1f22c2d7"},
+ {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c058ecb51763a67f019ae423b1cbe3fa90f7ee6280c31a1baa6ccc0c0e2d06e"},
+ {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9897115ad716c48f0120c1f0c4efae348ec47037319a6c63b2d7838bb53aaef4"},
+ {file = "jiter-0.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:351f4c90a24c4fb8c87c6a73af2944c440494ed2bea2094feecacb75c50398ae"},
+ {file = "jiter-0.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d45807b0f236c485e1e525e2ce3a854807dfe28ccf0d013dd4a563395e28008a"},
+ {file = "jiter-0.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1537a890724ba00fdba21787010ac6f24dad47f763410e9e1093277913592784"},
+ {file = "jiter-0.9.0-cp38-cp38-win32.whl", hash = "sha256:e3630ec20cbeaddd4b65513fa3857e1b7c4190d4481ef07fb63d0fad59033321"},
+ {file = "jiter-0.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:2685f44bf80e95f8910553bf2d33b9c87bf25fceae6e9f0c1355f75d2922b0ee"},
+ {file = "jiter-0.9.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:9ef340fae98065071ccd5805fe81c99c8f80484e820e40043689cf97fb66b3e2"},
+ {file = "jiter-0.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:efb767d92c63b2cd9ec9f24feeb48f49574a713870ec87e9ba0c2c6e9329c3e2"},
+ {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:113f30f87fb1f412510c6d7ed13e91422cfd329436364a690c34c8b8bd880c42"},
+ {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8793b6df019b988526f5a633fdc7456ea75e4a79bd8396a3373c371fc59f5c9b"},
+ {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7a9aaa5102dba4e079bb728076fadd5a2dca94c05c04ce68004cfd96f128ea34"},
+ {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d838650f6ebaf4ccadfb04522463e74a4c378d7e667e0eb1865cfe3990bfac49"},
+ {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0194f813efdf4b8865ad5f5c5f50f8566df7d770a82c51ef593d09e0b347020"},
+ {file = "jiter-0.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a7954a401d0a8a0b8bc669199db78af435aae1e3569187c2939c477c53cb6a0a"},
+ {file = "jiter-0.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4feafe787eb8a8d98168ab15637ca2577f6ddf77ac6c8c66242c2d028aa5420e"},
+ {file = "jiter-0.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:27cd1f2e8bb377f31d3190b34e4328d280325ad7ef55c6ac9abde72f79e84d2e"},
+ {file = "jiter-0.9.0-cp39-cp39-win32.whl", hash = "sha256:161d461dcbe658cf0bd0aa375b30a968b087cdddc624fc585f3867c63c6eca95"},
+ {file = "jiter-0.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:e8b36d8a16a61993be33e75126ad3d8aa29cf450b09576f3c427d27647fcb4aa"},
+ {file = "jiter-0.9.0.tar.gz", hash = "sha256:aadba0964deb424daa24492abc3d229c60c4a31bfee205aedbf1acc7639d7893"},
]
[[package]]
@@ -2404,6 +2540,7 @@ version = "1.4.2"
description = "Lightweight pipelining with Python functions"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"},
{file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"},
@@ -2415,6 +2552,7 @@ version = "0.10.0"
description = "A Python implementation of the JSON5 data format."
optional = false
python-versions = ">=3.8.0"
+groups = ["dev"]
files = [
{file = "json5-0.10.0-py3-none-any.whl", hash = "sha256:19b23410220a7271e8377f81ba8aacba2fdd56947fbb137ee5977cbe1f5e8dfa"},
{file = "json5-0.10.0.tar.gz", hash = "sha256:e66941c8f0a02026943c52c2eb34ebeb2a6f819a0be05920a6f5243cd30fd559"},
@@ -2429,6 +2567,8 @@ version = "1.33"
description = "Apply JSON-Patches (RFC 6902)"
optional = true
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"},
{file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"},
@@ -2443,10 +2583,12 @@ version = "3.0.0"
description = "Identify specific nodes in a JSON document (RFC 6901)"
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942"},
{file = "jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"},
]
+markers = {main = "extra == \"all\" or extra == \"llm\""}
[[package]]
name = "jsonschema"
@@ -2454,6 +2596,7 @@ version = "4.23.0"
description = "An implementation of JSON Schema validation for Python"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"},
{file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"},
@@ -2485,6 +2628,7 @@ version = "2023.12.1"
description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"},
{file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"},
@@ -2500,6 +2644,7 @@ version = "1.1.1"
description = "Jupyter metapackage. Install all the Jupyter components in one go."
optional = false
python-versions = "*"
+groups = ["dev"]
files = [
{file = "jupyter-1.1.1-py2.py3-none-any.whl", hash = "sha256:7a59533c22af65439b24bbe60373a4e95af8f16ac65a6c00820ad378e3f7cc83"},
{file = "jupyter-1.1.1.tar.gz", hash = "sha256:d55467bceabdea49d7e3624af7e33d59c37fff53ed3a350e1ac957bed731de7a"},
@@ -2519,6 +2664,7 @@ version = "8.6.3"
description = "Jupyter protocol implementation and client libraries"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jupyter_client-8.6.3-py3-none-any.whl", hash = "sha256:e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f"},
{file = "jupyter_client-8.6.3.tar.gz", hash = "sha256:35b3a0947c4a6e9d589eb97d7d4cd5e90f910ee73101611f01283732bd6d9419"},
@@ -2534,7 +2680,7 @@ traitlets = ">=5.3"
[package.extras]
docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"]
-test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest (<8.2.0)", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"]
+test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko ; sys_platform == \"win32\"", "pre-commit", "pytest (<8.2.0)", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"]
[[package]]
name = "jupyter-console"
@@ -2542,6 +2688,7 @@ version = "6.6.3"
description = "Jupyter terminal console"
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "jupyter_console-6.6.3-py3-none-any.whl", hash = "sha256:309d33409fcc92ffdad25f0bcdf9a4a9daa61b6f341177570fdac03de5352485"},
{file = "jupyter_console-6.6.3.tar.gz", hash = "sha256:566a4bf31c87adbfadf22cdf846e3069b59a71ed5da71d6ba4d8aaad14a53539"},
@@ -2566,6 +2713,7 @@ version = "5.7.2"
description = "Jupyter core package. A base package on which Jupyter projects rely."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jupyter_core-5.7.2-py3-none-any.whl", hash = "sha256:4f7315d2f6b4bcf2e3e7cb6e46772eba760ae459cd1f59d29eb57b0a01bd7409"},
{file = "jupyter_core-5.7.2.tar.gz", hash = "sha256:aa5f8d32bbf6b431ac830496da7392035d6f61b4f54872f15c4bd2a9c3f536d9"},
@@ -2586,6 +2734,7 @@ version = "0.10.0"
description = "Jupyter Event System library"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jupyter_events-0.10.0-py3-none-any.whl", hash = "sha256:4b72130875e59d57716d327ea70d3ebc3af1944d3717e5a498b8a06c6c159960"},
{file = "jupyter_events-0.10.0.tar.gz", hash = "sha256:670b8229d3cc882ec782144ed22e0d29e1c2d639263f92ca8383e66682845e22"},
@@ -2611,6 +2760,7 @@ version = "2.2.5"
description = "Multi-Language Server WebSocket proxy for Jupyter Notebook/Lab server"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jupyter-lsp-2.2.5.tar.gz", hash = "sha256:793147a05ad446f809fd53ef1cd19a9f5256fd0a2d6b7ce943a982cb4f545001"},
{file = "jupyter_lsp-2.2.5-py3-none-any.whl", hash = "sha256:45fbddbd505f3fbfb0b6cb2f1bc5e15e83ab7c79cd6e89416b248cb3c00c11da"},
@@ -2626,6 +2776,7 @@ version = "2.14.2"
description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jupyter_server-2.14.2-py3-none-any.whl", hash = "sha256:47ff506127c2f7851a17bf4713434208fc490955d0e8632e95014a9a9afbeefd"},
{file = "jupyter_server-2.14.2.tar.gz", hash = "sha256:66095021aa9638ced276c248b1d81862e4c50f292d575920bbe960de1c56b12b"},
@@ -2662,6 +2813,7 @@ version = "0.5.3"
description = "A Jupyter Server Extension Providing Terminals."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jupyter_server_terminals-0.5.3-py3-none-any.whl", hash = "sha256:41ee0d7dc0ebf2809c668e0fc726dfaf258fcd3e769568996ca731b6194ae9aa"},
{file = "jupyter_server_terminals-0.5.3.tar.gz", hash = "sha256:5ae0295167220e9ace0edcfdb212afd2b01ee8d179fe6f23c899590e9b8a5269"},
@@ -2681,6 +2833,7 @@ version = "4.3.5"
description = "JupyterLab computational environment"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jupyterlab-4.3.5-py3-none-any.whl", hash = "sha256:571bbdee20e4c5321ab5195bc41cf92a75a5cff886be5e57ce78dfa37a5e9fdb"},
{file = "jupyterlab-4.3.5.tar.gz", hash = "sha256:c779bf72ced007d7d29d5bcef128e7fdda96ea69299e19b04a43635a7d641f9d"},
@@ -2717,6 +2870,7 @@ version = "0.3.0"
description = "Pygments theme using JupyterLab CSS variables"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jupyterlab_pygments-0.3.0-py3-none-any.whl", hash = "sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780"},
{file = "jupyterlab_pygments-0.3.0.tar.gz", hash = "sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d"},
@@ -2728,6 +2882,7 @@ version = "2.27.3"
description = "A set of server components for JupyterLab and JupyterLab like applications."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "jupyterlab_server-2.27.3-py3-none-any.whl", hash = "sha256:e697488f66c3db49df675158a77b3b017520d772c6e1548c7d9bcc5df7944ee4"},
{file = "jupyterlab_server-2.27.3.tar.gz", hash = "sha256:eb36caca59e74471988f0ae25c77945610b887f777255aa21f8065def9e51ed4"},
@@ -2754,6 +2909,7 @@ version = "3.0.13"
description = "Jupyter interactive widgets for JupyterLab"
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "jupyterlab_widgets-3.0.13-py3-none-any.whl", hash = "sha256:e3cda2c233ce144192f1e29914ad522b2f4c40e77214b0cc97377ca3d323db54"},
{file = "jupyterlab_widgets-3.0.13.tar.gz", hash = "sha256:a2966d385328c1942b683a8cd96b89b8dd82c8b8f81dda902bb2bc06d46f5bed"},
@@ -2765,6 +2921,7 @@ version = "0.2.1"
description = "Static image export for web-based visualization libraries with zero dependencies"
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "kaleido-0.2.1-py2.py3-none-macosx_10_11_x86_64.whl", hash = "sha256:ca6f73e7ff00aaebf2843f73f1d3bacde1930ef5041093fe76b83a15785049a7"},
{file = "kaleido-0.2.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:bb9a5d1f710357d5d432ee240ef6658a6d124c3e610935817b4b42da9c787c05"},
@@ -2780,6 +2937,7 @@ version = "25.5.0"
description = "Store and access your passwords safely."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "keyring-25.5.0-py3-none-any.whl", hash = "sha256:e67f8ac32b04be4714b42fe84ce7dad9c40985b9ca827c592cc303e7c26d9741"},
{file = "keyring-25.5.0.tar.gz", hash = "sha256:4c753b3ec91717fe713c4edd522d625889d8973a349b0e582622f49766de58e6"},
@@ -2796,7 +2954,7 @@ pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""}
SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""}
[package.extras]
-check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"]
+check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""]
completion = ["shtab (>=1.1.0)"]
cover = ["pytest-cov"]
doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
@@ -2810,6 +2968,7 @@ version = "1.4.7"
description = "A fast implementation of the Cassowary constraint solver"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "kiwisolver-1.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8a9c83f75223d5e48b0bc9cb1bf2776cf01563e00ade8775ffe13b0b6e1af3a6"},
{file = "kiwisolver-1.4.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58370b1ffbd35407444d57057b57da5d6549d2d854fa30249771775c63b5fe17"},
@@ -2933,6 +3092,8 @@ version = "0.2.17"
description = "Building applications with LLMs through composability"
optional = true
python-versions = "<4.0,>=3.8.1"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "langchain-0.2.17-py3-none-any.whl", hash = "sha256:a97a33e775f8de074370aecab95db148b879c794695d9e443c95457dce5eb525"},
{file = "langchain-0.2.17.tar.gz", hash = "sha256:5a99ce94aae05925851777dba45cbf2c475565d1e91cbe7d82c5e329d514627e"},
@@ -2957,6 +3118,8 @@ version = "0.2.19"
description = "Community contributed LangChain integrations."
optional = true
python-versions = "<4.0,>=3.8.1"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "langchain_community-0.2.19-py3-none-any.whl", hash = "sha256:651d761f2d37d63f89de75d65858f6c7f6ea99c455622e9c13ca041622dad0c5"},
{file = "langchain_community-0.2.19.tar.gz", hash = "sha256:74f8db6992d03668c3d82e0d896845c413d167dad3b8e349fb2a9a57fd2d1396"},
@@ -2980,6 +3143,8 @@ version = "0.2.43"
description = "Building applications with LLMs through composability"
optional = true
python-versions = "<4.0,>=3.8.1"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "langchain_core-0.2.43-py3-none-any.whl", hash = "sha256:619601235113298ebf8252a349754b7c28d3cf7166c7c922da24944b78a9363a"},
{file = "langchain_core-0.2.43.tar.gz", hash = "sha256:42c2ef6adedb911f4254068b6adc9eb4c4075f6c8cb3d83590d3539a815695f5"},
@@ -3000,6 +3165,8 @@ version = "0.1.25"
description = "An integration package connecting OpenAI and LangChain"
optional = true
python-versions = "<4.0,>=3.8.1"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "langchain_openai-0.1.25-py3-none-any.whl", hash = "sha256:f0b34a233d0d9cb8fce6006c903e57085c493c4f0e32862b99063b96eaedb109"},
{file = "langchain_openai-0.1.25.tar.gz", hash = "sha256:eb116f744f820247a72f54313fb7c01524fba0927120d4e899e5e4ab41ad3928"},
@@ -3016,6 +3183,8 @@ version = "0.2.4"
description = "LangChain text splitting utilities"
optional = true
python-versions = "<4.0,>=3.8.1"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "langchain_text_splitters-0.2.4-py3-none-any.whl", hash = "sha256:2702dee5b7cbdd595ccbe43b8d38d01a34aa8583f4d6a5a68ad2305ae3e7b645"},
{file = "langchain_text_splitters-0.2.4.tar.gz", hash = "sha256:f7daa7a3b0aa8309ce248e2e2b6fc8115be01118d336c7f7f7dfacda0e89bf29"},
@@ -3030,6 +3199,7 @@ version = "1.0.9"
description = "Language detection library ported from Google's language-detection."
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "langdetect-1.0.9-py2-none-any.whl", hash = "sha256:7cbc0746252f19e76f77c0b1690aadf01963be835ef0cd4b56dddf2a8f1dfc2a"},
{file = "langdetect-1.0.9.tar.gz", hash = "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0"},
@@ -3044,6 +3214,8 @@ version = "0.1.147"
description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
optional = true
python-versions = "<4.0,>=3.8.1"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "langsmith-0.1.147-py3-none-any.whl", hash = "sha256:7166fc23b965ccf839d64945a78e9f1157757add228b086141eb03a60d699a15"},
{file = "langsmith-0.1.147.tar.gz", hash = "sha256:2e933220318a4e73034657103b3b1a3a6109cc5db3566a7e8e03be8d6d7def7a"},
@@ -3065,6 +3237,7 @@ version = "0.41.1"
description = "lightweight wrapper around basic LLVM functionality"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "llvmlite-0.41.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c1e1029d47ee66d3a0c4d6088641882f75b93db82bd0e6178f7bd744ebce42b9"},
{file = "llvmlite-0.41.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:150d0bc275a8ac664a705135e639178883293cf08c1a38de3bbaa2f693a0a867"},
@@ -3098,6 +3271,7 @@ version = "3.0.0"
description = "Python port of markdown-it. Markdown parsing, done right!"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"},
{file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"},
@@ -3122,6 +3296,7 @@ version = "2.1.5"
description = "Safely add untrusted strings to HTML/XML markup."
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"},
{file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"},
@@ -3191,6 +3366,8 @@ version = "3.22.0"
description = "A lightweight library for converting complex datatypes to and from native Python datatypes."
optional = true
python-versions = ">=3.8"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"},
{file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"},
@@ -3210,6 +3387,7 @@ version = "3.7.5"
description = "Python plotting package"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "matplotlib-3.7.5-cp310-cp310-macosx_10_12_universal2.whl", hash = "sha256:4a87b69cb1cb20943010f63feb0b2901c17a3b435f75349fd9865713bfa63925"},
{file = "matplotlib-3.7.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:d3ce45010fefb028359accebb852ca0c21bd77ec0f281952831d235228f15810"},
@@ -3278,6 +3456,7 @@ version = "0.1.7"
description = "Inline Matplotlib backend for Jupyter"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"},
{file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"},
@@ -3292,17 +3471,36 @@ version = "0.6.1"
description = "McCabe checker, plugin for flake8"
optional = false
python-versions = "*"
+groups = ["dev"]
files = [
{file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
{file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
]
+[[package]]
+name = "mdformat"
+version = "0.7.17"
+description = "CommonMark compliant Markdown formatter"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+ {file = "mdformat-0.7.17-py3-none-any.whl", hash = "sha256:91ffc5e203f5814a6ad17515c77767fd2737fc12ffd8b58b7bb1d8b9aa6effaa"},
+ {file = "mdformat-0.7.17.tar.gz", hash = "sha256:a9dbb1838d43bb1e6f03bd5dca9412c552544a9bc42d6abb5dc32adfe8ae7c0d"},
+]
+
+[package.dependencies]
+importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""}
+markdown-it-py = ">=1.0.0,<4.0.0"
+tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
+
[[package]]
name = "mdurl"
version = "0.1.2"
description = "Markdown URL utilities"
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},
{file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
@@ -3314,6 +3512,7 @@ version = "3.1.2"
description = "A sane and fast Markdown parser with useful plugins and renderers"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "mistune-3.1.2-py3-none-any.whl", hash = "sha256:4b47731332315cdca99e0ded46fc0004001c1299ff773dfb48fbe1fd226de319"},
{file = "mistune-3.1.2.tar.gz", hash = "sha256:733bf018ba007e8b5f2d3a9eb624034f6ee26c4ea769a98ec533ee111d504dff"},
@@ -3328,6 +3527,7 @@ version = "10.5.0"
description = "More routines for operating on iterables, beyond itertools"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "more-itertools-10.5.0.tar.gz", hash = "sha256:5482bfef7849c25dc3c6dd53a6173ae4795da2a41a80faea6700d9f5846c5da6"},
{file = "more_itertools-10.5.0-py3-none-any.whl", hash = "sha256:037b0d3203ce90cca8ab1defbbdac29d5f993fc20131f3664dc8d6acfa872aef"},
@@ -3339,6 +3539,7 @@ version = "1.3.0"
description = "Python library for arbitrary-precision floating-point arithmetic"
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"},
{file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"},
@@ -3347,7 +3548,7 @@ files = [
[package.extras]
develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"]
docs = ["sphinx"]
-gmpy = ["gmpy2 (>=2.1.0a4)"]
+gmpy = ["gmpy2 (>=2.1.0a4) ; platform_python_implementation != \"PyPy\""]
tests = ["pytest (>=4.6)"]
[[package]]
@@ -3356,6 +3557,7 @@ version = "6.1.0"
description = "multidict implementation"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"},
{file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"},
@@ -3454,23 +3656,13 @@ files = [
[package.dependencies]
typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""}
-[[package]]
-name = "multimethod"
-version = "1.10"
-description = "Multiple argument dispatching."
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "multimethod-1.10-py3-none-any.whl", hash = "sha256:afd84da9c3d0445c84f827e4d63ad42d17c6d29b122427c6dee9032ac2d2a0d4"},
- {file = "multimethod-1.10.tar.gz", hash = "sha256:daa45af3fe257f73abb69673fd54ddeaf31df0eb7363ad6e1251b7c9b192d8c5"},
-]
-
[[package]]
name = "multiprocess"
version = "0.70.16"
description = "better multiprocessing and multithreading in Python"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"},
{file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"},
@@ -3495,6 +3687,7 @@ version = "0.0.11"
description = "Non-blocking Python methods using decorators"
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "multitasking-0.0.11-py3-none-any.whl", hash = "sha256:1e5b37a5f8fc1e6cfaafd1a82b6b1cc6d2ed20037d3b89c25a84f499bd7b3dd4"},
{file = "multitasking-0.0.11.tar.gz", hash = "sha256:4d6bc3cc65f9b2dca72fb5a787850a88dae8f620c2b36ae9b55248e51bcd6026"},
@@ -3506,10 +3699,12 @@ version = "1.0.0"
description = "Type system extensions for programs checked with the mypy type checker."
optional = false
python-versions = ">=3.5"
+groups = ["main", "dev"]
files = [
{file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"},
{file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
]
+markers = {main = "extra == \"all\" or extra == \"llm\""}
[[package]]
name = "nbclient"
@@ -3517,6 +3712,7 @@ version = "0.10.1"
description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor."
optional = false
python-versions = ">=3.8.0"
+groups = ["dev"]
files = [
{file = "nbclient-0.10.1-py3-none-any.whl", hash = "sha256:949019b9240d66897e442888cfb618f69ef23dc71c01cb5fced8499c2cfc084d"},
{file = "nbclient-0.10.1.tar.gz", hash = "sha256:3e93e348ab27e712acd46fccd809139e356eb9a31aab641d1a7991a6eb4e6f68"},
@@ -3539,6 +3735,7 @@ version = "7.16.6"
description = "Converting Jupyter Notebooks (.ipynb files) to other formats. Output formats include asciidoc, html, latex, markdown, pdf, py, rst, script. nbconvert can be used both as a Python library (`import nbconvert`) or as a command line tool (invoked as `jupyter nbconvert ...`)."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "nbconvert-7.16.6-py3-none-any.whl", hash = "sha256:1375a7b67e0c2883678c48e506dc320febb57685e5ee67faa51b18a90f3a712b"},
{file = "nbconvert-7.16.6.tar.gz", hash = "sha256:576a7e37c6480da7b8465eefa66c17844243816ce1ccc372633c6b71c3c0f582"},
@@ -3576,6 +3773,7 @@ version = "5.10.4"
description = "The Jupyter Notebook format"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b"},
{file = "nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a"},
@@ -3597,6 +3795,7 @@ version = "1.6.0"
description = "Patch asyncio to allow nested event loops"
optional = false
python-versions = ">=3.5"
+groups = ["main", "dev"]
files = [
{file = "nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c"},
{file = "nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe"},
@@ -3608,6 +3807,7 @@ version = "3.1"
description = "Python package for creating and manipulating graphs and networks"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"},
{file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"},
@@ -3626,6 +3826,7 @@ version = "0.2.21"
description = "Python binding to Ammonia HTML sanitizer Rust crate"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "nh3-0.2.21-cp313-cp313t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:fcff321bd60c6c5c9cb4ddf2554e22772bb41ebd93ad88171bbbb6f271255286"},
{file = "nh3-0.2.21-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31eedcd7d08b0eae28ba47f43fd33a653b4cdb271d64f1aeda47001618348fde"},
@@ -3659,6 +3860,7 @@ version = "3.9.1"
description = "Natural Language Toolkit"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "nltk-3.9.1-py3-none-any.whl", hash = "sha256:4fa26829c5b00715afe3061398a8989dc643b92ce7dd93fb4585a70930d168a1"},
{file = "nltk-3.9.1.tar.gz", hash = "sha256:87d127bd3de4bd89a4f81265e5fa59cb1b199b27440175370f7417d2bc7ae868"},
@@ -3684,6 +3886,7 @@ version = "1.9.1"
description = "Node.js virtual environment builder"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+groups = ["dev"]
files = [
{file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"},
{file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"},
@@ -3695,6 +3898,7 @@ version = "7.3.2"
description = "Jupyter Notebook - A web-based notebook environment for interactive computing"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "notebook-7.3.2-py3-none-any.whl", hash = "sha256:e5f85fc59b69d3618d73cf27544418193ff8e8058d5bf61d315ce4f473556288"},
{file = "notebook-7.3.2.tar.gz", hash = "sha256:705e83a1785f45b383bf3ee13cb76680b92d24f56fb0c7d2136fe1d850cd3ca8"},
@@ -3710,7 +3914,7 @@ tornado = ">=6.2.0"
[package.extras]
dev = ["hatch", "pre-commit"]
docs = ["myst-parser", "nbsphinx", "pydata-sphinx-theme", "sphinx (>=1.3.6)", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"]
-test = ["importlib-resources (>=5.0)", "ipykernel", "jupyter-server[test] (>=2.4.0,<3)", "jupyterlab-server[test] (>=2.27.1,<3)", "nbval", "pytest (>=7.0)", "pytest-console-scripts", "pytest-timeout", "pytest-tornasync", "requests"]
+test = ["importlib-resources (>=5.0) ; python_version < \"3.10\"", "ipykernel", "jupyter-server[test] (>=2.4.0,<3)", "jupyterlab-server[test] (>=2.27.1,<3)", "nbval", "pytest (>=7.0)", "pytest-console-scripts", "pytest-timeout", "pytest-tornasync", "requests"]
[[package]]
name = "notebook-shim"
@@ -3718,6 +3922,7 @@ version = "0.2.4"
description = "A shim layer for notebook traits and config"
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef"},
{file = "notebook_shim-0.2.4.tar.gz", hash = "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb"},
@@ -3735,6 +3940,7 @@ version = "0.58.1"
description = "compiling Python code using LLVM"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "numba-0.58.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:07f2fa7e7144aa6f275f27260e73ce0d808d3c62b30cff8906ad1dec12d87bbe"},
{file = "numba-0.58.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7bf1ddd4f7b9c2306de0384bf3854cac3edd7b4d8dffae2ec1b925e4c436233f"},
@@ -3770,6 +3976,7 @@ version = "1.24.4"
description = "Fundamental package for array computing in Python"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"},
{file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"},
@@ -3807,6 +4014,8 @@ version = "12.4.5.8"
description = "CUBLAS native runtime libraries"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0f8aa1706812e00b9f19dfe0cdb3999b092ccb8ca168c0db5b8ea712456fd9b3"},
{file = "nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:2fc8da60df463fdefa81e323eef2e36489e1c94335b5358bcb38360adf75ac9b"},
@@ -3819,6 +4028,8 @@ version = "12.4.127"
description = "CUDA profiling tools runtime libs."
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:79279b35cf6f91da114182a5ce1864997fd52294a87a16179ce275773799458a"},
{file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9dec60f5ac126f7bb551c055072b69d85392b13311fcc1bcda2202d172df30fb"},
@@ -3831,6 +4042,8 @@ version = "12.4.127"
description = "NVRTC native runtime libraries"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0eedf14185e04b76aa05b1fea04133e59f465b6f960c0cbf4e37c3cb6b0ea198"},
{file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a178759ebb095827bd30ef56598ec182b85547f1508941a3d560eb7ea1fbf338"},
@@ -3843,6 +4056,8 @@ version = "12.4.127"
description = "CUDA Runtime native Libraries"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:961fe0e2e716a2a1d967aab7caee97512f71767f852f67432d572e36cb3a11f3"},
{file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:64403288fa2136ee8e467cdc9c9427e0434110899d07c779f25b5c068934faa5"},
@@ -3855,6 +4070,8 @@ version = "9.1.0.70"
description = "cuDNN runtime libraries"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f"},
{file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-win_amd64.whl", hash = "sha256:6278562929433d68365a07a4a1546c237ba2849852c0d4b2262a486e805b977a"},
@@ -3869,6 +4086,8 @@ version = "11.2.1.3"
description = "CUFFT native runtime libraries"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5dad8008fc7f92f5ddfa2101430917ce2ffacd86824914c82e28990ad7f00399"},
{file = "nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9"},
@@ -3884,6 +4103,8 @@ version = "10.3.5.147"
description = "CURAND native runtime libraries"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1f173f09e3e3c76ab084aba0de819c49e56614feae5c12f69883f4ae9bb5fad9"},
{file = "nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a88f583d4e0bb643c49743469964103aa59f7f708d862c3ddb0fc07f851e3b8b"},
@@ -3896,6 +4117,8 @@ version = "11.6.1.9"
description = "CUDA solver native runtime libraries"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d338f155f174f90724bbde3758b7ac375a70ce8e706d70b018dd3375545fc84e"},
{file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260"},
@@ -3913,6 +4136,8 @@ version = "12.3.1.170"
description = "CUSPARSE native runtime libraries"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_aarch64.whl", hash = "sha256:9d32f62896231ebe0480efd8a7f702e143c98cfaa0e8a76df3386c1ba2b54df3"},
{file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1"},
@@ -3928,6 +4153,8 @@ version = "2.21.5"
description = "NVIDIA Collective Communication Library (NCCL) Runtime"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine != \"aarch64\""
files = [
{file = "nvidia_nccl_cu12-2.21.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:8579076d30a8c24988834445f8d633c697d42397e92ffc3f63fa26766d25e0a0"},
]
@@ -3938,7 +4165,10 @@ version = "12.4.127"
description = "Nvidia JIT LTO Library"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
+ {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4abe7fef64914ccfa909bc2ba39739670ecc9e820c83ccc7a6ed414122599b83"},
{file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57"},
{file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:fd9020c501d27d135f983c6d3e244b197a7ccad769e34df53a42e276b0e25fa1"},
]
@@ -3949,6 +4179,8 @@ version = "12.4.127"
description = "NVIDIA Tools Extension"
optional = false
python-versions = ">=3"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7959ad635db13edf4fc65c06a6e9f9e55fc2f92596db928d169c0bb031e88ef3"},
{file = "nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a"},
@@ -3957,13 +4189,14 @@ files = [
[[package]]
name = "openai"
-version = "1.65.4"
+version = "1.66.2"
description = "The official Python library for the openai API"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
- {file = "openai-1.65.4-py3-none-any.whl", hash = "sha256:15566d46574b94eae3d18efc2f9a4ebd1366d1d44bfc1bdafeea7a5cf8271bcb"},
- {file = "openai-1.65.4.tar.gz", hash = "sha256:0b08c58625d556f5c6654701af1023689c173eb0989ce8f73c7fd0eb22203c76"},
+ {file = "openai-1.66.2-py3-none-any.whl", hash = "sha256:75194057ee6bb8b732526387b6041327a05656d976fc21c064e21c8ac6b07999"},
+ {file = "openai-1.66.2.tar.gz", hash = "sha256:9b3a843c25f81ee09b6469d483d9fba779d5c6ea41861180772f043481b0598d"},
]
[package.dependencies]
@@ -3986,6 +4219,8 @@ version = "3.10.15"
description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy"
optional = true
python-versions = ">=3.8"
+groups = ["main"]
+markers = "(extra == \"all\" or extra == \"llm\") and platform_python_implementation != \"PyPy\""
files = [
{file = "orjson-3.10.15-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:552c883d03ad185f720d0c09583ebde257e41b9521b74ff40e08b7dec4559c04"},
{file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e3e8d438d02e4854f70bfdc03a6bcdb697358dbaa6bcd19cbe24d24ece1f8"},
@@ -4074,6 +4309,7 @@ version = "7.7.0"
description = "A decorator to automatically detect mismatch when overriding a method."
optional = false
python-versions = ">=3.6"
+groups = ["dev"]
files = [
{file = "overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"},
{file = "overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a"},
@@ -4085,6 +4321,7 @@ version = "24.2"
description = "Core utilities for Python packages"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"},
{file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"},
@@ -4096,6 +4333,7 @@ version = "2.0.3"
description = "Powerful data structures for data analysis, time series, and statistics"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"},
{file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"},
@@ -4127,8 +4365,8 @@ files = [
[package.dependencies]
numpy = [
{version = ">=1.20.3", markers = "python_version < \"3.10\""},
+ {version = ">=1.21.0", markers = "python_version >= \"3.10\""},
{version = ">=1.23.2", markers = "python_version >= \"3.11\""},
- {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""},
]
python-dateutil = ">=2.8.2"
pytz = ">=2020.1"
@@ -4163,6 +4401,7 @@ version = "1.5.1"
description = "Utilities for writing pandoc filters in python"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+groups = ["dev"]
files = [
{file = "pandocfilters-1.5.1-py2.py3-none-any.whl", hash = "sha256:93be382804a9cdb0a7267585f157e5d1731bbe5545a85b268d6f5fe6232de2bc"},
{file = "pandocfilters-1.5.1.tar.gz", hash = "sha256:002b4a555ee4ebc03f8b66307e287fa492e4a77b4ea14d3f934328297bb4939e"},
@@ -4174,6 +4413,7 @@ version = "2.6.0"
description = "Parameterize and run Jupyter and nteract Notebooks"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "papermill-2.6.0-py3-none-any.whl", hash = "sha256:0f09da6ef709f3f14dde77cb1af052d05b14019189869affff374c9e612f2dd5"},
{file = "papermill-2.6.0.tar.gz", hash = "sha256:9fe2a91912fd578f391b4cc8d6d105e73124dcd0cde2a43c3c4a1c77ac88ea24"},
@@ -4208,6 +4448,7 @@ version = "0.8.4"
description = "A Python Parser"
optional = false
python-versions = ">=3.6"
+groups = ["main", "dev"]
files = [
{file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"},
{file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"},
@@ -4223,6 +4464,7 @@ version = "0.12.1"
description = "Utility library for gitignore style pattern matching of file paths."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"},
{file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"},
@@ -4234,6 +4476,7 @@ version = "1.0.1"
description = "A Python package for describing statistical models and for building design matrices."
optional = false
python-versions = ">=3.6"
+groups = ["main"]
files = [
{file = "patsy-1.0.1-py2.py3-none-any.whl", hash = "sha256:751fb38f9e97e62312e921a1954b81e1bb2bcda4f5eeabaf94db251ee791509c"},
{file = "patsy-1.0.1.tar.gz", hash = "sha256:e786a9391eec818c054e359b737bbce692f051aee4c661f4141cc88fb459c0c4"},
@@ -4251,6 +4494,7 @@ version = "14.7.0"
description = "API Documentation for Python Projects"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "pdoc-14.7.0-py3-none-any.whl", hash = "sha256:72377a907efc6b2c5b3c56b717ef34f11d93621dced3b663f3aede0b844c0ad2"},
{file = "pdoc-14.7.0.tar.gz", hash = "sha256:2d28af9c0acc39180744ad0543e4bbc3223ecba0d1302db315ec521c51f71f93"},
@@ -4271,6 +4515,7 @@ version = "3.17.9"
description = "a little orm"
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "peewee-3.17.9.tar.gz", hash = "sha256:fe15cd001758e324c8e3ca8c8ed900e7397c2907291789e1efc383e66b9bc7a8"},
]
@@ -4281,6 +4526,8 @@ version = "4.9.0"
description = "Pexpect allows easy control of interactive console applications."
optional = false
python-versions = "*"
+groups = ["main", "dev"]
+markers = "sys_platform != \"win32\""
files = [
{file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"},
{file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"},
@@ -4289,52 +4536,13 @@ files = [
[package.dependencies]
ptyprocess = ">=0.5"
-[[package]]
-name = "phik"
-version = "0.12.4"
-description = "Phi_K correlation analyzer library"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "phik-0.12.4-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:778d00e33762c1e85681f65ef011933faabdc80ab53262f221cccf75eea535d5"},
- {file = "phik-0.12.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d92cc961ee60b317896589bab087901440b2bc749dbd5e266bc3dfe25dbff19a"},
- {file = "phik-0.12.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f48d0dd94323401ed069bbaa673a879f3f002e5ef6fabda19eb3d0a5f8e3947f"},
- {file = "phik-0.12.4-cp310-cp310-win_amd64.whl", hash = "sha256:ea5030640fda8380d7db9ea28fbde37a1565c0b1699bcb7152d6772a6ad278af"},
- {file = "phik-0.12.4-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:2b2f518310c6f3144a5e3d1bc3489c8be17ebe4da6b8520f4e01fa3e544b0fed"},
- {file = "phik-0.12.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f7a6614184eac1b55100c4a7c9899f370ae97599b41b2982f59f7e1da9511cd"},
- {file = "phik-0.12.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ea158b31d51e34153241bd3cac24c9a9a463af575c063abb8ca8d30352b4b12"},
- {file = "phik-0.12.4-cp311-cp311-win_amd64.whl", hash = "sha256:f315699c695e5646b29911b577d584ae76d0fcc1dee539634e512518fcd4108d"},
- {file = "phik-0.12.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:951b06ed32fa0fe6ee73f98407e4d435f90a1750ecb0f250df46eb75741a33bf"},
- {file = "phik-0.12.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b6ba2fa65c4b2a3c36aded0f47333c3069c0520bb426c3f937656a58a5041957"},
- {file = "phik-0.12.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3868a8f9277ab338eacb634bb06dd83278344dc19154f77e06c9cb8712959404"},
- {file = "phik-0.12.4-cp312-cp312-win_amd64.whl", hash = "sha256:247ea90b2d067bb360e798e5645dbcea7753b3bf78436287d92247285c4aa58a"},
- {file = "phik-0.12.4-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:6b38483f02c8a2d471dd14ebc367f83cd619a3672033f1ce52382815cdb9382d"},
- {file = "phik-0.12.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0df90db67dadae940973ffd0692c2e9a207da46b8764e200cb7e6f2552d43154"},
- {file = "phik-0.12.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85c329bd206bfdca689f72f1bb270707f19d5533882b3cde560ce0cbf4b27551"},
- {file = "phik-0.12.4-cp38-cp38-win_amd64.whl", hash = "sha256:eb43bd2b3b6b068b4d2f85a303cfdc256294637f3a598234058cfdbdc75d8538"},
- {file = "phik-0.12.4-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:c2c7482e8ca1e9f688eacd69baccf838fc535b9d3c13523b2d3b53b4aff04c5d"},
- {file = "phik-0.12.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7eb9c0a22d01007a4c51d48489c4f3ebe738461e092061c90da7c1ccf8d51e60"},
- {file = "phik-0.12.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1dd26c71de023852aa452897e41a55176d6d87c268323d0814514cd32a9fadc1"},
- {file = "phik-0.12.4-cp39-cp39-win_amd64.whl", hash = "sha256:c15e987d90d34990fee0ef157fb00c9c69befdf520689ac5f320ff0ab74fa399"},
- {file = "phik-0.12.4.tar.gz", hash = "sha256:d4d53274685e56fb08088505b4eec70be07f2f8044e7961ca02b399e42c37025"},
-]
-
-[package.dependencies]
-joblib = ">=0.14.1"
-matplotlib = ">=2.2.3"
-numpy = ">=1.18.0"
-pandas = ">=0.25.1"
-scipy = ">=1.5.2"
-
-[package.extras]
-test = ["pytest (>=4.0.2)", "pytest-pylint (>=0.13.0)"]
-
[[package]]
name = "pickleshare"
version = "0.7.5"
description = "Tiny 'shelve'-like database with concurrency support"
optional = false
python-versions = "*"
+groups = ["main", "dev"]
files = [
{file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"},
{file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"},
@@ -4346,6 +4554,7 @@ version = "10.4.0"
description = "Python Imaging Library (Fork)"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "pillow-10.4.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:4d9667937cfa347525b319ae34375c37b9ee6b525440f3ef48542fcf66f2731e"},
{file = "pillow-10.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:543f3dc61c18dafb755773efc89aae60d06b6596a63914107f75459cf984164d"},
@@ -4434,7 +4643,7 @@ docs = ["furo", "olefile", "sphinx (>=7.3)", "sphinx-copybutton", "sphinx-inline
fpx = ["olefile"]
mic = ["olefile"]
tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"]
-typing = ["typing-extensions"]
+typing = ["typing-extensions ; python_version < \"3.10\""]
xmp = ["defusedxml"]
[[package]]
@@ -4443,6 +4652,7 @@ version = "1.12.1.2"
description = "Query metadata from sdists / bdists / installed packages."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "pkginfo-1.12.1.2-py3-none-any.whl", hash = "sha256:c783ac885519cab2c34927ccfa6bf64b5a704d7c69afaea583dd9b7afe969343"},
{file = "pkginfo-1.12.1.2.tar.gz", hash = "sha256:5cd957824ac36f140260964eba3c6be6442a8359b8c48f4adf90210f33a04b7b"},
@@ -4457,6 +4667,8 @@ version = "1.3.10"
description = "Resolve a name to an object."
optional = false
python-versions = ">=3.6"
+groups = ["dev"]
+markers = "python_version == \"3.8\""
files = [
{file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"},
{file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"},
@@ -4468,6 +4680,7 @@ version = "4.3.6"
description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"},
{file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"},
@@ -4484,6 +4697,7 @@ version = "5.24.1"
description = "An open-source, interactive data visualization library for Python"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "plotly-5.24.1-py3-none-any.whl", hash = "sha256:f67073a1e637eb0dc3e46324d9d51e2fe76e9727c892dde64ddf1e1b51f29089"},
{file = "plotly-5.24.1.tar.gz", hash = "sha256:dbc8ac8339d248a4bcc36e08a5659bacfe1b079390b8953533f4eb22169b4bae"},
@@ -4499,6 +4713,7 @@ version = "0.4.1"
description = "Plotly Express - a high level wrapper for Plotly.py"
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "plotly_express-0.4.1-py2.py3-none-any.whl", hash = "sha256:5f112922b0a6225dc7c010e3b86295a74449e3eac6cac8faa95175e99b7698ce"},
{file = "plotly_express-0.4.1.tar.gz", hash = "sha256:ff73a41ce02fb43d1d8e8fa131ef3e6589857349ca216b941b8f3f862bce0278"},
@@ -4518,6 +4733,7 @@ version = "1.8.2"
description = "Blazingly fast DataFrame library"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "polars-1.8.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:114be1ebfb051b794fb9e1f15999430c79cc0824595e237d3f45632be3e56d73"},
{file = "polars-1.8.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e4fc36cfe48972d4c5be21a7cb119d6378fb7af0bb3eeb61456b66a1f43228e3"},
@@ -4549,7 +4765,7 @@ pyarrow = ["pyarrow (>=7.0.0)"]
pydantic = ["pydantic"]
sqlalchemy = ["polars[pandas]", "sqlalchemy"]
style = ["great-tables (>=0.8.0)"]
-timezone = ["backports-zoneinfo", "tzdata"]
+timezone = ["backports-zoneinfo ; python_version < \"3.9\"", "tzdata ; platform_system == \"Windows\""]
xlsx2csv = ["xlsx2csv (>=0.8.0)"]
xlsxwriter = ["xlsxwriter"]
@@ -4559,6 +4775,7 @@ version = "3.5.0"
description = "A framework for managing and maintaining multi-language pre-commit hooks."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "pre_commit-3.5.0-py2.py3-none-any.whl", hash = "sha256:841dc9aef25daba9a0238cd27984041fa0467b4199fc4852e27950664919f660"},
{file = "pre_commit-3.5.0.tar.gz", hash = "sha256:5804465c675b659b0862f07907f96295d490822a450c4c40e747d0b1c6ebcb32"},
@@ -4577,6 +4794,7 @@ version = "0.21.1"
description = "Python client for the Prometheus monitoring system."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "prometheus_client-0.21.1-py3-none-any.whl", hash = "sha256:594b45c410d6f4f8888940fe80b5cc2521b305a1fafe1c58609ef715a001f301"},
{file = "prometheus_client-0.21.1.tar.gz", hash = "sha256:252505a722ac04b0456be05c05f75f45d760c2911ffc45f2a06bcaed9f3ae3fb"},
@@ -4591,6 +4809,7 @@ version = "3.0.50"
description = "Library for building powerful interactive command lines in Python"
optional = false
python-versions = ">=3.8.0"
+groups = ["main", "dev"]
files = [
{file = "prompt_toolkit-3.0.50-py3-none-any.whl", hash = "sha256:9b6427eb19e479d98acff65196a307c555eb567989e6d88ebbb1b509d9779198"},
{file = "prompt_toolkit-3.0.50.tar.gz", hash = "sha256:544748f3860a2623ca5cd6d2795e7a14f3d0e1c3c9728359013f79877fc89bab"},
@@ -4605,6 +4824,7 @@ version = "0.2.0"
description = "Accelerated property cache"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "propcache-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58"},
{file = "propcache-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b"},
@@ -4712,6 +4932,7 @@ version = "1.6.4"
description = "A decorator for caching properties in classes (forked from cached-property)."
optional = false
python-versions = ">= 3.5"
+groups = ["main"]
files = [
{file = "property-cached-1.6.4.zip", hash = "sha256:3e9c4ef1ed3653909147510481d7df62a3cfb483461a6986a6f1dcd09b2ebb73"},
{file = "property_cached-1.6.4-py2.py3-none-any.whl", hash = "sha256:135fc059ec969c1646424a0db15e7fbe1b5f8c36c0006d0b3c91ba568c11e7d8"},
@@ -4723,6 +4944,7 @@ version = "7.0.0"
description = "Cross-platform lib for process and system monitoring in Python. NOTE: the syntax of this script MUST be kept compatible with Python 2.7."
optional = false
python-versions = ">=3.6"
+groups = ["dev"]
files = [
{file = "psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25"},
{file = "psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da"},
@@ -4746,6 +4968,7 @@ version = "0.11.1"
description = "Fast python callback/event system modeled after Qt Signals"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "psygnal-0.11.1-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:8d9187700fc608abefeb287bf2e0980a26c62471921ffd1a3cd223ccc554181b"},
{file = "psygnal-0.11.1-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:cec87aee468a1fe564094a64bc3c30edc86ce34d7bb37ab69332c7825b873396"},
@@ -4785,10 +5008,12 @@ version = "0.7.0"
description = "Run a subprocess in a pseudo terminal"
optional = false
python-versions = "*"
+groups = ["main", "dev"]
files = [
{file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"},
{file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"},
]
+markers = {main = "sys_platform != \"win32\"", dev = "sys_platform != \"win32\" or os_name != \"nt\""}
[[package]]
name = "pure-eval"
@@ -4796,6 +5021,7 @@ version = "0.2.3"
description = "Safely evaluate AST nodes without side effects"
optional = false
python-versions = "*"
+groups = ["main", "dev"]
files = [
{file = "pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0"},
{file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"},
@@ -4810,6 +5036,7 @@ version = "17.0.0"
description = "Python library for Apache Arrow"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "pyarrow-17.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a5c8b238d47e48812ee577ee20c9a2779e6a5904f1708ae240f53ecbee7c9f07"},
{file = "pyarrow-17.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db023dc4c6cae1015de9e198d41250688383c3f9af8f565370ab2b4cb5f62655"},
@@ -4861,6 +5088,8 @@ version = "4.4.0"
description = "Python interface for c-ares"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
+markers = "sys_platform == \"linux\" or sys_platform == \"darwin\""
files = [
{file = "pycares-4.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:24da119850841d16996713d9c3374ca28a21deee056d609fbbed29065d17e1f6"},
{file = "pycares-4.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8f64cb58729689d4d0e78f0bfb4c25ce2f851d0274c0273ac751795c04b8798a"},
@@ -4927,6 +5156,8 @@ version = "1.2"
description = "MS-COCO Caption Evaluation for Python 3"
optional = true
python-versions = ">=3"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "pycocoevalcap-1.2-py3-none-any.whl", hash = "sha256:083ed7910f1aec000b0a237ef6665f74edf19954204d0b1cbdb8399ed132228d"},
{file = "pycocoevalcap-1.2.tar.gz", hash = "sha256:7857f4d596ca2fa0b1a9a3c2067588a4257556077b7ad614d00b2b7b8f57cdde"},
@@ -4941,6 +5172,8 @@ version = "2.0.7"
description = "Official APIs for the MS-COCO dataset"
optional = true
python-versions = ">=3.5"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "pycocotools-2.0.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a6683a002fcb4500edbcec94bdf48be69f578a9aa5c638db38614df1f45cc935"},
{file = "pycocotools-2.0.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d517ec315e53ef8df9f6b0899ebc4c79bd61fd715383861949bb1c3fca2c6d5"},
@@ -4974,6 +5207,7 @@ version = "2.8.0"
description = "Python style guide checker"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+groups = ["dev"]
files = [
{file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"},
{file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"},
@@ -4985,10 +5219,12 @@ version = "2.22"
description = "C parser in Python"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"},
{file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
]
+markers = {main = "platform_python_implementation != \"CPython\" or sys_platform == \"linux\" or sys_platform == \"darwin\""}
[[package]]
name = "pydantic"
@@ -4996,6 +5232,7 @@ version = "2.10.6"
description = "Data validation using Python type hints"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "pydantic-2.10.6-py3-none-any.whl", hash = "sha256:427d664bf0b8a2b34ff5dd0f5a18df00591adcee7198fbd71981054cef37b584"},
{file = "pydantic-2.10.6.tar.gz", hash = "sha256:ca5daa827cce33de7a42be142548b0096bf05a7e7b365aebfa5f8eeec7128236"},
@@ -5008,7 +5245,7 @@ typing-extensions = ">=4.12.2"
[package.extras]
email = ["email-validator (>=2.0.0)"]
-timezone = ["tzdata"]
+timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""]
[[package]]
name = "pydantic-core"
@@ -5016,6 +5253,7 @@ version = "2.27.2"
description = "Core functionality for Pydantic validation and serialization"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa"},
{file = "pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c"},
@@ -5128,6 +5366,7 @@ version = "8.0.5"
description = "The kitchen sink of Python utility libraries for doing \"stuff\" in a functional way. Based on the Lo-Dash Javascript library."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "pydash-8.0.5-py3-none-any.whl", hash = "sha256:b2625f8981862e19911daa07f80ed47b315ce20d9b5eb57aaf97aaf570c3892f"},
{file = "pydash-8.0.5.tar.gz", hash = "sha256:7cc44ebfe5d362f4f5f06c74c8684143c5ac481376b059ff02570705523f9e2e"},
@@ -5145,6 +5384,7 @@ version = "2.4.0"
description = "passive checker of Python programs"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+groups = ["dev"]
files = [
{file = "pyflakes-2.4.0-py2.py3-none-any.whl", hash = "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"},
{file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"},
@@ -5156,6 +5396,7 @@ version = "2.19.1"
description = "Pygments is a syntax highlighting package written in Python."
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"},
{file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"},
@@ -5170,6 +5411,7 @@ version = "3.1.4"
description = "pyparsing module - Classes and methods to define and execute parsing grammars"
optional = false
python-versions = ">=3.6.8"
+groups = ["main"]
files = [
{file = "pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c"},
{file = "pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"},
@@ -5184,6 +5426,8 @@ version = "0.3.4"
description = "pysbd (Python Sentence Boundary Disambiguation) is a rule-based sentence boundary detection that works out-of-the-box across many languages."
optional = true
python-versions = ">=3"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "pysbd-0.3.4-py3-none-any.whl", hash = "sha256:cd838939b7b0b185fcf86b0baf6636667dfb6e474743beeff878e9f42e022953"},
]
@@ -5194,6 +5438,7 @@ version = "2.9.0.post0"
description = "Extensions to the standard Python datetime module"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+groups = ["main", "dev"]
files = [
{file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"},
{file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"},
@@ -5208,6 +5453,7 @@ version = "1.0.1"
description = "Read key-value pairs from a .env file and set them as environment variables"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"},
{file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"},
@@ -5218,20 +5464,21 @@ cli = ["click (>=5.0)"]
[[package]]
name = "python-json-logger"
-version = "3.2.1"
+version = "3.3.0"
description = "JSON Log Formatter for the Python Logging Package"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
- {file = "python_json_logger-3.2.1-py3-none-any.whl", hash = "sha256:cdc17047eb5374bd311e748b42f99d71223f3b0e186f4206cc5d52aefe85b090"},
- {file = "python_json_logger-3.2.1.tar.gz", hash = "sha256:8eb0554ea17cb75b05d2848bc14fb02fbdbd9d6972120781b974380bfa162008"},
+ {file = "python_json_logger-3.3.0-py3-none-any.whl", hash = "sha256:dd980fae8cffb24c13caf6e158d3d61c0d6d22342f932cb6e9deedab3d35eec7"},
+ {file = "python_json_logger-3.3.0.tar.gz", hash = "sha256:12b7e74b17775e7d565129296105bbe3910842d9d0eb083fc83a6a617aa8df84"},
]
[package.dependencies]
typing_extensions = {version = "*", markers = "python_version < \"3.10\""}
[package.extras]
-dev = ["backports.zoneinfo", "black", "build", "freezegun", "mdx_truly_sane_lists", "mike", "mkdocs", "mkdocs-awesome-pages-plugin", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-material (>=8.5)", "mkdocstrings[python]", "msgspec", "msgspec-python313-pre", "mypy", "orjson", "pylint", "pytest", "tzdata", "validate-pyproject[all]"]
+dev = ["backports.zoneinfo ; python_version < \"3.9\"", "black", "build", "freezegun", "mdx_truly_sane_lists", "mike", "mkdocs", "mkdocs-awesome-pages-plugin", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-material (>=8.5)", "mkdocstrings[python]", "msgspec ; implementation_name != \"pypy\"", "mypy", "orjson ; implementation_name != \"pypy\"", "pylint", "pytest", "tzdata", "validate-pyproject[all]"]
[[package]]
name = "pytz"
@@ -5239,73 +5486,38 @@ version = "2025.1"
description = "World timezone definitions, modern and historical"
optional = false
python-versions = "*"
+groups = ["main", "dev"]
files = [
{file = "pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57"},
{file = "pytz-2025.1.tar.gz", hash = "sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e"},
]
-
-[[package]]
-name = "pywavelets"
-version = "1.4.1"
-description = "PyWavelets, wavelet transform module"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "PyWavelets-1.4.1-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:d854411eb5ee9cb4bc5d0e66e3634aeb8f594210f6a1bed96dbed57ec70f181c"},
- {file = "PyWavelets-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:231b0e0b1cdc1112f4af3c24eea7bf181c418d37922a67670e9bf6cfa2d544d4"},
- {file = "PyWavelets-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:754fa5085768227c4f4a26c1e0c78bc509a266d9ebd0eb69a278be7e3ece943c"},
- {file = "PyWavelets-1.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da7b9c006171be1f9ddb12cc6e0d3d703b95f7f43cb5e2c6f5f15d3233fcf202"},
- {file = "PyWavelets-1.4.1-cp310-cp310-win32.whl", hash = "sha256:67a0d28a08909f21400cb09ff62ba94c064882ffd9e3a6b27880a111211d59bd"},
- {file = "PyWavelets-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:91d3d393cffa634f0e550d88c0e3f217c96cfb9e32781f2960876f1808d9b45b"},
- {file = "PyWavelets-1.4.1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:64c6bac6204327321db30b775060fbe8e8642316e6bff17f06b9f34936f88875"},
- {file = "PyWavelets-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3f19327f2129fb7977bc59b966b4974dfd72879c093e44a7287500a7032695de"},
- {file = "PyWavelets-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad987748f60418d5f4138db89d82ba0cb49b086e0cbb8fd5c3ed4a814cfb705e"},
- {file = "PyWavelets-1.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:875d4d620eee655346e3589a16a73790cf9f8917abba062234439b594e706784"},
- {file = "PyWavelets-1.4.1-cp311-cp311-win32.whl", hash = "sha256:7231461d7a8eb3bdc7aa2d97d9f67ea5a9f8902522818e7e2ead9c2b3408eeb1"},
- {file = "PyWavelets-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:daf0aa79842b571308d7c31a9c43bc99a30b6328e6aea3f50388cd8f69ba7dbc"},
- {file = "PyWavelets-1.4.1-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:ab7da0a17822cd2f6545626946d3b82d1a8e106afc4b50e3387719ba01c7b966"},
- {file = "PyWavelets-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:578af438a02a86b70f1975b546f68aaaf38f28fb082a61ceb799816049ed18aa"},
- {file = "PyWavelets-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb5ca8d11d3f98e89e65796a2125be98424d22e5ada360a0dbabff659fca0fc"},
- {file = "PyWavelets-1.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:058b46434eac4c04dd89aeef6fa39e4b6496a951d78c500b6641fd5b2cc2f9f4"},
- {file = "PyWavelets-1.4.1-cp38-cp38-win32.whl", hash = "sha256:de7cd61a88a982edfec01ea755b0740e94766e00a1ceceeafef3ed4c85c605cd"},
- {file = "PyWavelets-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:7ab8d9db0fe549ab2ee0bea61f614e658dd2df419d5b75fba47baa761e95f8f2"},
- {file = "PyWavelets-1.4.1-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:23bafd60350b2b868076d976bdd92f950b3944f119b4754b1d7ff22b7acbf6c6"},
- {file = "PyWavelets-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d0e56cd7a53aed3cceca91a04d62feb3a0aca6725b1912d29546c26f6ea90426"},
- {file = "PyWavelets-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:030670a213ee8fefa56f6387b0c8e7d970c7f7ad6850dc048bd7c89364771b9b"},
- {file = "PyWavelets-1.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71ab30f51ee4470741bb55fc6b197b4a2b612232e30f6ac069106f0156342356"},
- {file = "PyWavelets-1.4.1-cp39-cp39-win32.whl", hash = "sha256:47cac4fa25bed76a45bc781a293c26ac63e8eaae9eb8f9be961758d22b58649c"},
- {file = "PyWavelets-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:88aa5449e109d8f5e7f0adef85f7f73b1ab086102865be64421a3a3d02d277f4"},
- {file = "PyWavelets-1.4.1.tar.gz", hash = "sha256:6437af3ddf083118c26d8f97ab43b0724b956c9f958e9ea788659f6a2834ba93"},
-]
-
-[package.dependencies]
-numpy = ">=1.17.3"
+markers = {dev = "python_version == \"3.8\""}
[[package]]
name = "pywin32"
-version = "308"
+version = "309"
description = "Python for Window Extensions"
optional = false
python-versions = "*"
-files = [
- {file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"},
- {file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"},
- {file = "pywin32-308-cp310-cp310-win_arm64.whl", hash = "sha256:a5ab5381813b40f264fa3495b98af850098f814a25a63589a8e9eb12560f450c"},
- {file = "pywin32-308-cp311-cp311-win32.whl", hash = "sha256:5d8c8015b24a7d6855b1550d8e660d8daa09983c80e5daf89a273e5c6fb5095a"},
- {file = "pywin32-308-cp311-cp311-win_amd64.whl", hash = "sha256:575621b90f0dc2695fec346b2d6302faebd4f0f45c05ea29404cefe35d89442b"},
- {file = "pywin32-308-cp311-cp311-win_arm64.whl", hash = "sha256:100a5442b7332070983c4cd03f2e906a5648a5104b8a7f50175f7906efd16bb6"},
- {file = "pywin32-308-cp312-cp312-win32.whl", hash = "sha256:587f3e19696f4bf96fde9d8a57cec74a57021ad5f204c9e627e15c33ff568897"},
- {file = "pywin32-308-cp312-cp312-win_amd64.whl", hash = "sha256:00b3e11ef09ede56c6a43c71f2d31857cf7c54b0ab6e78ac659497abd2834f47"},
- {file = "pywin32-308-cp312-cp312-win_arm64.whl", hash = "sha256:9b4de86c8d909aed15b7011182c8cab38c8850de36e6afb1f0db22b8959e3091"},
- {file = "pywin32-308-cp313-cp313-win32.whl", hash = "sha256:1c44539a37a5b7b21d02ab34e6a4d314e0788f1690d65b48e9b0b89f31abbbed"},
- {file = "pywin32-308-cp313-cp313-win_amd64.whl", hash = "sha256:fd380990e792eaf6827fcb7e187b2b4b1cede0585e3d0c9e84201ec27b9905e4"},
- {file = "pywin32-308-cp313-cp313-win_arm64.whl", hash = "sha256:ef313c46d4c18dfb82a2431e3051ac8f112ccee1a34f29c263c583c568db63cd"},
- {file = "pywin32-308-cp37-cp37m-win32.whl", hash = "sha256:1f696ab352a2ddd63bd07430080dd598e6369152ea13a25ebcdd2f503a38f1ff"},
- {file = "pywin32-308-cp37-cp37m-win_amd64.whl", hash = "sha256:13dcb914ed4347019fbec6697a01a0aec61019c1046c2b905410d197856326a6"},
- {file = "pywin32-308-cp38-cp38-win32.whl", hash = "sha256:5794e764ebcabf4ff08c555b31bd348c9025929371763b2183172ff4708152f0"},
- {file = "pywin32-308-cp38-cp38-win_amd64.whl", hash = "sha256:3b92622e29d651c6b783e368ba7d6722b1634b8e70bd376fd7610fe1992e19de"},
- {file = "pywin32-308-cp39-cp39-win32.whl", hash = "sha256:7873ca4dc60ab3287919881a7d4f88baee4a6e639aa6962de25a98ba6b193341"},
- {file = "pywin32-308-cp39-cp39-win_amd64.whl", hash = "sha256:71b3322d949b4cc20776436a9c9ba0eeedcbc9c650daa536df63f0ff111bb920"},
+groups = ["dev"]
+markers = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\""
+files = [
+ {file = "pywin32-309-cp310-cp310-win32.whl", hash = "sha256:5b78d98550ca093a6fe7ab6d71733fbc886e2af9d4876d935e7f6e1cd6577ac9"},
+ {file = "pywin32-309-cp310-cp310-win_amd64.whl", hash = "sha256:728d08046f3d65b90d4c77f71b6fbb551699e2005cc31bbffd1febd6a08aa698"},
+ {file = "pywin32-309-cp310-cp310-win_arm64.whl", hash = "sha256:c667bcc0a1e6acaca8984eb3e2b6e42696fc035015f99ff8bc6c3db4c09a466a"},
+ {file = "pywin32-309-cp311-cp311-win32.whl", hash = "sha256:d5df6faa32b868baf9ade7c9b25337fa5eced28eb1ab89082c8dae9c48e4cd51"},
+ {file = "pywin32-309-cp311-cp311-win_amd64.whl", hash = "sha256:e7ec2cef6df0926f8a89fd64959eba591a1eeaf0258082065f7bdbe2121228db"},
+ {file = "pywin32-309-cp311-cp311-win_arm64.whl", hash = "sha256:54ee296f6d11db1627216e9b4d4c3231856ed2d9f194c82f26c6cb5650163f4c"},
+ {file = "pywin32-309-cp312-cp312-win32.whl", hash = "sha256:de9acacced5fa82f557298b1fed5fef7bd49beee04190f68e1e4783fbdc19926"},
+ {file = "pywin32-309-cp312-cp312-win_amd64.whl", hash = "sha256:6ff9eebb77ffc3d59812c68db33c0a7817e1337e3537859499bd27586330fc9e"},
+ {file = "pywin32-309-cp312-cp312-win_arm64.whl", hash = "sha256:619f3e0a327b5418d833f44dc87859523635cf339f86071cc65a13c07be3110f"},
+ {file = "pywin32-309-cp313-cp313-win32.whl", hash = "sha256:008bffd4afd6de8ca46c6486085414cc898263a21a63c7f860d54c9d02b45c8d"},
+ {file = "pywin32-309-cp313-cp313-win_amd64.whl", hash = "sha256:bd0724f58492db4cbfbeb1fcd606495205aa119370c0ddc4f70e5771a3ab768d"},
+ {file = "pywin32-309-cp313-cp313-win_arm64.whl", hash = "sha256:8fd9669cfd41863b688a1bc9b1d4d2d76fd4ba2128be50a70b0ea66b8d37953b"},
+ {file = "pywin32-309-cp38-cp38-win32.whl", hash = "sha256:617b837dc5d9dfa7e156dbfa7d3906c009a2881849a80a9ae7519f3dd8c6cb86"},
+ {file = "pywin32-309-cp38-cp38-win_amd64.whl", hash = "sha256:0be3071f555480fbfd86a816a1a773880ee655bf186aa2931860dbb44e8424f8"},
+ {file = "pywin32-309-cp39-cp39-win32.whl", hash = "sha256:72ae9ae3a7a6473223589a1621f9001fe802d59ed227fd6a8503c9af67c1d5f4"},
+ {file = "pywin32-309-cp39-cp39-win_amd64.whl", hash = "sha256:88bc06d6a9feac70783de64089324568ecbc65866e2ab318eab35da3811fd7ef"},
]
[[package]]
@@ -5314,6 +5526,8 @@ version = "0.2.3"
description = "A (partial) reimplementation of pywin32 using ctypes/cffi"
optional = false
python-versions = ">=3.6"
+groups = ["dev"]
+markers = "sys_platform == \"win32\""
files = [
{file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"},
{file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"},
@@ -5325,6 +5539,8 @@ version = "2.0.14"
description = "Pseudo terminal support for Windows from Python."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
+markers = "os_name == \"nt\""
files = [
{file = "pywinpty-2.0.14-cp310-none-win_amd64.whl", hash = "sha256:0b149c2918c7974f575ba79f5a4aad58bd859a52fa9eb1296cc22aa412aa411f"},
{file = "pywinpty-2.0.14-cp311-none-win_amd64.whl", hash = "sha256:cf2a43ac7065b3e0dc8510f8c1f13a75fb8fde805efa3b8cff7599a1ef497bc7"},
@@ -5340,6 +5556,7 @@ version = "6.0.2"
description = "YAML parser and emitter for Python"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
{file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
@@ -5398,120 +5615,105 @@ files = [
[[package]]
name = "pyzmq"
-version = "26.2.1"
+version = "26.3.0"
description = "Python bindings for 0MQ"
optional = false
-python-versions = ">=3.7"
-files = [
- {file = "pyzmq-26.2.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f39d1227e8256d19899d953e6e19ed2ccb689102e6d85e024da5acf410f301eb"},
- {file = "pyzmq-26.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a23948554c692df95daed595fdd3b76b420a4939d7a8a28d6d7dea9711878641"},
- {file = "pyzmq-26.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95f5728b367a042df146cec4340d75359ec6237beebf4a8f5cf74657c65b9257"},
- {file = "pyzmq-26.2.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95f7b01b3f275504011cf4cf21c6b885c8d627ce0867a7e83af1382ebab7b3ff"},
- {file = "pyzmq-26.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80a00370a2ef2159c310e662c7c0f2d030f437f35f478bb8b2f70abd07e26b24"},
- {file = "pyzmq-26.2.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:8531ed35dfd1dd2af95f5d02afd6545e8650eedbf8c3d244a554cf47d8924459"},
- {file = "pyzmq-26.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cdb69710e462a38e6039cf17259d328f86383a06c20482cc154327968712273c"},
- {file = "pyzmq-26.2.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e7eeaef81530d0b74ad0d29eec9997f1c9230c2f27242b8d17e0ee67662c8f6e"},
- {file = "pyzmq-26.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:361edfa350e3be1f987e592e834594422338d7174364763b7d3de5b0995b16f3"},
- {file = "pyzmq-26.2.1-cp310-cp310-win32.whl", hash = "sha256:637536c07d2fb6a354988b2dd1d00d02eb5dd443f4bbee021ba30881af1c28aa"},
- {file = "pyzmq-26.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:45fad32448fd214fbe60030aa92f97e64a7140b624290834cc9b27b3a11f9473"},
- {file = "pyzmq-26.2.1-cp310-cp310-win_arm64.whl", hash = "sha256:d9da0289d8201c8a29fd158aaa0dfe2f2e14a181fd45e2dc1fbf969a62c1d594"},
- {file = "pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:c059883840e634a21c5b31d9b9a0e2b48f991b94d60a811092bc37992715146a"},
- {file = "pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed038a921df836d2f538e509a59cb638df3e70ca0fcd70d0bf389dfcdf784d2a"},
- {file = "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9027a7fcf690f1a3635dc9e55e38a0d6602dbbc0548935d08d46d2e7ec91f454"},
- {file = "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6d75fcb00a1537f8b0c0bb05322bc7e35966148ffc3e0362f0369e44a4a1de99"},
- {file = "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0019cc804ac667fb8c8eaecdb66e6d4a68acf2e155d5c7d6381a5645bd93ae4"},
- {file = "pyzmq-26.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:f19dae58b616ac56b96f2e2290f2d18730a898a171f447f491cc059b073ca1fa"},
- {file = "pyzmq-26.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f5eeeb82feec1fc5cbafa5ee9022e87ffdb3a8c48afa035b356fcd20fc7f533f"},
- {file = "pyzmq-26.2.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:000760e374d6f9d1a3478a42ed0c98604de68c9e94507e5452951e598ebecfba"},
- {file = "pyzmq-26.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:817fcd3344d2a0b28622722b98500ae9c8bfee0f825b8450932ff19c0b15bebd"},
- {file = "pyzmq-26.2.1-cp311-cp311-win32.whl", hash = "sha256:88812b3b257f80444a986b3596e5ea5c4d4ed4276d2b85c153a6fbc5ca457ae7"},
- {file = "pyzmq-26.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:ef29630fde6022471d287c15c0a2484aba188adbfb978702624ba7a54ddfa6c1"},
- {file = "pyzmq-26.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:f32718ee37c07932cc336096dc7403525301fd626349b6eff8470fe0f996d8d7"},
- {file = "pyzmq-26.2.1-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:a6549ecb0041dafa55b5932dcbb6c68293e0bd5980b5b99f5ebb05f9a3b8a8f3"},
- {file = "pyzmq-26.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:0250c94561f388db51fd0213cdccbd0b9ef50fd3c57ce1ac937bf3034d92d72e"},
- {file = "pyzmq-26.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36ee4297d9e4b34b5dc1dd7ab5d5ea2cbba8511517ef44104d2915a917a56dc8"},
- {file = "pyzmq-26.2.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2a9cb17fd83b7a3a3009901aca828feaf20aa2451a8a487b035455a86549c09"},
- {file = "pyzmq-26.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:786dd8a81b969c2081b31b17b326d3a499ddd1856e06d6d79ad41011a25148da"},
- {file = "pyzmq-26.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:2d88ba221a07fc2c5581565f1d0fe8038c15711ae79b80d9462e080a1ac30435"},
- {file = "pyzmq-26.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1c84c1297ff9f1cd2440da4d57237cb74be21fdfe7d01a10810acba04e79371a"},
- {file = "pyzmq-26.2.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:46d4ebafc27081a7f73a0f151d0c38d4291656aa134344ec1f3d0199ebfbb6d4"},
- {file = "pyzmq-26.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:91e2bfb8e9a29f709d51b208dd5f441dc98eb412c8fe75c24ea464734ccdb48e"},
- {file = "pyzmq-26.2.1-cp312-cp312-win32.whl", hash = "sha256:4a98898fdce380c51cc3e38ebc9aa33ae1e078193f4dc641c047f88b8c690c9a"},
- {file = "pyzmq-26.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:a0741edbd0adfe5f30bba6c5223b78c131b5aa4a00a223d631e5ef36e26e6d13"},
- {file = "pyzmq-26.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:e5e33b1491555843ba98d5209439500556ef55b6ab635f3a01148545498355e5"},
- {file = "pyzmq-26.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:099b56ef464bc355b14381f13355542e452619abb4c1e57a534b15a106bf8e23"},
- {file = "pyzmq-26.2.1-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:651726f37fcbce9f8dd2a6dab0f024807929780621890a4dc0c75432636871be"},
- {file = "pyzmq-26.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57dd4d91b38fa4348e237a9388b4423b24ce9c1695bbd4ba5a3eada491e09399"},
- {file = "pyzmq-26.2.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d51a7bfe01a48e1064131f3416a5439872c533d756396be2b39e3977b41430f9"},
- {file = "pyzmq-26.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7154d228502e18f30f150b7ce94f0789d6b689f75261b623f0fdc1eec642aab"},
- {file = "pyzmq-26.2.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:f1f31661a80cc46aba381bed475a9135b213ba23ca7ff6797251af31510920ce"},
- {file = "pyzmq-26.2.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:290c96f479504439b6129a94cefd67a174b68ace8a8e3f551b2239a64cfa131a"},
- {file = "pyzmq-26.2.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:f2c307fbe86e18ab3c885b7e01de942145f539165c3360e2af0f094dd440acd9"},
- {file = "pyzmq-26.2.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:b314268e716487bfb86fcd6f84ebbe3e5bec5fac75fdf42bc7d90fdb33f618ad"},
- {file = "pyzmq-26.2.1-cp313-cp313-win32.whl", hash = "sha256:edb550616f567cd5603b53bb52a5f842c0171b78852e6fc7e392b02c2a1504bb"},
- {file = "pyzmq-26.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:100a826a029c8ef3d77a1d4c97cbd6e867057b5806a7276f2bac1179f893d3bf"},
- {file = "pyzmq-26.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:6991ee6c43e0480deb1b45d0c7c2bac124a6540cba7db4c36345e8e092da47ce"},
- {file = "pyzmq-26.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:25e720dba5b3a3bb2ad0ad5d33440babd1b03438a7a5220511d0c8fa677e102e"},
- {file = "pyzmq-26.2.1-cp313-cp313t-macosx_10_15_universal2.whl", hash = "sha256:9ec6abfb701437142ce9544bd6a236addaf803a32628d2260eb3dbd9a60e2891"},
- {file = "pyzmq-26.2.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e1eb9d2bfdf5b4e21165b553a81b2c3bd5be06eeddcc4e08e9692156d21f1f6"},
- {file = "pyzmq-26.2.1-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:90dc731d8e3e91bcd456aa7407d2eba7ac6f7860e89f3766baabb521f2c1de4a"},
- {file = "pyzmq-26.2.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b6a93d684278ad865fc0b9e89fe33f6ea72d36da0e842143891278ff7fd89c3"},
- {file = "pyzmq-26.2.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:c1bb37849e2294d519117dd99b613c5177934e5c04a5bb05dd573fa42026567e"},
- {file = "pyzmq-26.2.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:632a09c6d8af17b678d84df442e9c3ad8e4949c109e48a72f805b22506c4afa7"},
- {file = "pyzmq-26.2.1-cp313-cp313t-musllinux_1_1_i686.whl", hash = "sha256:fc409c18884eaf9ddde516d53af4f2db64a8bc7d81b1a0c274b8aa4e929958e8"},
- {file = "pyzmq-26.2.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:17f88622b848805d3f6427ce1ad5a2aa3cf61f12a97e684dab2979802024d460"},
- {file = "pyzmq-26.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3ef584f13820d2629326fe20cc04069c21c5557d84c26e277cfa6235e523b10f"},
- {file = "pyzmq-26.2.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:160194d1034902937359c26ccfa4e276abffc94937e73add99d9471e9f555dd6"},
- {file = "pyzmq-26.2.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:574b285150afdbf0a0424dddf7ef9a0d183988eb8d22feacb7160f7515e032cb"},
- {file = "pyzmq-26.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44dba28c34ce527cf687156c81f82bf1e51f047838d5964f6840fd87dfecf9fe"},
- {file = "pyzmq-26.2.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9fbdb90b85c7624c304f72ec7854659a3bd901e1c0ffb2363163779181edeb68"},
- {file = "pyzmq-26.2.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a7ad34a2921e8f76716dc7205c9bf46a53817e22b9eec2e8a3e08ee4f4a72468"},
- {file = "pyzmq-26.2.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:866c12b7c90dd3a86983df7855c6f12f9407c8684db6aa3890fc8027462bda82"},
- {file = "pyzmq-26.2.1-cp37-cp37m-win32.whl", hash = "sha256:eeb37f65350d5c5870517f02f8bbb2ac0fbec7b416c0f4875219fef305a89a45"},
- {file = "pyzmq-26.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4eb3197f694dfb0ee6af29ef14a35f30ae94ff67c02076eef8125e2d98963cd0"},
- {file = "pyzmq-26.2.1-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:36d4e7307db7c847fe37413f333027d31c11d5e6b3bacbb5022661ac635942ba"},
- {file = "pyzmq-26.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1c6ae0e95d0a4b0cfe30f648a18e764352d5415279bdf34424decb33e79935b8"},
- {file = "pyzmq-26.2.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5b4fc44f5360784cc02392f14235049665caaf7c0fe0b04d313e763d3338e463"},
- {file = "pyzmq-26.2.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:51431f6b2750eb9b9d2b2952d3cc9b15d0215e1b8f37b7a3239744d9b487325d"},
- {file = "pyzmq-26.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdbc78ae2065042de48a65f1421b8af6b76a0386bb487b41955818c3c1ce7bed"},
- {file = "pyzmq-26.2.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d14f50d61a89b0925e4d97a0beba6053eb98c426c5815d949a43544f05a0c7ec"},
- {file = "pyzmq-26.2.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:004837cb958988c75d8042f5dac19a881f3d9b3b75b2f574055e22573745f841"},
- {file = "pyzmq-26.2.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0b2007f28ce1b8acebdf4812c1aab997a22e57d6a73b5f318b708ef9bcabbe95"},
- {file = "pyzmq-26.2.1-cp38-cp38-win32.whl", hash = "sha256:269c14904da971cb5f013100d1aaedb27c0a246728c341d5d61ddd03f463f2f3"},
- {file = "pyzmq-26.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:31fff709fef3b991cfe7189d2cfe0c413a1d0e82800a182cfa0c2e3668cd450f"},
- {file = "pyzmq-26.2.1-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:a4bffcadfd40660f26d1b3315a6029fd4f8f5bf31a74160b151f5c577b2dc81b"},
- {file = "pyzmq-26.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e76ad4729c2f1cf74b6eb1bdd05f6aba6175999340bd51e6caee49a435a13bf5"},
- {file = "pyzmq-26.2.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8b0f5bab40a16e708e78a0c6ee2425d27e1a5d8135c7a203b4e977cee37eb4aa"},
- {file = "pyzmq-26.2.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e8e47050412f0ad3a9b2287779758073cbf10e460d9f345002d4779e43bb0136"},
- {file = "pyzmq-26.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f18ce33f422d119b13c1363ed4cce245b342b2c5cbbb76753eabf6aa6f69c7d"},
- {file = "pyzmq-26.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ceb0d78b7ef106708a7e2c2914afe68efffc0051dc6a731b0dbacd8b4aee6d68"},
- {file = "pyzmq-26.2.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ebdd96bd637fd426d60e86a29ec14b8c1ab64b8d972f6a020baf08a30d1cf46"},
- {file = "pyzmq-26.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:03719e424150c6395b9513f53a5faadcc1ce4b92abdf68987f55900462ac7eec"},
- {file = "pyzmq-26.2.1-cp39-cp39-win32.whl", hash = "sha256:ef5479fac31df4b304e96400fc67ff08231873ee3537544aa08c30f9d22fce38"},
- {file = "pyzmq-26.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:f92a002462154c176dac63a8f1f6582ab56eb394ef4914d65a9417f5d9fde218"},
- {file = "pyzmq-26.2.1-cp39-cp39-win_arm64.whl", hash = "sha256:1fd4b3efc6f62199886440d5e27dd3ccbcb98dfddf330e7396f1ff421bfbb3c2"},
- {file = "pyzmq-26.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:380816d298aed32b1a97b4973a4865ef3be402a2e760204509b52b6de79d755d"},
- {file = "pyzmq-26.2.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97cbb368fd0debdbeb6ba5966aa28e9a1ae3396c7386d15569a6ca4be4572b99"},
- {file = "pyzmq-26.2.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abf7b5942c6b0dafcc2823ddd9154f419147e24f8df5b41ca8ea40a6db90615c"},
- {file = "pyzmq-26.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3fe6e28a8856aea808715f7a4fc11f682b9d29cac5d6262dd8fe4f98edc12d53"},
- {file = "pyzmq-26.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bd8fdee945b877aa3bffc6a5a8816deb048dab0544f9df3731ecd0e54d8c84c9"},
- {file = "pyzmq-26.2.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ee7152f32c88e0e1b5b17beb9f0e2b14454235795ef68c0c120b6d3d23d12833"},
- {file = "pyzmq-26.2.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:baa1da72aecf6a490b51fba7a51f1ce298a1e0e86d0daef8265c8f8f9848eb77"},
- {file = "pyzmq-26.2.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49135bb327fca159262d8fd14aa1f4a919fe071b04ed08db4c7c37d2f0647162"},
- {file = "pyzmq-26.2.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8bacc1a10c150d58e8a9ee2b2037a70f8d903107e0f0b6e079bf494f2d09c091"},
- {file = "pyzmq-26.2.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:09dac387ce62d69bec3f06d51610ca1d660e7849eb45f68e38e7f5cf1f49cbcb"},
- {file = "pyzmq-26.2.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:70b3a46ecd9296e725ccafc17d732bfc3cdab850b54bd913f843a0a54dfb2c04"},
- {file = "pyzmq-26.2.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:59660e15c797a3b7a571c39f8e0b62a1f385f98ae277dfe95ca7eaf05b5a0f12"},
- {file = "pyzmq-26.2.1-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0f50db737d688e96ad2a083ad2b453e22865e7e19c7f17d17df416e91ddf67eb"},
- {file = "pyzmq-26.2.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a003200b6cd64e89b5725ff7e284a93ab24fd54bbac8b4fa46b1ed57be693c27"},
- {file = "pyzmq-26.2.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:f9ba5def063243793dec6603ad1392f735255cbc7202a3a484c14f99ec290705"},
- {file = "pyzmq-26.2.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1238c2448c58b9c8d6565579393148414a42488a5f916b3f322742e561f6ae0d"},
- {file = "pyzmq-26.2.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8eddb3784aed95d07065bcf94d07e8c04024fdb6b2386f08c197dfe6b3528fda"},
- {file = "pyzmq-26.2.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0f19c2097fffb1d5b07893d75c9ee693e9cbc809235cf3f2267f0ef6b015f24"},
- {file = "pyzmq-26.2.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0995fd3530f2e89d6b69a2202e340bbada3191014352af978fa795cb7a446331"},
- {file = "pyzmq-26.2.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7c6160fe513654e65665332740f63de29ce0d165e053c0c14a161fa60dd0da01"},
- {file = "pyzmq-26.2.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8ec8e3aea6146b761d6c57fcf8f81fcb19f187afecc19bf1701a48db9617a217"},
- {file = "pyzmq-26.2.1.tar.gz", hash = "sha256:17d72a74e5e9ff3829deb72897a175333d3ef5b5413948cae3cf7ebf0b02ecca"},
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+ {file = "pyzmq-26.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:1586944f4736515af5c6d3a5b150c7e8ca2a2d6e46b23057320584d6f2438f4a"},
+ {file = "pyzmq-26.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa7efc695d1fc9f72d91bf9b6c6fe2d7e1b4193836ec530a98faf7d7a7577a58"},
+ {file = "pyzmq-26.3.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd84441e4021cec6e4dd040550386cd9c9ea1d9418ea1a8002dbb7b576026b2b"},
+ {file = "pyzmq-26.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9176856f36c34a8aa5c0b35ddf52a5d5cd8abeece57c2cd904cfddae3fd9acd3"},
+ {file = "pyzmq-26.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:49334faa749d55b77f084389a80654bf2e68ab5191c0235066f0140c1b670d64"},
+ {file = "pyzmq-26.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fd30fc80fe96efb06bea21667c5793bbd65c0dc793187feb39b8f96990680b00"},
+ {file = "pyzmq-26.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b2eddfbbfb473a62c3a251bb737a6d58d91907f6e1d95791431ebe556f47d916"},
+ {file = "pyzmq-26.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:70b3acb9ad729a53d4e751dace35404a024f188aad406013454216aba5485b4e"},
+ {file = "pyzmq-26.3.0-cp310-cp310-win32.whl", hash = "sha256:c1bd75d692cd7c6d862a98013bfdf06702783b75cffbf5dae06d718fecefe8f2"},
+ {file = "pyzmq-26.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:d7165bcda0dbf203e5ad04d79955d223d84b2263df4db92f525ba370b03a12ab"},
+ {file = "pyzmq-26.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:e34a63f71d2ecffb3c643909ad2d488251afeb5ef3635602b3448e609611a7ed"},
+ {file = "pyzmq-26.3.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:2833602d9d42c94b9d0d2a44d2b382d3d3a4485be018ba19dddc401a464c617a"},
+ {file = "pyzmq-26.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8270d104ec7caa0bdac246d31d48d94472033ceab5ba142881704350b28159c"},
+ {file = "pyzmq-26.3.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c208a977843d18d3bd185f323e4eaa912eb4869cb230947dc6edd8a27a4e558a"},
+ {file = "pyzmq-26.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eddc2be28a379c218e0d92e4a432805dcb0ca5870156a90b54c03cd9799f9f8a"},
+ {file = "pyzmq-26.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c0b519fa2159c42272f8a244354a0e110d65175647e5185b04008ec00df9f079"},
+ {file = "pyzmq-26.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1595533de3a80bf8363372c20bafa963ec4bf9f2b8f539b1d9a5017f430b84c9"},
+ {file = "pyzmq-26.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bbef99eb8d18ba9a40f00e8836b8040cdcf0f2fa649684cf7a66339599919d21"},
+ {file = "pyzmq-26.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:979486d444ca3c469cd1c7f6a619ce48ff08b3b595d451937db543754bfacb65"},
+ {file = "pyzmq-26.3.0-cp311-cp311-win32.whl", hash = "sha256:4b127cfe10b4c56e4285b69fd4b38ea1d368099ea4273d8fb349163fce3cd598"},
+ {file = "pyzmq-26.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:cf736cc1298ef15280d9fcf7a25c09b05af016656856dc6fe5626fd8912658dd"},
+ {file = "pyzmq-26.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:2dc46ec09f5d36f606ac8393303149e69d17121beee13c8dac25e2a2078e31c4"},
+ {file = "pyzmq-26.3.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:c80653332c6136da7f4d4e143975e74ac0fa14f851f716d90583bc19e8945cea"},
+ {file = "pyzmq-26.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e317ee1d4528a03506cb1c282cd9db73660a35b3564096de37de7350e7d87a7"},
+ {file = "pyzmq-26.3.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:943a22ebb3daacb45f76a9bcca9a7b74e7d94608c0c0505da30af900b998ca8d"},
+ {file = "pyzmq-26.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3fc9e71490d989144981ea21ef4fdfaa7b6aa84aff9632d91c736441ce2f6b00"},
+ {file = "pyzmq-26.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e281a8071a06888575a4eb523c4deeefdcd2f5fe4a2d47e02ac8bf3a5b49f695"},
+ {file = "pyzmq-26.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:be77efd735bb1064605be8dec6e721141c1421ef0b115ef54e493a64e50e9a52"},
+ {file = "pyzmq-26.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7a4ac2ffa34f1212dd586af90f4ba894e424f0cabb3a49cdcff944925640f6ac"},
+ {file = "pyzmq-26.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ba698c7c252af83b6bba9775035263f0df5f807f0404019916d4b71af8161f66"},
+ {file = "pyzmq-26.3.0-cp312-cp312-win32.whl", hash = "sha256:214038aaa88e801e54c2ef0cfdb2e6df27eb05f67b477380a452b595c5ecfa37"},
+ {file = "pyzmq-26.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:bad7fe0372e505442482ca3ccbc0d6f38dae81b1650f57a0aa6bbee18e7df495"},
+ {file = "pyzmq-26.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:b7b578d604e79e99aa39495becea013fd043fa9f36e4b490efa951f3d847a24d"},
+ {file = "pyzmq-26.3.0-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:fa85953df84beb7b8b73cb3ec3f5d92b62687a09a8e71525c6734e020edf56fd"},
+ {file = "pyzmq-26.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:209d09f0ab6ddbcebe64630d1e6ca940687e736f443c265ae15bc4bfad833597"},
+ {file = "pyzmq-26.3.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d35cc1086f1d4f907df85c6cceb2245cb39a04f69c3f375993363216134d76d4"},
+ {file = "pyzmq-26.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b380e9087078ba91e45fb18cdd0c25275ffaa045cf63c947be0ddae6186bc9d9"},
+ {file = "pyzmq-26.3.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6d64e74143587efe7c9522bb74d1448128fdf9897cc9b6d8b9927490922fd558"},
+ {file = "pyzmq-26.3.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:efba4f53ac7752eea6d8ca38a4ddac579e6e742fba78d1e99c12c95cd2acfc64"},
+ {file = "pyzmq-26.3.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:9b0137a1c40da3b7989839f9b78a44de642cdd1ce20dcef341de174c8d04aa53"},
+ {file = "pyzmq-26.3.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a995404bd3982c089e57b428c74edd5bfc3b0616b3dbcd6a8e270f1ee2110f36"},
+ {file = "pyzmq-26.3.0-cp313-cp313-win32.whl", hash = "sha256:240b1634b9e530ef6a277d95cbca1a6922f44dfddc5f0a3cd6c722a8de867f14"},
+ {file = "pyzmq-26.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:fe67291775ea4c2883764ba467eb389c29c308c56b86c1e19e49c9e1ed0cbeca"},
+ {file = "pyzmq-26.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:73ca9ae9a9011b714cf7650450cd9c8b61a135180b708904f1f0a05004543dce"},
+ {file = "pyzmq-26.3.0-cp313-cp313t-macosx_10_15_universal2.whl", hash = "sha256:fea7efbd7e49af9d7e5ed6c506dfc7de3d1a628790bd3a35fd0e3c904dc7d464"},
+ {file = "pyzmq-26.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4430c7cba23bb0e2ee203eee7851c1654167d956fc6d4b3a87909ccaf3c5825"},
+ {file = "pyzmq-26.3.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:016d89bee8c7d566fad75516b4e53ec7c81018c062d4c51cd061badf9539be52"},
+ {file = "pyzmq-26.3.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04bfe59852d76d56736bfd10ac1d49d421ab8ed11030b4a0332900691507f557"},
+ {file = "pyzmq-26.3.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:1fe05bd0d633a0f672bb28cb8b4743358d196792e1caf04973b7898a0d70b046"},
+ {file = "pyzmq-26.3.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:2aa1a9f236d5b835fb8642f27de95f9edcfd276c4bc1b6ffc84f27c6fb2e2981"},
+ {file = "pyzmq-26.3.0-cp313-cp313t-musllinux_1_1_i686.whl", hash = "sha256:21399b31753bf321043ea60c360ed5052cc7be20739785b1dff1820f819e35b3"},
+ {file = "pyzmq-26.3.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:d015efcd96aca8882057e7e6f06224f79eecd22cad193d3e6a0a91ec67590d1f"},
+ {file = "pyzmq-26.3.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:18183cc3851b995fdc7e5f03d03b8a4e1b12b0f79dff1ec1da75069af6357a05"},
+ {file = "pyzmq-26.3.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:da87e977f92d930a3683e10ba2b38bcc59adfc25896827e0b9d78b208b7757a6"},
+ {file = "pyzmq-26.3.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cf6db401f4957afbf372a4730c6d5b2a234393af723983cbf4bcd13d54c71e1a"},
+ {file = "pyzmq-26.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03caa2ffd64252122139d50ec92987f89616b9b92c9ba72920b40e92709d5e26"},
+ {file = "pyzmq-26.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:fbf206e5329e20937fa19bd41cf3af06d5967f8f7e86b59d783b26b40ced755c"},
+ {file = "pyzmq-26.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6fb539a6382a048308b409d8c66d79bf636eda1b24f70c78f2a1fd16e92b037b"},
+ {file = "pyzmq-26.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7897b8c8bbbb2bd8cad887bffcb07aede71ef1e45383bd4d6ac049bf0af312a4"},
+ {file = "pyzmq-26.3.0-cp38-cp38-win32.whl", hash = "sha256:91dead2daca698ae52ce70ee2adbb94ddd9b5f96877565fd40aa4efd18ecc6a3"},
+ {file = "pyzmq-26.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:8c088e009a6d6b9f563336adb906e3a8d3fd64db129acc8d8fd0e9fe22b2dac8"},
+ {file = "pyzmq-26.3.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:2eaed0d911fb3280981d5495978152fab6afd9fe217fd16f411523665089cef1"},
+ {file = "pyzmq-26.3.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7998b60ef1c105846fb3bfca494769fde3bba6160902e7cd27a8df8257890ee9"},
+ {file = "pyzmq-26.3.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:96c0006a8d1d00e46cb44c8e8d7316d4a232f3d8f2ed43179d4578dbcb0829b6"},
+ {file = "pyzmq-26.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e17cc198dc50a25a0f245e6b1e56f692df2acec3ccae82d1f60c34bfb72bbec"},
+ {file = "pyzmq-26.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:92a30840f4f2a31f7049d0a7de5fc69dd03b19bd5d8e7fed8d0bde49ce49b589"},
+ {file = "pyzmq-26.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f52eba83272a26b444f4b8fc79f2e2c83f91d706d693836c9f7ccb16e6713c31"},
+ {file = "pyzmq-26.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:952085a09ff32115794629ba47f8940896d7842afdef1283332109d38222479d"},
+ {file = "pyzmq-26.3.0-cp39-cp39-win32.whl", hash = "sha256:0240289e33e3fbae44a5db73e54e955399179332a6b1d47c764a4983ec1524c3"},
+ {file = "pyzmq-26.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:b2db7c82f08b8ce44c0b9d1153ce63907491972a7581e8b6adea71817f119df8"},
+ {file = "pyzmq-26.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:2d3459b6311463c96abcb97808ee0a1abb0d932833edb6aa81c30d622fd4a12d"},
+ {file = "pyzmq-26.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ad03f4252d9041b0635c37528dfa3f44b39f46024ae28c8567f7423676ee409b"},
+ {file = "pyzmq-26.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f3dfb68cf7bf4cfdf34283a75848e077c5defa4907506327282afe92780084d"},
+ {file = "pyzmq-26.3.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:356ec0e39c5a9cda872b65aca1fd8a5d296ffdadf8e2442b70ff32e73ef597b1"},
+ {file = "pyzmq-26.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:749d671b0eec8e738bbf0b361168369d8c682b94fcd458c20741dc4d69ef5278"},
+ {file = "pyzmq-26.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f950f17ae608e0786298340163cac25a4c5543ef25362dd5ddb6dcb10b547be9"},
+ {file = "pyzmq-26.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b4fc9903a73c25be9d5fe45c87faababcf3879445efa16140146b08fccfac017"},
+ {file = "pyzmq-26.3.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c15b69af22030960ac63567e98ad8221cddf5d720d9cf03d85021dfd452324ef"},
+ {file = "pyzmq-26.3.0-pp311-pypy311_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2cf9ab0dff4dbaa2e893eb608373c97eb908e53b7d9793ad00ccbd082c0ee12f"},
+ {file = "pyzmq-26.3.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ec332675f6a138db57aad93ae6387953763f85419bdbd18e914cb279ee1c451"},
+ {file = "pyzmq-26.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:eb96568a22fe070590942cd4780950e2172e00fb033a8b76e47692583b1bd97c"},
+ {file = "pyzmq-26.3.0-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:009a38241c76184cb004c869e82a99f0aee32eda412c1eb44df5820324a01d25"},
+ {file = "pyzmq-26.3.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4c22a12713707467abedc6d75529dd365180c4c2a1511268972c6e1d472bd63e"},
+ {file = "pyzmq-26.3.0-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1614fcd116275d24f2346ffca4047a741c546ad9d561cbf7813f11226ca4ed2c"},
+ {file = "pyzmq-26.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e2cafe7e9c7fed690e8ecf65af119f9c482923b5075a78f6f7629c63e1b4b1d"},
+ {file = "pyzmq-26.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:14e0b81753424bd374075df6cc30b87f2c99e5f022501d97eff66544ca578941"},
+ {file = "pyzmq-26.3.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:21c6ddb98557a77cfe3366af0c5600fb222a1b2de5f90d9cd052b324e0c295e8"},
+ {file = "pyzmq-26.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fc81d5d60c9d40e692de14b8d884d43cf67562402b931681f0ccb3ce6b19875"},
+ {file = "pyzmq-26.3.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52b064fafef772d0f5dbf52d4c39f092be7bc62d9a602fe6e82082e001326de3"},
+ {file = "pyzmq-26.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b72206eb041f780451c61e1e89dbc3705f3d66aaaa14ee320d4f55864b13358a"},
+ {file = "pyzmq-26.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ab78dc21c7b1e13053086bcf0b4246440b43b5409904b73bfd1156654ece8a1"},
+ {file = "pyzmq-26.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:0b42403ad7d1194dca9574cd3c56691c345f4601fa2d0a33434f35142baec7ac"},
+ {file = "pyzmq-26.3.0.tar.gz", hash = "sha256:f1cd68b8236faab78138a8fc703f7ca0ad431b17a3fcac696358600d4e6243b3"},
]
[package.dependencies]
@@ -5523,6 +5725,8 @@ version = "0.2.7"
description = ""
optional = true
python-versions = "*"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "ragas-0.2.7-py3-none-any.whl", hash = "sha256:1a06fa50bcf80e23dcccd36c41d0b601f1caa93155260de8c0879f0a8231e099"},
{file = "ragas-0.2.7.tar.gz", hash = "sha256:26137158db551ff32b90b6b225675c2f902ba12cb833a4e7adbef0bfa5c8353a"},
@@ -5552,6 +5756,7 @@ version = "43.0"
description = "readme_renderer is a library for rendering readme descriptions for Warehouse"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "readme_renderer-43.0-py3-none-any.whl", hash = "sha256:19db308d86ecd60e5affa3b2a98f017af384678c63c88e5d4556a380e674f3f9"},
{file = "readme_renderer-43.0.tar.gz", hash = "sha256:1818dd28140813509eeed8d62687f7cd4f7bad90d4db586001c5dc09d4fde311"},
@@ -5571,6 +5776,7 @@ version = "0.35.1"
description = "JSON Referencing + Python"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"},
{file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"},
@@ -5586,6 +5792,7 @@ version = "2024.11.6"
description = "Alternative regular expression module, to replace re."
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "regex-2024.11.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91"},
{file = "regex-2024.11.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:658f90550f38270639e83ce492f27d2c8d2cd63805c65a13a14d36ca126753f0"},
@@ -5689,6 +5896,7 @@ version = "2.32.3"
description = "Python HTTP for Humans."
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"},
{file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
@@ -5710,10 +5918,12 @@ version = "1.0.0"
description = "A utility belt for advanced users of python-requests"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+groups = ["main", "dev"]
files = [
{file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"},
{file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"},
]
+markers = {main = "extra == \"all\" or extra == \"llm\""}
[package.dependencies]
requests = ">=2.0.1,<3.0.0"
@@ -5724,6 +5934,7 @@ version = "0.1.4"
description = "A pure python RFC3339 validator"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+groups = ["dev"]
files = [
{file = "rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"},
{file = "rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b"},
@@ -5738,6 +5949,7 @@ version = "2.0.0"
description = "Validating URI References per RFC 3986"
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd"},
{file = "rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"},
@@ -5752,6 +5964,7 @@ version = "0.1.1"
description = "Pure python rfc3986 validator"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+groups = ["dev"]
files = [
{file = "rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9"},
{file = "rfc3986_validator-0.1.1.tar.gz", hash = "sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055"},
@@ -5763,6 +5976,7 @@ version = "13.9.4"
description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
optional = false
python-versions = ">=3.8.0"
+groups = ["dev"]
files = [
{file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"},
{file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"},
@@ -5782,6 +5996,7 @@ version = "1.0.1"
description = "Full Python ROUGE Score Implementation (not a wrapper)"
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "rouge-1.0.1-py3-none-any.whl", hash = "sha256:28d118536e8c774dc47d1d15ec266479b4dd0914c4672ce117d4002789bdc644"},
{file = "rouge-1.0.1.tar.gz", hash = "sha256:12b48346ca47d6bcf3c45061f315452b9ccec0620ee895ec85b7efc3d54aae34"},
@@ -5796,6 +6011,7 @@ version = "0.20.1"
description = "Python bindings to Rust's persistent data structures (rpds)"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "rpds_py-0.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a649dfd735fff086e8a9d0503a9f0c7d01b7912a333c7ae77e1515c08c146dad"},
{file = "rpds_py-0.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f16bc1334853e91ddaaa1217045dd7be166170beec337576818461268a3de67f"},
@@ -5908,6 +6124,7 @@ version = "0.5.3"
description = ""
optional = false
python-versions = ">=3.7"
+groups = ["main"]
files = [
{file = "safetensors-0.5.3-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd20eb133db8ed15b40110b7c00c6df51655a2998132193de2f75f72d99c7073"},
{file = "safetensors-0.5.3-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:21d01c14ff6c415c485616b8b0bf961c46b3b343ca59110d38d744e577f9cce7"},
@@ -5945,6 +6162,7 @@ version = "1.3.2"
description = "A set of python modules for machine learning and data mining"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "scikit-learn-1.3.2.tar.gz", hash = "sha256:a2f54c76accc15a34bfb9066e6c7a56c1e7235dda5762b990792330b52ccfb05"},
{file = "scikit_learn-1.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e326c0eb5cf4d6ba40f93776a20e9a7a69524c4db0757e7ce24ba222471ee8a1"},
@@ -5992,6 +6210,7 @@ version = "1.10.1"
description = "Fundamental algorithms for scientific computing in Python"
optional = false
python-versions = "<3.12,>=3.8"
+groups = ["main"]
files = [
{file = "scipy-1.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7354fd7527a4b0377ce55f286805b34e8c54b91be865bac273f527e1b839019"},
{file = "scipy-1.10.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4b3f429188c66603a1a5c549fb414e4d3bdc2a24792e061ffbd607d3d75fd84e"},
@@ -6030,6 +6249,7 @@ version = "0.1.9.7"
description = "Credit Risk Scorecard"
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "scorecardpy-0.1.9.7.tar.gz", hash = "sha256:a81c7e6f3bf5f10a87b61af73b25f1fc8bc5acbadf5d9e38c3addb02df128d03"},
]
@@ -6048,6 +6268,7 @@ version = "0.13.2"
description = "Statistical data visualization"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987"},
{file = "seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7"},
@@ -6069,6 +6290,8 @@ version = "3.3.3"
description = "Python bindings to FreeDesktop.org Secret Service API"
optional = false
python-versions = ">=3.6"
+groups = ["dev"]
+markers = "sys_platform == \"linux\""
files = [
{file = "SecretStorage-3.3.3-py3-none-any.whl", hash = "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99"},
{file = "SecretStorage-3.3.3.tar.gz", hash = "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77"},
@@ -6084,15 +6307,16 @@ version = "1.8.3"
description = "Send file to trash natively under Mac OS X, Windows and Linux"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
+groups = ["dev"]
files = [
{file = "Send2Trash-1.8.3-py3-none-any.whl", hash = "sha256:0c31227e0bd08961c7665474a3d1ef7193929fedda4233843689baa056be46c9"},
{file = "Send2Trash-1.8.3.tar.gz", hash = "sha256:b18e7a3966d99871aefeb00cfbcfdced55ce4871194810fc71f4aa484b953abf"},
]
[package.extras]
-nativelib = ["pyobjc-framework-Cocoa", "pywin32"]
-objc = ["pyobjc-framework-Cocoa"]
-win32 = ["pywin32"]
+nativelib = ["pyobjc-framework-Cocoa ; sys_platform == \"darwin\"", "pywin32 ; sys_platform == \"win32\""]
+objc = ["pyobjc-framework-Cocoa ; sys_platform == \"darwin\""]
+win32 = ["pywin32 ; sys_platform == \"win32\""]
[[package]]
name = "sentencepiece"
@@ -6100,6 +6324,8 @@ version = "0.2.0"
description = "SentencePiece python wrapper"
optional = true
python-versions = "*"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"huggingface\" or extra == \"llm\""
files = [
{file = "sentencepiece-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:188779e1298a1c8b8253c7d3ad729cb0a9891e5cef5e5d07ce4592c54869e227"},
{file = "sentencepiece-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bed9cf85b296fa2b76fc2547b9cbb691a523864cebaee86304c43a7b4cb1b452"},
@@ -6162,6 +6388,7 @@ version = "1.45.1"
description = "Python client for Sentry (https://sentry.io)"
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "sentry_sdk-1.45.1-py2.py3-none-any.whl", hash = "sha256:608887855ccfe39032bfd03936e3a1c4f4fc99b3a4ac49ced54a4220de61c9c1"},
{file = "sentry_sdk-1.45.1.tar.gz", hash = "sha256:a16c997c0f4e3df63c0fc5e4207ccb1ab37900433e0f72fef88315d317829a26"},
@@ -6205,23 +6432,24 @@ tornado = ["tornado (>=5)"]
[[package]]
name = "setuptools"
-version = "75.3.0"
+version = "75.3.2"
description = "Easily download, build, install, upgrade, and uninstall Python packages"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
- {file = "setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd"},
- {file = "setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686"},
+ {file = "setuptools-75.3.2-py3-none-any.whl", hash = "sha256:90ab613b6583fc02d5369cbca13ea26ea0e182d1df2d943ee9cbe81d4c61add9"},
+ {file = "setuptools-75.3.2.tar.gz", hash = "sha256:3c1383e1038b68556a382c1e8ded8887cd20141b0eb5708a6c8d277de49364f5"},
]
[package.extras]
-check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"]
-core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"]
+check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.5.2) ; sys_platform != \"cygwin\""]
+core = ["importlib-metadata (>=6) ; python_version < \"3.10\"", "importlib-resources (>=5.10.2) ; python_version < \"3.9\"", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"]
cover = ["pytest-cov"]
doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
enabler = ["pytest-enabler (>=2.2)"]
-test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"]
-type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.12.*)", "pytest-mypy"]
+test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "ruff (<=0.7.1)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"]
+type = ["importlib-metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.12.*)", "pytest-mypy"]
[[package]]
name = "shap"
@@ -6229,6 +6457,7 @@ version = "0.44.1"
description = "A unified approach to explain the output of any machine learning model."
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "shap-0.44.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:93a94961a355249855f13f1ed564466afa1c5fae84f868dd56e50e936f4f9b57"},
{file = "shap-0.44.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2919f2b255e31363182afb1627b374eb6c4724c90b0318719cbe90a316682f5"},
@@ -6271,7 +6500,7 @@ tqdm = ">=4.27.0"
docs = ["ipython", "matplotlib", "myst-parser (==2.0.0)", "nbsphinx (==0.9.3)", "numpydoc", "requests", "sphinx (==7.2.6)", "sphinx-github-changelog (==1.2.1)", "sphinx-rtd-theme (==2.0.0)"]
others = ["lime"]
plots = ["ipython", "matplotlib"]
-test = ["catboost", "gpboost", "lightgbm", "ngboost", "opencv-python", "protobuf (==3.20.3)", "pyod", "pyspark", "pytest", "pytest-cov", "pytest-mpl", "sentencepiece", "tensorflow", "torch", "torchvision", "transformers", "xgboost"]
+test = ["catboost", "gpboost", "lightgbm", "ngboost ; python_version < \"3.11\"", "opencv-python", "protobuf (==3.20.3)", "pyod", "pyspark", "pytest", "pytest-cov", "pytest-mpl", "sentencepiece", "tensorflow", "torch", "torchvision", "transformers", "xgboost"]
test-core = ["pytest", "pytest-cov", "pytest-mpl"]
test-notebooks = ["datasets", "jupyter", "keras", "nbconvert", "nbformat", "nlp", "transformers"]
@@ -6281,6 +6510,7 @@ version = "1.17.0"
description = "Python 2 and 3 compatibility utilities"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+groups = ["main", "dev"]
files = [
{file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"},
{file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"},
@@ -6292,6 +6522,7 @@ version = "0.0.7"
description = "A small package for big slicing."
optional = false
python-versions = ">=3.6"
+groups = ["main"]
files = [
{file = "slicer-0.0.7-py3-none-any.whl", hash = "sha256:0b94faa5251c0f23782c03f7b7eedda91d80144059645f452c4bc80fab875976"},
{file = "slicer-0.0.7.tar.gz", hash = "sha256:f5d5f7b45f98d155b9c0ba6554fa9770c6b26d5793a3e77a1030fb56910ebeec"},
@@ -6303,6 +6534,7 @@ version = "1.3.1"
description = "Sniff out which async library your code is running under"
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"},
{file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"},
@@ -6314,6 +6546,7 @@ version = "2.2.0"
description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms."
optional = false
python-versions = "*"
+groups = ["dev"]
files = [
{file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"},
{file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"},
@@ -6325,6 +6558,7 @@ version = "2.6"
description = "A modern CSS selector implementation for Beautiful Soup."
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"},
{file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"},
@@ -6336,6 +6570,7 @@ version = "6.2.1"
description = "Python documentation generator"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "Sphinx-6.2.1.tar.gz", hash = "sha256:6d56a34697bb749ffa0152feafc4b19836c755d90a7c59b72bc7dfd371b9cc6b"},
{file = "sphinx-6.2.1-py3-none-any.whl", hash = "sha256:97787ff1fa3256a3eef9eda523a63dbf299f7b47e053cfcf684a1c2a8380c912"},
@@ -6371,6 +6606,7 @@ version = "0.5.5"
description = "sphinx builder that outputs markdown files"
optional = false
python-versions = "*"
+groups = ["dev"]
files = [
{file = "sphinx-markdown-builder-0.5.5.tar.gz", hash = "sha256:6ead53c08d8835329e32418dcdbac4db710a1c4e5e8db687d23b9e88882d9d16"},
{file = "sphinx_markdown_builder-0.5.5-py2.py3-none-any.whl", hash = "sha256:3c8909579dfa83ce5a8fb48e2d01dc257fc0676931170cb92cd528f9fceee76f"},
@@ -6389,6 +6625,7 @@ version = "1.3.0"
description = "Read the Docs theme for Sphinx"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
+groups = ["dev"]
files = [
{file = "sphinx_rtd_theme-1.3.0-py2.py3-none-any.whl", hash = "sha256:46ddef89cc2416a81ecfbeaceab1881948c014b1b6e4450b815311a89fb977b0"},
{file = "sphinx_rtd_theme-1.3.0.tar.gz", hash = "sha256:590b030c7abb9cf038ec053b95e5380b5c70d61591eb0b552063fbe7c41f0931"},
@@ -6408,6 +6645,7 @@ version = "1.0.4"
description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"},
{file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"},
@@ -6423,6 +6661,7 @@ version = "1.0.2"
description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document."
optional = false
python-versions = ">=3.5"
+groups = ["dev"]
files = [
{file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"},
{file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"},
@@ -6438,6 +6677,7 @@ version = "2.0.1"
description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"},
{file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"},
@@ -6453,6 +6693,7 @@ version = "4.1"
description = "Extension to include jQuery on newer Sphinx releases"
optional = false
python-versions = ">=2.7"
+groups = ["dev"]
files = [
{file = "sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a"},
{file = "sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae"},
@@ -6467,6 +6708,7 @@ version = "1.0.1"
description = "A sphinx extension which renders display math in HTML via JavaScript"
optional = false
python-versions = ">=3.5"
+groups = ["dev"]
files = [
{file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"},
{file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"},
@@ -6481,6 +6723,7 @@ version = "1.0.3"
description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document."
optional = false
python-versions = ">=3.5"
+groups = ["dev"]
files = [
{file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"},
{file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"},
@@ -6496,6 +6739,7 @@ version = "1.1.5"
description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)."
optional = false
python-versions = ">=3.5"
+groups = ["dev"]
files = [
{file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"},
{file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"},
@@ -6507,68 +6751,70 @@ test = ["pytest"]
[[package]]
name = "sqlalchemy"
-version = "2.0.38"
+version = "2.0.39"
description = "Database Abstraction Library"
optional = true
python-versions = ">=3.7"
-files = [
- {file = "SQLAlchemy-2.0.38-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5e1d9e429028ce04f187a9f522818386c8b076723cdbe9345708384f49ebcec6"},
- {file = "SQLAlchemy-2.0.38-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b87a90f14c68c925817423b0424381f0e16d80fc9a1a1046ef202ab25b19a444"},
- {file = "SQLAlchemy-2.0.38-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:402c2316d95ed90d3d3c25ad0390afa52f4d2c56b348f212aa9c8d072a40eee5"},
- {file = "SQLAlchemy-2.0.38-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6493bc0eacdbb2c0f0d260d8988e943fee06089cd239bd7f3d0c45d1657a70e2"},
- {file = "SQLAlchemy-2.0.38-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0561832b04c6071bac3aad45b0d3bb6d2c4f46a8409f0a7a9c9fa6673b41bc03"},
- {file = "SQLAlchemy-2.0.38-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:49aa2cdd1e88adb1617c672a09bf4ebf2f05c9448c6dbeba096a3aeeb9d4d443"},
- {file = "SQLAlchemy-2.0.38-cp310-cp310-win32.whl", hash = "sha256:64aa8934200e222f72fcfd82ee71c0130a9c07d5725af6fe6e919017d095b297"},
- {file = "SQLAlchemy-2.0.38-cp310-cp310-win_amd64.whl", hash = "sha256:c57b8e0841f3fce7b703530ed70c7c36269c6d180ea2e02e36b34cb7288c50c7"},
- {file = "SQLAlchemy-2.0.38-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bf89e0e4a30714b357f5d46b6f20e0099d38b30d45fa68ea48589faf5f12f62d"},
- {file = "SQLAlchemy-2.0.38-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8455aa60da49cb112df62b4721bd8ad3654a3a02b9452c783e651637a1f21fa2"},
- {file = "SQLAlchemy-2.0.38-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f53c0d6a859b2db58332e0e6a921582a02c1677cc93d4cbb36fdf49709b327b2"},
- {file = "SQLAlchemy-2.0.38-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3c4817dff8cef5697f5afe5fec6bc1783994d55a68391be24cb7d80d2dbc3a6"},
- {file = "SQLAlchemy-2.0.38-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9cea5b756173bb86e2235f2f871b406a9b9d722417ae31e5391ccaef5348f2c"},
- {file = "SQLAlchemy-2.0.38-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:40e9cdbd18c1f84631312b64993f7d755d85a3930252f6276a77432a2b25a2f3"},
- {file = "SQLAlchemy-2.0.38-cp311-cp311-win32.whl", hash = "sha256:cb39ed598aaf102251483f3e4675c5dd6b289c8142210ef76ba24aae0a8f8aba"},
- {file = "SQLAlchemy-2.0.38-cp311-cp311-win_amd64.whl", hash = "sha256:f9d57f1b3061b3e21476b0ad5f0397b112b94ace21d1f439f2db472e568178ae"},
- {file = "SQLAlchemy-2.0.38-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12d5b06a1f3aeccf295a5843c86835033797fea292c60e72b07bcb5d820e6dd3"},
- {file = "SQLAlchemy-2.0.38-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e036549ad14f2b414c725349cce0772ea34a7ab008e9cd67f9084e4f371d1f32"},
- {file = "SQLAlchemy-2.0.38-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee3bee874cb1fadee2ff2b79fc9fc808aa638670f28b2145074538d4a6a5028e"},
- {file = "SQLAlchemy-2.0.38-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e185ea07a99ce8b8edfc788c586c538c4b1351007e614ceb708fd01b095ef33e"},
- {file = "SQLAlchemy-2.0.38-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b79ee64d01d05a5476d5cceb3c27b5535e6bb84ee0f872ba60d9a8cd4d0e6579"},
- {file = "SQLAlchemy-2.0.38-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:afd776cf1ebfc7f9aa42a09cf19feadb40a26366802d86c1fba080d8e5e74bdd"},
- {file = "SQLAlchemy-2.0.38-cp312-cp312-win32.whl", hash = "sha256:a5645cd45f56895cfe3ca3459aed9ff2d3f9aaa29ff7edf557fa7a23515a3725"},
- {file = "SQLAlchemy-2.0.38-cp312-cp312-win_amd64.whl", hash = "sha256:1052723e6cd95312f6a6eff9a279fd41bbae67633415373fdac3c430eca3425d"},
- {file = "SQLAlchemy-2.0.38-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ecef029b69843b82048c5b347d8e6049356aa24ed644006c9a9d7098c3bd3bfd"},
- {file = "SQLAlchemy-2.0.38-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c8bcad7fc12f0cc5896d8e10fdf703c45bd487294a986903fe032c72201596b"},
- {file = "SQLAlchemy-2.0.38-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a0ef3f98175d77180ffdc623d38e9f1736e8d86b6ba70bff182a7e68bed7727"},
- {file = "SQLAlchemy-2.0.38-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b0ac78898c50e2574e9f938d2e5caa8fe187d7a5b69b65faa1ea4648925b096"},
- {file = "SQLAlchemy-2.0.38-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9eb4fa13c8c7a2404b6a8e3772c17a55b1ba18bc711e25e4d6c0c9f5f541b02a"},
- {file = "SQLAlchemy-2.0.38-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5dba1cdb8f319084f5b00d41207b2079822aa8d6a4667c0f369fce85e34b0c86"},
- {file = "SQLAlchemy-2.0.38-cp313-cp313-win32.whl", hash = "sha256:eae27ad7580529a427cfdd52c87abb2dfb15ce2b7a3e0fc29fbb63e2ed6f8120"},
- {file = "SQLAlchemy-2.0.38-cp313-cp313-win_amd64.whl", hash = "sha256:b335a7c958bc945e10c522c069cd6e5804f4ff20f9a744dd38e748eb602cbbda"},
- {file = "SQLAlchemy-2.0.38-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:40310db77a55512a18827488e592965d3dec6a3f1e3d8af3f8243134029daca3"},
- {file = "SQLAlchemy-2.0.38-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d3043375dd5bbcb2282894cbb12e6c559654c67b5fffb462fda815a55bf93f7"},
- {file = "SQLAlchemy-2.0.38-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70065dfabf023b155a9c2a18f573e47e6ca709b9e8619b2e04c54d5bcf193178"},
- {file = "SQLAlchemy-2.0.38-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:c058b84c3b24812c859300f3b5abf300daa34df20d4d4f42e9652a4d1c48c8a4"},
- {file = "SQLAlchemy-2.0.38-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0398361acebb42975deb747a824b5188817d32b5c8f8aba767d51ad0cc7bb08d"},
- {file = "SQLAlchemy-2.0.38-cp37-cp37m-win32.whl", hash = "sha256:a2bc4e49e8329f3283d99840c136ff2cd1a29e49b5624a46a290f04dff48e079"},
- {file = "SQLAlchemy-2.0.38-cp37-cp37m-win_amd64.whl", hash = "sha256:9cd136184dd5f58892f24001cdce986f5d7e96059d004118d5410671579834a4"},
- {file = "SQLAlchemy-2.0.38-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:665255e7aae5f38237b3a6eae49d2358d83a59f39ac21036413fab5d1e810578"},
- {file = "SQLAlchemy-2.0.38-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:92f99f2623ff16bd4aaf786ccde759c1f676d39c7bf2855eb0b540e1ac4530c8"},
- {file = "SQLAlchemy-2.0.38-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa498d1392216fae47eaf10c593e06c34476ced9549657fca713d0d1ba5f7248"},
- {file = "SQLAlchemy-2.0.38-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9afbc3909d0274d6ac8ec891e30210563b2c8bdd52ebbda14146354e7a69373"},
- {file = "SQLAlchemy-2.0.38-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:57dd41ba32430cbcc812041d4de8d2ca4651aeefad2626921ae2a23deb8cd6ff"},
- {file = "SQLAlchemy-2.0.38-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3e35d5565b35b66905b79ca4ae85840a8d40d31e0b3e2990f2e7692071b179ca"},
- {file = "SQLAlchemy-2.0.38-cp38-cp38-win32.whl", hash = "sha256:f0d3de936b192980209d7b5149e3c98977c3810d401482d05fb6d668d53c1c63"},
- {file = "SQLAlchemy-2.0.38-cp38-cp38-win_amd64.whl", hash = "sha256:3868acb639c136d98107c9096303d2d8e5da2880f7706f9f8c06a7f961961149"},
- {file = "SQLAlchemy-2.0.38-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:07258341402a718f166618470cde0c34e4cec85a39767dce4e24f61ba5e667ea"},
- {file = "SQLAlchemy-2.0.38-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a826f21848632add58bef4f755a33d45105d25656a0c849f2dc2df1c71f6f50"},
- {file = "SQLAlchemy-2.0.38-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:386b7d136919bb66ced64d2228b92d66140de5fefb3c7df6bd79069a269a7b06"},
- {file = "SQLAlchemy-2.0.38-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f2951dc4b4f990a4b394d6b382accb33141d4d3bd3ef4e2b27287135d6bdd68"},
- {file = "SQLAlchemy-2.0.38-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8bf312ed8ac096d674c6aa9131b249093c1b37c35db6a967daa4c84746bc1bc9"},
- {file = "SQLAlchemy-2.0.38-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6db316d6e340f862ec059dc12e395d71f39746a20503b124edc255973977b728"},
- {file = "SQLAlchemy-2.0.38-cp39-cp39-win32.whl", hash = "sha256:c09a6ea87658695e527104cf857c70f79f14e9484605e205217aae0ec27b45fc"},
- {file = "SQLAlchemy-2.0.38-cp39-cp39-win_amd64.whl", hash = "sha256:12f5c9ed53334c3ce719155424dc5407aaa4f6cadeb09c5b627e06abb93933a1"},
- {file = "SQLAlchemy-2.0.38-py3-none-any.whl", hash = "sha256:63178c675d4c80def39f1febd625a6333f44c0ba269edd8a468b156394b27753"},
- {file = "sqlalchemy-2.0.38.tar.gz", hash = "sha256:e5a4d82bdb4bf1ac1285a68eab02d253ab73355d9f0fe725a97e1e0fa689decb"},
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
+files = [
+ {file = "SQLAlchemy-2.0.39-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:66a40003bc244e4ad86b72abb9965d304726d05a939e8c09ce844d27af9e6d37"},
+ {file = "SQLAlchemy-2.0.39-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67de057fbcb04a066171bd9ee6bcb58738d89378ee3cabff0bffbf343ae1c787"},
+ {file = "SQLAlchemy-2.0.39-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:533e0f66c32093a987a30df3ad6ed21170db9d581d0b38e71396c49718fbb1ca"},
+ {file = "SQLAlchemy-2.0.39-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:7399d45b62d755e9ebba94eb89437f80512c08edde8c63716552a3aade61eb42"},
+ {file = "SQLAlchemy-2.0.39-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:788b6ff6728072b313802be13e88113c33696a9a1f2f6d634a97c20f7ef5ccce"},
+ {file = "SQLAlchemy-2.0.39-cp37-cp37m-win32.whl", hash = "sha256:01da15490c9df352fbc29859d3c7ba9cd1377791faeeb47c100832004c99472c"},
+ {file = "SQLAlchemy-2.0.39-cp37-cp37m-win_amd64.whl", hash = "sha256:f2bcb085faffcacf9319b1b1445a7e1cfdc6fb46c03f2dce7bc2d9a4b3c1cdc5"},
+ {file = "SQLAlchemy-2.0.39-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b761a6847f96fdc2d002e29e9e9ac2439c13b919adfd64e8ef49e75f6355c548"},
+ {file = "SQLAlchemy-2.0.39-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0d7e3866eb52d914aea50c9be74184a0feb86f9af8aaaa4daefe52b69378db0b"},
+ {file = "SQLAlchemy-2.0.39-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:995c2bacdddcb640c2ca558e6760383dcdd68830160af92b5c6e6928ffd259b4"},
+ {file = "SQLAlchemy-2.0.39-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:344cd1ec2b3c6bdd5dfde7ba7e3b879e0f8dd44181f16b895940be9b842fd2b6"},
+ {file = "SQLAlchemy-2.0.39-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5dfbc543578058c340360f851ddcecd7a1e26b0d9b5b69259b526da9edfa8875"},
+ {file = "SQLAlchemy-2.0.39-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3395e7ed89c6d264d38bea3bfb22ffe868f906a7985d03546ec7dc30221ea980"},
+ {file = "SQLAlchemy-2.0.39-cp38-cp38-win32.whl", hash = "sha256:bf555f3e25ac3a70c67807b2949bfe15f377a40df84b71ab2c58d8593a1e036e"},
+ {file = "SQLAlchemy-2.0.39-cp38-cp38-win_amd64.whl", hash = "sha256:463ecfb907b256e94bfe7bcb31a6d8c7bc96eca7cbe39803e448a58bb9fcad02"},
+ {file = "sqlalchemy-2.0.39-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6827f8c1b2f13f1420545bd6d5b3f9e0b85fe750388425be53d23c760dcf176b"},
+ {file = "sqlalchemy-2.0.39-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9f119e7736967c0ea03aff91ac7d04555ee038caf89bb855d93bbd04ae85b41"},
+ {file = "sqlalchemy-2.0.39-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4600c7a659d381146e1160235918826c50c80994e07c5b26946a3e7ec6c99249"},
+ {file = "sqlalchemy-2.0.39-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a06e6c8e31c98ddc770734c63903e39f1947c9e3e5e4bef515c5491b7737dde"},
+ {file = "sqlalchemy-2.0.39-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c4c433f78c2908ae352848f56589c02b982d0e741b7905228fad628999799de4"},
+ {file = "sqlalchemy-2.0.39-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7bd5c5ee1448b6408734eaa29c0d820d061ae18cb17232ce37848376dcfa3e92"},
+ {file = "sqlalchemy-2.0.39-cp310-cp310-win32.whl", hash = "sha256:87a1ce1f5e5dc4b6f4e0aac34e7bb535cb23bd4f5d9c799ed1633b65c2bcad8c"},
+ {file = "sqlalchemy-2.0.39-cp310-cp310-win_amd64.whl", hash = "sha256:871f55e478b5a648c08dd24af44345406d0e636ffe021d64c9b57a4a11518304"},
+ {file = "sqlalchemy-2.0.39-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a28f9c238f1e143ff42ab3ba27990dfb964e5d413c0eb001b88794c5c4a528a9"},
+ {file = "sqlalchemy-2.0.39-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:08cf721bbd4391a0e765fe0fe8816e81d9f43cece54fdb5ac465c56efafecb3d"},
+ {file = "sqlalchemy-2.0.39-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a8517b6d4005facdbd7eb4e8cf54797dbca100a7df459fdaff4c5123265c1cd"},
+ {file = "sqlalchemy-2.0.39-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b2de1523d46e7016afc7e42db239bd41f2163316935de7c84d0e19af7e69538"},
+ {file = "sqlalchemy-2.0.39-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:412c6c126369ddae171c13987b38df5122cb92015cba6f9ee1193b867f3f1530"},
+ {file = "sqlalchemy-2.0.39-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6b35e07f1d57b79b86a7de8ecdcefb78485dab9851b9638c2c793c50203b2ae8"},
+ {file = "sqlalchemy-2.0.39-cp311-cp311-win32.whl", hash = "sha256:3eb14ba1a9d07c88669b7faf8f589be67871d6409305e73e036321d89f1d904e"},
+ {file = "sqlalchemy-2.0.39-cp311-cp311-win_amd64.whl", hash = "sha256:78f1b79132a69fe8bd6b5d91ef433c8eb40688ba782b26f8c9f3d2d9ca23626f"},
+ {file = "sqlalchemy-2.0.39-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c457a38351fb6234781d054260c60e531047e4d07beca1889b558ff73dc2014b"},
+ {file = "sqlalchemy-2.0.39-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:018ee97c558b499b58935c5a152aeabf6d36b3d55d91656abeb6d93d663c0c4c"},
+ {file = "sqlalchemy-2.0.39-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5493a8120d6fc185f60e7254fc056a6742f1db68c0f849cfc9ab46163c21df47"},
+ {file = "sqlalchemy-2.0.39-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2cf5b5ddb69142511d5559c427ff00ec8c0919a1e6c09486e9c32636ea2b9dd"},
+ {file = "sqlalchemy-2.0.39-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f03143f8f851dd8de6b0c10784363712058f38209e926723c80654c1b40327a"},
+ {file = "sqlalchemy-2.0.39-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:06205eb98cb3dd52133ca6818bf5542397f1dd1b69f7ea28aa84413897380b06"},
+ {file = "sqlalchemy-2.0.39-cp312-cp312-win32.whl", hash = "sha256:7f5243357e6da9a90c56282f64b50d29cba2ee1f745381174caacc50d501b109"},
+ {file = "sqlalchemy-2.0.39-cp312-cp312-win_amd64.whl", hash = "sha256:2ed107331d188a286611cea9022de0afc437dd2d3c168e368169f27aa0f61338"},
+ {file = "sqlalchemy-2.0.39-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fe193d3ae297c423e0e567e240b4324d6b6c280a048e64c77a3ea6886cc2aa87"},
+ {file = "sqlalchemy-2.0.39-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:79f4f502125a41b1b3b34449e747a6abfd52a709d539ea7769101696bdca6716"},
+ {file = "sqlalchemy-2.0.39-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a10ca7f8a1ea0fd5630f02feb055b0f5cdfcd07bb3715fc1b6f8cb72bf114e4"},
+ {file = "sqlalchemy-2.0.39-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6b0a1c7ed54a5361aaebb910c1fa864bae34273662bb4ff788a527eafd6e14d"},
+ {file = "sqlalchemy-2.0.39-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:52607d0ebea43cf214e2ee84a6a76bc774176f97c5a774ce33277514875a718e"},
+ {file = "sqlalchemy-2.0.39-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c08a972cbac2a14810463aec3a47ff218bb00c1a607e6689b531a7c589c50723"},
+ {file = "sqlalchemy-2.0.39-cp313-cp313-win32.whl", hash = "sha256:23c5aa33c01bd898f879db158537d7e7568b503b15aad60ea0c8da8109adf3e7"},
+ {file = "sqlalchemy-2.0.39-cp313-cp313-win_amd64.whl", hash = "sha256:4dabd775fd66cf17f31f8625fc0e4cfc5765f7982f94dc09b9e5868182cb71c0"},
+ {file = "sqlalchemy-2.0.39-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2600a50d590c22d99c424c394236899ba72f849a02b10e65b4c70149606408b5"},
+ {file = "sqlalchemy-2.0.39-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4eff9c270afd23e2746e921e80182872058a7a592017b2713f33f96cc5f82e32"},
+ {file = "sqlalchemy-2.0.39-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d7332868ce891eda48896131991f7f2be572d65b41a4050957242f8e935d5d7"},
+ {file = "sqlalchemy-2.0.39-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:125a7763b263218a80759ad9ae2f3610aaf2c2fbbd78fff088d584edf81f3782"},
+ {file = "sqlalchemy-2.0.39-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:04545042969833cb92e13b0a3019549d284fd2423f318b6ba10e7aa687690a3c"},
+ {file = "sqlalchemy-2.0.39-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:805cb481474e111ee3687c9047c5f3286e62496f09c0e82e8853338aaaa348f8"},
+ {file = "sqlalchemy-2.0.39-cp39-cp39-win32.whl", hash = "sha256:34d5c49f18778a3665d707e6286545a30339ad545950773d43977e504815fa70"},
+ {file = "sqlalchemy-2.0.39-cp39-cp39-win_amd64.whl", hash = "sha256:35e72518615aa5384ef4fae828e3af1b43102458b74a8c481f69af8abf7e802a"},
+ {file = "sqlalchemy-2.0.39-py3-none-any.whl", hash = "sha256:a1c6b0a5e3e326a466d809b651c63f278b1256146a377a528b6938a279da334f"},
+ {file = "sqlalchemy-2.0.39.tar.gz", hash = "sha256:5d2d1fe548def3267b4c70a8568f108d1fed7cbbeccb9cc166e05af2abc25c22"},
]
[package.dependencies]
@@ -6606,6 +6852,7 @@ version = "0.6.3"
description = "Extract data from python stack frames and tracebacks for informative displays"
optional = false
python-versions = "*"
+groups = ["main", "dev"]
files = [
{file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"},
{file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"},
@@ -6625,6 +6872,7 @@ version = "0.14.1"
description = "Statistical computations and models for Python"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "statsmodels-0.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:43af9c0b07c9d72f275cf14ea54a481a3f20911f0b443181be4769def258fdeb"},
{file = "statsmodels-0.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a16975ab6ad505d837ba9aee11f92a8c5b49c4fa1ff45b60fe23780b19e5705e"},
@@ -6659,7 +6907,7 @@ files = [
[package.dependencies]
numpy = [
- {version = ">=1.18,<2", markers = "python_version != \"3.10\" or platform_system != \"Windows\" or platform_python_implementation == \"PyPy\""},
+ {version = ">=1.18,<2"},
{version = ">=1.22.3,<2", markers = "python_version == \"3.10\" and platform_system == \"Windows\" and platform_python_implementation != \"PyPy\""},
]
packaging = ">=21.3"
@@ -6669,7 +6917,7 @@ scipy = ">=1.4,<1.9.2 || >1.9.2"
[package.extras]
build = ["cython (>=0.29.33)"]
-develop = ["colorama", "cython (>=0.29.33)", "cython (>=0.29.33,<4.0.0)", "flake8", "isort", "joblib", "matplotlib (>=3)", "oldest-supported-numpy (>=2022.4.18)", "pytest (>=7.3.0)", "pytest-cov", "pytest-randomly", "pytest-xdist", "pywinpty", "setuptools-scm[toml] (>=8.0,<9.0)"]
+develop = ["colorama", "cython (>=0.29.33)", "cython (>=0.29.33,<4.0.0)", "flake8", "isort", "joblib", "matplotlib (>=3)", "oldest-supported-numpy (>=2022.4.18)", "pytest (>=7.3.0)", "pytest-cov", "pytest-randomly", "pytest-xdist", "pywinpty ; os_name == \"nt\"", "setuptools-scm[toml] (>=8.0,<9.0)"]
docs = ["ipykernel", "jupyter-client", "matplotlib", "nbconvert", "nbformat", "numpydoc", "pandas-datareader", "sphinx"]
[[package]]
@@ -6678,6 +6926,8 @@ version = "1.12.1"
description = "Computer algebra system (CAS) in Python"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
+markers = "python_version == \"3.8\""
files = [
{file = "sympy-1.12.1-py3-none-any.whl", hash = "sha256:9b2cbc7f1a640289430e13d2a56f02f867a1da0190f2f99d8968c2f74da0e515"},
{file = "sympy-1.12.1.tar.gz", hash = "sha256:2877b03f998cd8c08f07cd0de5b767119cd3ef40d09f41c30d722f6686b0fb88"},
@@ -6692,6 +6942,8 @@ version = "1.13.1"
description = "Computer algebra system (CAS) in Python"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
+markers = "python_version >= \"3.9\""
files = [
{file = "sympy-1.13.1-py3-none-any.whl", hash = "sha256:db36cdc64bf61b9b24578b6f7bab1ecdd2452cf008f34faa33776680c26d66f8"},
{file = "sympy-1.13.1.tar.gz", hash = "sha256:9cebf7e04ff162015ce31c9c6c9144daa34a93bd082f54fd8f12deca4f47515f"},
@@ -6709,6 +6961,7 @@ version = "0.8.10"
description = "Pretty-print tabular data"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+groups = ["main"]
files = [
{file = "tabulate-0.8.10-py3-none-any.whl", hash = "sha256:0ba055423dbaa164b9e456abe7920c5e8ed33fcc16f6d1b2f2d152c8e1e8b4fc"},
{file = "tabulate-0.8.10.tar.gz", hash = "sha256:6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519"},
@@ -6723,6 +6976,7 @@ version = "8.5.0"
description = "Retry code until it succeeds"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687"},
{file = "tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78"},
@@ -6738,6 +6992,7 @@ version = "0.18.1"
description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0"},
{file = "terminado-0.18.1.tar.gz", hash = "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e"},
@@ -6759,6 +7014,7 @@ version = "0.18.0.post0"
description = "Simple, Pythonic text processing. Sentiment analysis, part-of-speech tagging, noun phrase parsing, and more."
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "textblob-0.18.0.post0-py3-none-any.whl", hash = "sha256:dd0c7ec4eb7b9346ec0a3f136a63eba13e0f59890d2a693d3d6aeb8371949dca"},
{file = "textblob-0.18.0.post0.tar.gz", hash = "sha256:8131c52c630bcdf61d04c359f939c98d5b836a01fba224d9e7ae22fc274e0ccb"},
@@ -6778,6 +7034,7 @@ version = "3.5.0"
description = "threadpoolctl"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"},
{file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"},
@@ -6789,6 +7046,7 @@ version = "0.7.0"
description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "tiktoken-0.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485f3cc6aba7c6b6ce388ba634fbba656d9ee27f766216f45146beb4ac18b25f"},
{file = "tiktoken-0.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e54be9a2cd2f6d6ffa3517b064983fb695c9a9d8aa7d574d1ef3c3f931a99225"},
@@ -6841,6 +7099,7 @@ version = "1.2.1"
description = "A tiny CSS parser"
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "tinycss2-1.2.1-py3-none-any.whl", hash = "sha256:2b80a96d41e7c3914b8cda8bc7f705a4d9c49275616e886103dd839dfc847847"},
{file = "tinycss2-1.2.1.tar.gz", hash = "sha256:8cff3a8f066c2ec677c06dbc7b45619804a6938478d9d73c284b29d14ecb0627"},
@@ -6859,6 +7118,7 @@ version = "0.20.3"
description = ""
optional = false
python-versions = ">=3.7"
+groups = ["main"]
files = [
{file = "tokenizers-0.20.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:31ccab28dbb1a9fe539787210b0026e22debeab1662970f61c2d921f7557f7e4"},
{file = "tokenizers-0.20.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6361191f762bda98c773da418cf511cbaa0cb8d0a1196f16f8c0119bde68ff8"},
@@ -6988,6 +7248,8 @@ version = "2.2.1"
description = "A lil' TOML parser"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
+markers = "python_version == \"3.10\" or python_version == \"3.9\" or python_version == \"3.8\""
files = [
{file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"},
{file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"},
@@ -7029,6 +7291,7 @@ version = "2.5.1"
description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration"
optional = false
python-versions = ">=3.8.0"
+groups = ["main"]
files = [
{file = "torch-2.5.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:71328e1bbe39d213b8721678f9dcac30dfc452a46d586f1d514a6aa0a99d4744"},
{file = "torch-2.5.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:34bfa1a852e5714cbfa17f27c49d8ce35e1b7af5608c4bc6e81392c352dbc601"},
@@ -7083,6 +7346,7 @@ version = "6.4.2"
description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"},
{file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"},
@@ -7103,6 +7367,7 @@ version = "4.67.1"
description = "Fast, Extensible Progress Meter"
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"},
{file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"},
@@ -7124,6 +7389,7 @@ version = "5.14.3"
description = "Traitlets Python configuration system"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"},
{file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"},
@@ -7139,6 +7405,7 @@ version = "4.46.3"
description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow"
optional = false
python-versions = ">=3.8.0"
+groups = ["main"]
files = [
{file = "transformers-4.46.3-py3-none-any.whl", hash = "sha256:a12ef6f52841fd190a3e5602145b542d03507222f2c64ebb7ee92e8788093aef"},
{file = "transformers-4.46.3.tar.gz", hash = "sha256:8ee4b3ae943fe33e82afff8e837f4b052058b07ca9be3cb5b729ed31295f72cc"},
@@ -7208,6 +7475,8 @@ version = "3.1.0"
description = "A language and compiler for custom Deep Learning operations"
optional = false
python-versions = "*"
+groups = ["main"]
+markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""
files = [
{file = "triton-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b0dd10a925263abbe9fa37dcde67a5e9b2383fc269fdf59f5657cac38c5d1d8"},
{file = "triton-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f34f6e7885d1bf0eaaf7ba875a5f0ce6f3c13ba98f9503651c1e6dc6757ed5c"},
@@ -7230,6 +7499,7 @@ version = "4.0.2"
description = "Collection of utilities for publishing packages on PyPI"
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "twine-4.0.2-py3-none-any.whl", hash = "sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8"},
{file = "twine-4.0.2.tar.gz", hash = "sha256:9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8"},
@@ -7246,31 +7516,13 @@ rfc3986 = ">=1.4.0"
rich = ">=12.0.0"
urllib3 = ">=1.26.0"
-[[package]]
-name = "typeguard"
-version = "4.4.0"
-description = "Run-time type checker for Python"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "typeguard-4.4.0-py3-none-any.whl", hash = "sha256:8ca34c14043f53b2caae7040549ba431770869bcd6287cfa8239db7ecb882b4a"},
- {file = "typeguard-4.4.0.tar.gz", hash = "sha256:463bd8697a65a4aa576a63767c369b1ecfba8a5ba735edfe3223127b6ecfa28c"},
-]
-
-[package.dependencies]
-importlib-metadata = {version = ">=3.6", markers = "python_version < \"3.10\""}
-typing-extensions = ">=4.10.0"
-
-[package.extras]
-doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.3.0)"]
-test = ["coverage[toml] (>=7)", "mypy (>=1.2.0)", "pytest (>=7)"]
-
[[package]]
name = "types-python-dateutil"
version = "2.9.0.20241206"
description = "Typing stubs for python-dateutil"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "types_python_dateutil-2.9.0.20241206-py3-none-any.whl", hash = "sha256:e248a4bc70a486d3e3ec84d0dc30eec3a5f979d6e7ee4123ae043eedbb987f53"},
{file = "types_python_dateutil-2.9.0.20241206.tar.gz", hash = "sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb"},
@@ -7282,6 +7534,7 @@ version = "4.12.2"
description = "Backported and Experimental Type Hints for Python 3.8+"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
{file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
@@ -7293,6 +7546,8 @@ version = "0.9.0"
description = "Runtime inspection utilities for typing module."
optional = true
python-versions = "*"
+groups = ["main"]
+markers = "extra == \"all\" or extra == \"llm\""
files = [
{file = "typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f"},
{file = "typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78"},
@@ -7308,6 +7563,7 @@ version = "2025.1"
description = "Provider of IANA time zone data"
optional = false
python-versions = ">=2"
+groups = ["main"]
files = [
{file = "tzdata-2025.1-py2.py3-none-any.whl", hash = "sha256:7e127113816800496f027041c570f50bcd464a020098a3b6b199517772303639"},
{file = "tzdata-2025.1.tar.gz", hash = "sha256:24894909e88cdb28bd1636c6887801df64cb485bd593f2fd83ef29075a81d694"},
@@ -7319,6 +7575,7 @@ version = "0.5"
description = "Modifies strings to all use the same (single/double) quote where possible."
optional = false
python-versions = "*"
+groups = ["dev"]
files = [
{file = "unify-0.5.tar.gz", hash = "sha256:8ddce812b2457212b7598fe574c9e6eb3ad69710f445391338270c7f8a71723c"},
]
@@ -7332,6 +7589,7 @@ version = "0.1.1"
description = "Transforms tokens into original source code (while preserving whitespace)."
optional = false
python-versions = "*"
+groups = ["dev"]
files = [
{file = "untokenize-0.1.1.tar.gz", hash = "sha256:3865dbbbb8efb4bb5eaa72f1be7f3e0be00ea8b7f125c69cbd1f5fda926f37a2"},
]
@@ -7342,6 +7600,7 @@ version = "1.3.0"
description = "RFC 6570 URI Template Processor"
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "uri-template-1.3.0.tar.gz", hash = "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7"},
{file = "uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363"},
@@ -7356,13 +7615,14 @@ version = "2.2.3"
description = "HTTP library with thread-safe connection pooling, file post, and more."
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"},
{file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"},
]
[package.extras]
-brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
+brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""]
h2 = ["h2 (>=4,<5)"]
socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
zstd = ["zstandard (>=0.18.0)"]
@@ -7373,6 +7633,7 @@ version = "20.29.3"
description = "Virtual Python Environment builder"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "virtualenv-20.29.3-py3-none-any.whl", hash = "sha256:3e3d00f5807e83b234dfb6122bf37cfadf4be216c53a49ac059d02414f819170"},
{file = "virtualenv-20.29.3.tar.gz", hash = "sha256:95e39403fcf3940ac45bc717597dba16110b74506131845d9b687d5e73d947ac"},
@@ -7385,35 +7646,7 @@ platformdirs = ">=3.9.1,<5"
[package.extras]
docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"]
-test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"]
-
-[[package]]
-name = "visions"
-version = "0.7.6"
-description = "Visions"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "visions-0.7.6-py3-none-any.whl", hash = "sha256:72b7f8dbc374e9d6055e938c8c67b0b8da52f3bcb8320f25d86b1a57457e7aa6"},
- {file = "visions-0.7.6.tar.gz", hash = "sha256:00f494a7f78917db2292e11ea832c6e026b64783e688b11da24f4c271ef1631d"},
-]
-
-[package.dependencies]
-attrs = ">=19.3.0"
-imagehash = {version = "*", optional = true, markers = "extra == \"type_image_path\""}
-multimethod = ">=1.4"
-networkx = ">=2.4"
-numpy = ">=1.23.2"
-pandas = ">=2.0.0"
-Pillow = {version = "*", optional = true, markers = "extra == \"type_image_path\""}
-
-[package.extras]
-all = ["Pillow", "attrs (>=19.3.0)", "imagehash", "matplotlib", "multimethod (>=1.4)", "networkx (>=2.4)", "numpy (>=1.23.2)", "pandas (>=2.0.0)", "pydot", "pygraphviz", "shapely"]
-dev = ["IPython", "Sphinx-copybutton", "black (>=20.8b1)", "isort (>=5.0.9)", "mypy (>=0.770)", "nbsphinx", "recommonmark (>=0.6.0)", "setuptools (>=46.1.3)", "sphinx-autodoc-typehints (>=1.10.3)", "sphinx-rtd-theme (>=0.4.3)", "wheel (>=0.34.2)"]
-plotting = ["matplotlib", "pydot", "pygraphviz"]
-test = ["Pillow", "big-o (>=0.10.1)", "black (>=19.10b0)", "check-manifest (>=0.41)", "imagehash", "isort (>=5.0.9)", "matplotlib", "mypy (>=0.800)", "numba", "pandas", "pre-commit", "pyarrow (>=1.0.1)", "pydot", "pyspark", "pytest (>=5.2.0)", "pytest-spark (>=0.6.0)", "shapely", "twine (>=3.1.1)"]
-type-geometry = ["shapely"]
-type-image-path = ["Pillow", "imagehash"]
+test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""]
[[package]]
name = "wcwidth"
@@ -7421,6 +7654,7 @@ version = "0.2.13"
description = "Measures the displayed width of unicode strings in a terminal"
optional = false
python-versions = "*"
+groups = ["main", "dev"]
files = [
{file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"},
{file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"},
@@ -7432,6 +7666,7 @@ version = "24.8.0"
description = "A library for working with the color formats defined by HTML and CSS."
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "webcolors-24.8.0-py3-none-any.whl", hash = "sha256:fc4c3b59358ada164552084a8ebee637c221e4059267d0f8325b3b560f6c7f0a"},
{file = "webcolors-24.8.0.tar.gz", hash = "sha256:08b07af286a01bcd30d583a7acadf629583d1f79bfef27dd2c2c5c263817277d"},
@@ -7447,6 +7682,7 @@ version = "0.5.1"
description = "Character encoding aliases for legacy web content"
optional = false
python-versions = "*"
+groups = ["dev"]
files = [
{file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"},
{file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"},
@@ -7458,6 +7694,7 @@ version = "1.8.0"
description = "WebSocket client for Python with low level API options"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
files = [
{file = "websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526"},
{file = "websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"},
@@ -7474,6 +7711,8 @@ version = "0.45.1"
description = "A built-package format for Python"
optional = false
python-versions = ">=3.8"
+groups = ["dev"]
+markers = "python_version == \"3.8\""
files = [
{file = "wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248"},
{file = "wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729"},
@@ -7488,103 +7727,19 @@ version = "4.0.13"
description = "Jupyter interactive widgets for Jupyter Notebook"
optional = false
python-versions = ">=3.7"
+groups = ["main", "dev"]
files = [
{file = "widgetsnbextension-4.0.13-py3-none-any.whl", hash = "sha256:74b2692e8500525cc38c2b877236ba51d34541e6385eeed5aec15a70f88a6c71"},
{file = "widgetsnbextension-4.0.13.tar.gz", hash = "sha256:ffcb67bc9febd10234a362795f643927f4e0c05d9342c727b65d2384f8feacb6"},
]
-[[package]]
-name = "wordcloud"
-version = "1.9.4"
-description = "A little word cloud generator"
-optional = false
-python-versions = ">=3.7"
-files = [
- {file = "wordcloud-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:61a84e7311fce8415943edcb7b2ba65b4bfec1dc6dff8fe5a8ea76e278447fb2"},
- {file = "wordcloud-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e8752750726f31385f364823d3ef1d9c8ec829e5c07706c36beb40679945c71"},
- {file = "wordcloud-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990dfd6dd43a1c7fa156be865eb98aba167a986b65f56cbf50e24772107fcd70"},
- {file = "wordcloud-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a70fe8999cd63aec64daa0377b720be6e5ff344963b828caeb4c2a081599a3a0"},
- {file = "wordcloud-1.9.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:37dcd5500cc2ea02950739390e89e2efa6624c2f54b5e2df1ee961fce685b2d7"},
- {file = "wordcloud-1.9.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f5cc5c902dc2492b9fc0e29a1f5c688422d7e6eb9e5c0e43f0331d1c8e1341ba"},
- {file = "wordcloud-1.9.4-cp310-cp310-win32.whl", hash = "sha256:c20fbb51af2046c940b4fead4bafffc30b4191f5fb477c3af844446d8956bfd4"},
- {file = "wordcloud-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:61a153e76d73c72f5cc6c89ee80ddad70758a207c3c6b1d86be8635ec70164f1"},
- {file = "wordcloud-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:af168eeaed67a675f35b5668a7804c4d64f8e4f62a273b909eb5cc39efc4c294"},
- {file = "wordcloud-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3092bf85cb20158c8b90d78650dc0226985109ac6fe13a0086ac47b9581b62ce"},
- {file = "wordcloud-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddfb852f551681f5e33feb934505e060952b6aa98aaa48c781cdbf101f84e7cc"},
- {file = "wordcloud-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57ad8064a634a4870fcd00a9694c0a7839c6dfbac3d32522c69d5e1e9cbfd911"},
- {file = "wordcloud-1.9.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ea14858973ad8561a20a5475eb8d7ad33622bc5f27c60206fbb3e10a036cee26"},
- {file = "wordcloud-1.9.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b27759f12dd235468ff8c1df875b106b23dbf2c74aae05cdcdc3ccd8e23ea89c"},
- {file = "wordcloud-1.9.4-cp311-cp311-win32.whl", hash = "sha256:0ac3d87627022fb8cce17297298be96c91185edd55ecf8906f89f981b55974f0"},
- {file = "wordcloud-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:85368249df056527f1b64e80e68636abb61f0f6bd2d1c430894d2af1feea7f73"},
- {file = "wordcloud-1.9.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3910494ce5acb27731fd5678d146e8aa8f588d5fdb455810c817ff4b84ee0f67"},
- {file = "wordcloud-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b1c29a0089ee90778700cc96305fa830a6a5bbb342eaaa59d6ac8d37a9b232f"},
- {file = "wordcloud-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f369ae7bef16341c2bb208e658d5e4c56517046eb6176f89ac95525eaf8ace09"},
- {file = "wordcloud-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ec6ffba61ca20123e7c09103a5692bbc3163f75ee0bdc7893e80e0e2786ccd2"},
- {file = "wordcloud-1.9.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cdc4aac2bcce77fd91dbfe91db5a8c0cdc239e10d8954356d2ebf79a3b43646c"},
- {file = "wordcloud-1.9.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e4942fbed48a88a0c42c5b0a057651fc09d26b31be8b6c069adaaa5051836040"},
- {file = "wordcloud-1.9.4-cp312-cp312-win32.whl", hash = "sha256:96b801fe4b2aa39bb6c5e68b4a74c81fd8996dd5fb5cea31fda518dc5f77ad82"},
- {file = "wordcloud-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:360977705d0808a1795fcbe98afb5dc4833cb4bb8e421cbb10e93ef0bce816ff"},
- {file = "wordcloud-1.9.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:88c4c99f43b13df0e812fac0e4680cca2afd3ce16ade506812127ed7c7b9d132"},
- {file = "wordcloud-1.9.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2367ec70b2f195c278f91caf4674871ee9218eb57250e01a02b986d34e55f88e"},
- {file = "wordcloud-1.9.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6104a52936886dbc785844ab6986b5321a312238abb242ee4062c7b3fdcca7c"},
- {file = "wordcloud-1.9.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:81bbe75b2725730bf5cbabfe86a5c38960e7ce1166f76ba7001964d8de50b3a7"},
- {file = "wordcloud-1.9.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:a936b8e03c32cc84c99ad8f1bdaf261dfef6c44d31ca5b0c7d0df147220dbb3c"},
- {file = "wordcloud-1.9.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:046300566df97b48640bd3efd94957a56941ada98cc23f811bc3f9b6a0ac1350"},
- {file = "wordcloud-1.9.4-cp313-cp313-win32.whl", hash = "sha256:22357990a01d87579dbd38a06c2a5c7b601179c4e17517b1b8f73d25faa6a5ed"},
- {file = "wordcloud-1.9.4-cp313-cp313-win_amd64.whl", hash = "sha256:8c9a5af2fbcf029a19e827adbee58e86efe7536dca7a42380a8601113a86069b"},
- {file = "wordcloud-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:42affa75c1b033cb0a0afb674f653c4af16d51d97a0852c5770b659b903d9af5"},
- {file = "wordcloud-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0876722c35cf4d5d7717ab81ba98b946e07b0e869252248fdd9ea1fd6c977cc"},
- {file = "wordcloud-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:489079ef173fe83ccff8baffd7a3c2d5fedfd31221c25ad21b4de770ea37b49f"},
- {file = "wordcloud-1.9.4-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:3f3dc2dacca48eac9b130a8938b473db81cfbeeb1a738530a7098913941a8211"},
- {file = "wordcloud-1.9.4-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:2e509c4588ae2ce47ee5cc5cf353422e7f7ecc38f450998654ed50565c8a550d"},
- {file = "wordcloud-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:8009f53ba0c3b2d6f2b1dad83e0fb165ebcdfbd000ce62ebe0917106f51d975d"},
- {file = "wordcloud-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:30b1a59b9073eaaa4f2b0f27d5b6b6c3eb6aaa3a6e0b3dbb2220036b25b37dac"},
- {file = "wordcloud-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8a685babefe032716c1a00b7d8cec3f6bfdc1c89fd839578432fc53824a02fea"},
- {file = "wordcloud-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b78b9fb292a243cf8fcdf63b9cc1fd157ec6abbf1a6e675303668b85e948f616"},
- {file = "wordcloud-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f51ab42c00bc4782ab45701de45226a269ca0850df14e1bd63a60da73271724e"},
- {file = "wordcloud-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38ee69d9404504cf2419d60c3017af7ab9e88f4ba6cf47bc1c96b2d5e58ef513"},
- {file = "wordcloud-1.9.4-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9955223708f196c1e431ae3b86074409bc256c5868e4f50eb9c36c6f06f8b1a3"},
- {file = "wordcloud-1.9.4-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3585ab8f4f09f1508f2d351ed48f9b56472ae26eaf6e2d2e76e975abd715d7a2"},
- {file = "wordcloud-1.9.4-cp38-cp38-win32.whl", hash = "sha256:d7d0b89c2ada0e65d84a6ebbdd8d36876b5da1a143cce2f7dcdaff6714232d24"},
- {file = "wordcloud-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:bd7caefe91d4084c1608d816052eeb605d9a7aee0c908f3a9d7421ee6363bde0"},
- {file = "wordcloud-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e5b2f7195adef0a071dc24a568d8a7715bc5cf5d752b4560f51da3aa4467dcf8"},
- {file = "wordcloud-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:34843fa49135c4ed3739dea050696e707fd00e7335ee4ed62c33639589f90adf"},
- {file = "wordcloud-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6570cc4e48e8e951d24ef6599cd8bf7ff405fbe995ff6d596bcdfa290a6206a8"},
- {file = "wordcloud-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17f944805a17b8343eb877c9aa1dc9e5339eb14c02dd00ec80feccea899bbf81"},
- {file = "wordcloud-1.9.4-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:7c1cd2a6ef876f5f9fe0255e44f131a6113f883447ed1cf8bdb86f569603bac9"},
- {file = "wordcloud-1.9.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2b129584327ba21d05869fcf9495f10f7b31a34a580c431c4942a71ce2317e79"},
- {file = "wordcloud-1.9.4-cp39-cp39-win32.whl", hash = "sha256:526dfd822600f158210a191a59cc4bdcaaa1ff05ab2aa199040d857a518b1db6"},
- {file = "wordcloud-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:ac32b851a19b7d2a9ee5e0aebc8210bf16eadc42c5c0da82e36d447552c8ec48"},
- {file = "wordcloud-1.9.4-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:f733cca468eae79af83cdda1de2434f1799cefef461ed892e7679d5a4c929fa1"},
- {file = "wordcloud-1.9.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a99f96efe5983c6eed17abb8766ced713ddf18b26450da74addc91570922e62"},
- {file = "wordcloud-1.9.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80773ec6a9caa2048602bc347151e3b6e68e1d8fab148dfd0d2e7d4302ce5c01"},
- {file = "wordcloud-1.9.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ca95392bba150190cca8df4a97854b554bdeb28007f28bf4698bd7e1af91b310"},
- {file = "wordcloud-1.9.4-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:eed94b42676f4cfa9b9bdac777e3a1f046b16250216dd8ddcb583c4b6e4b1286"},
- {file = "wordcloud-1.9.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b38aae2ff7aa10ad00d57a5b87ed4a573ef04dbc9119d4a304349c9cb3e03b6e"},
- {file = "wordcloud-1.9.4-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3057be0d071afd57afb9be84fec767abdd78eac6396ead0f0f55c6775170945"},
- {file = "wordcloud-1.9.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:9c39351d2cffc15e3794f7afab78e9135d700f61c5b51904c55d9f3729d1a0df"},
- {file = "wordcloud-1.9.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:914745f0312d248c1a0e1f16ae7b3ce82f78924a2b050ca912d2453c62586da4"},
- {file = "wordcloud-1.9.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:885d51d20cc7b0dad2306fb76b867de20e759e005a1a6e183f3865b5e5f53985"},
- {file = "wordcloud-1.9.4-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:61fc126ed9ce8d55bf20acbdc00284f5a6da66900197a2dd7b62c5ac37585ac5"},
- {file = "wordcloud-1.9.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:c7b8536955f5026b0587ff829265392185b6b4bc923f2ed933c805fcac412b28"},
- {file = "wordcloud-1.9.4-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:6a30ed8aa50b98edb113f72ef619581c221ba3678adeeed88345263c90092561"},
- {file = "wordcloud-1.9.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a62627e5b081b23a4586104d4b01d064db7b53342ae123b511326585eaf7433c"},
- {file = "wordcloud-1.9.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e137493365770f59655c7308ff76addc95ada2c6bd50ac119e4c33091e2e4e08"},
- {file = "wordcloud-1.9.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:665f8e7de3dcc1e43aa5bdd9560d56ed51026ba638a33472eede2b9051108adb"},
- {file = "wordcloud-1.9.4.tar.gz", hash = "sha256:b273d8a5ded97d3ead904046b49464dcb71119ee79df875072a4c105cadd347a"},
-]
-
-[package.dependencies]
-matplotlib = "*"
-numpy = ">=1.6.1"
-pillow = "*"
-
[[package]]
name = "xgboost"
version = "2.1.4"
description = "XGBoost Python Package"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "xgboost-2.1.4-py3-none-macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64.whl", hash = "sha256:78d88da184562deff25c820d943420342014dd55e0f4c017cc4563c2148df5ee"},
{file = "xgboost-2.1.4-py3-none-macosx_12_0_arm64.whl", hash = "sha256:523db01d4e74b05c61a985028bde88a4dd380eadc97209310621996d7d5d14a7"},
@@ -7615,6 +7770,7 @@ version = "3.5.0"
description = "Python binding for xxHash"
optional = false
python-versions = ">=3.7"
+groups = ["main"]
files = [
{file = "xxhash-3.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ece616532c499ee9afbb83078b1b952beffef121d989841f7f4b3dc5ac0fd212"},
{file = "xxhash-3.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3171f693dbc2cef6477054a665dc255d996646b4023fe56cb4db80e26f4cc520"},
@@ -7747,6 +7903,7 @@ version = "0.43.0"
description = "A formatter for Python code"
optional = false
python-versions = ">=3.7"
+groups = ["dev"]
files = [
{file = "yapf-0.43.0-py3-none-any.whl", hash = "sha256:224faffbc39c428cb095818cf6ef5511fdab6f7430a10783fdfb292ccf2852ca"},
{file = "yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e"},
@@ -7762,6 +7919,7 @@ version = "1.15.2"
description = "Yet another URL library"
optional = false
python-versions = ">=3.8"
+groups = ["main"]
files = [
{file = "yarl-1.15.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e4ee8b8639070ff246ad3649294336b06db37a94bdea0d09ea491603e0be73b8"},
{file = "yarl-1.15.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a7cf963a357c5f00cb55b1955df8bbe68d2f2f65de065160a1c26b85a1e44172"},
@@ -7868,53 +8026,13 @@ idna = ">=2.0"
multidict = ">=4.0"
propcache = ">=0.2.0"
-[[package]]
-name = "ydata-profiling"
-version = "4.13.0"
-description = "Generate profile report for pandas DataFrame"
-optional = false
-python-versions = "<3.13,>=3.7"
-files = [
- {file = "ydata-profiling-4.13.0.tar.gz", hash = "sha256:07541bde9d93169f72f9616beae09312826e6a3f5b7e4d9df05e6edbf39d876c"},
- {file = "ydata_profiling-4.13.0-py2.py3-none-any.whl", hash = "sha256:b6f027766bdcdd61f70694b7fa8c4ae3131e0e332aad1dc1797c49bb68c3c42a"},
-]
-
-[package.dependencies]
-dacite = ">=1.8"
-htmlmin = "0.1.12"
-imagehash = "4.3.1"
-jinja2 = ">=2.11.1,<3.2"
-matplotlib = ">=3.5,<=3.10"
-multimethod = ">=1.4,<2"
-numba = ">=0.56.0,<1"
-numpy = ">=1.16.0,<2.2"
-pandas = ">1.1,<1.4.0 || >1.4.0,<3.0"
-phik = ">=0.11.1,<0.13"
-pydantic = ">=2"
-PyYAML = ">=5.0.0,<6.1"
-requests = ">=2.24.0,<3"
-scipy = ">=1.4.1,<1.14"
-seaborn = ">=0.10.1,<0.14"
-statsmodels = ">=0.13.2,<1"
-tqdm = ">=4.48.2,<5"
-typeguard = ">=3,<5"
-visions = {version = ">=0.7.5,<0.7.7", extras = ["type-image-path"]}
-wordcloud = ">=1.9.3"
-
-[package.extras]
-dev = ["autodoc-pydantic", "black (>=20.8b1)", "isort (>=5.0.7)", "myst-parser (>=0.18.1)", "pre-commit (>=2.8.2)", "sphinx-autodoc-typehints (>=1.10.3)", "sphinx-multiversion (>=0.2.3)", "sphinx-rtd-theme (>=0.4.3)", "twine", "virtualenv (>=20.0.33)", "wheel"]
-docs = ["mike (>=2.1.1,<2.2.0)", "mkdocs (>=1.6.0,<1.7.0)", "mkdocs-badges", "mkdocs-material (>=9.0.12,<10.0.0)", "mkdocs-material-extensions (>=1.1.1,<2.0.0)", "mkdocs-table-reader-plugin (<=2.2.0)", "mkdocstrings[python] (>=0.20.0,<1.0.0)"]
-notebook = ["ipywidgets (>=7.5.1)", "jupyter (>=1.0.0)"]
-spark = ["numpy (>=1.16.0,<1.24)", "pandas (>1.1,!=1.4.0,<2)", "pyarrow (>=2.0.0)", "pyspark (>=2.3.0)", "visions[type-image-path] (==0.7.5)"]
-test = ["codecov", "coverage (>=6.5,<8)", "kaggle", "nbval", "pyarrow", "pytest", "pytest-cov", "pytest-spark", "twine (>=3.1.1)"]
-unicode = ["tangled-up-in-unicode (==0.2.0)"]
-
[[package]]
name = "yfinance"
version = "0.2.54"
description = "Download market data from Yahoo! Finance API"
optional = false
python-versions = "*"
+groups = ["main"]
files = [
{file = "yfinance-0.2.54-py2.py3-none-any.whl", hash = "sha256:8754f90332158d5d19bf754c1b230864ca2d1d313182a3f94a7bc7718bbe7d90"},
{file = "yfinance-0.2.54.tar.gz", hash = "sha256:a4ab8e2ecba4fda5a36bff0bdc602a014adc732e5eda5d3ac283836ce40356e8"},
@@ -7941,17 +8059,19 @@ version = "3.20.2"
description = "Backport of pathlib-compatible object wrapper for zip files"
optional = false
python-versions = ">=3.8"
+groups = ["main", "dev"]
files = [
{file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"},
{file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"},
]
+markers = {main = "python_version == \"3.8\" or python_version == \"3.9\""}
[package.extras]
-check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"]
+check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""]
cover = ["pytest-cov"]
doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
enabler = ["pytest-enabler (>=2.2)"]
-test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"]
+test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"]
type = ["pytest-mypy"]
[extras]
@@ -7961,6 +8081,6 @@ llm = ["langchain-openai", "pycocoevalcap", "ragas", "sentencepiece", "torch", "
pytorch = ["torch"]
[metadata]
-lock-version = "2.0"
+lock-version = "2.1"
python-versions = ">=3.8.1,<3.12"
-content-hash = "0765c86a78b5cf83a7a88a16e23f0c5e3a410de541ec12321e82db6f4321c675"
+content-hash = "4bde059ccf6ad967c0764953db99d4497be76eaa81d4dcc590ab149467fa4b45"
diff --git a/pyproject.toml b/pyproject.toml
index e05f66c10..4e996a8f1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,7 +10,7 @@ description = "ValidMind Library"
license = "Commercial License"
name = "validmind"
readme = "README.pypi.md"
-version = "2.8.12"
+version = "2.8.20"
[tool.poetry.dependencies]
aiohttp = {extras = ["speedups"], version = "*"}
@@ -39,7 +39,7 @@ polars = "*"
pycocoevalcap = {version = "^1.2", optional = true}
python = ">=3.8.1,<3.12"
python-dotenv = "*"
-ragas = {version = ">=0.2.3", optional = true}
+ragas = {version = ">=0.2.3,<=0.2.7", optional = true}
rouge = ">=1"
scikit-learn = "*,<1.6.0"
scipy = "*"
@@ -56,17 +56,19 @@ torch = {version = ">=1.10.0", optional = true}
tqdm = "*"
transformers = {version = "^4.32.0", optional = true}
xgboost = ">=1.5.2,<3"
-ydata-profiling = "*"
yfinance = "^0.2.48"
[tool.poetry.group.dev.dependencies]
black = "^22.1.0"
click = "*"
cython = "^0.29.34"
+docstring_parser = "*"
flake8 = "^4.0.1"
+griffe = "*"
ipykernel = "^6.22.0"
isort = "^5.12.0"
jupyter = "^1.0.0"
+mdformat = "*"
papermill = "^2.4.0"
pdoc = "^14.4.0"
pre-commit = "^3.3.3"
diff --git a/scripts/generate_quarto_docs.py b/scripts/generate_quarto_docs.py
new file mode 100644
index 000000000..34b85849e
--- /dev/null
+++ b/scripts/generate_quarto_docs.py
@@ -0,0 +1,687 @@
+#!/usr/bin/env python3
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, Set, List, Optional
+from jinja2 import Environment, FileSystemLoader
+import mdformat
+from docstring_parser import parse, Style
+from glob import glob
+import subprocess
+import re
+import inspect
+
+# Add at module level
+_alias_cache = {} # Cache for resolved aliases
+
+def resolve_alias(member: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]:
+ """Resolve an alias to its target member."""
+ if member.get('kind') == 'alias' and member.get('target_path'):
+ target_path = member['target_path']
+
+ # Check cache first
+ if target_path in _alias_cache:
+ return _alias_cache[target_path]
+
+ path_parts = target_path.split('.')
+ # Skip resolution if it's not in our codebase
+ if path_parts[0] != 'validmind':
+ return member
+
+ # Skip known modules that aren't in the documentation
+ if len(path_parts) > 1 and path_parts[1] in ['ai', 'internal']:
+ # Silently return the member without warning for expected missing paths
+ return member
+
+ current = data[path_parts[0]] # Start at validmind
+ for part in path_parts[1:]:
+ if part in current.get('members', {}):
+ current = current['members'][part]
+ else:
+ # If we can't find the direct path, try alternative approaches
+ # For test suites, specially handle class aliases
+ if 'test_suites' in path_parts and current.get('name') == 'test_suites':
+ # If we're looking for a class in test_suites but can't find it directly,
+ # check if it exists anywhere else in the codebase
+ class_name = path_parts[-1]
+ found_class = find_class_in_all_modules(class_name, data)
+ if found_class:
+ # Cache the result if found
+ _alias_cache[target_path] = found_class
+ return found_class
+
+ print(f"Warning: Could not resolve alias path {target_path}, part '{part}' not found")
+ return member
+
+
+ # Cache the result
+ _alias_cache[target_path] = current
+ return current
+ return member
+
+def get_all_members(members: Dict[str, Any]) -> Set[str]:
+ """Extract the __all__ list from a module's members if present."""
+ if '__all__' in members:
+ all_elements = members['__all__'].get('value', {}).get('elements', [])
+ return {elem.strip("'") for elem in all_elements}
+ return set()
+
+def get_all_list(members: Dict[str, Any]) -> List[str]:
+ """Extract the __all__ list from a module's members if present, preserving order."""
+ if '__all__' in members:
+ all_elements = members['__all__'].get('value', {}).get('elements', [])
+ return [elem.strip("'") for elem in all_elements]
+ return []
+
+def sort_members(members, is_errors_module=False):
+ """Sort members by kind and name."""
+ if isinstance(members, dict):
+ members = members.values()
+
+ def get_sort_key(member):
+ name = str(member.get('name', ''))
+ kind = member.get('kind', '')
+
+ if is_errors_module and kind == 'class':
+ # Base errors first
+ if name == 'BaseError':
+ return ('0', '0', name) # Use strings for consistent comparison
+ elif name == 'APIRequestError':
+ return ('0', '1', name)
+ # Then group by category
+ elif name.startswith('API') or name.endswith('APIError'):
+ return ('1', '0', name)
+ elif 'Model' in name:
+ return ('2', '0', name)
+ elif 'Test' in name:
+ return ('3', '0', name)
+ elif name.startswith('Invalid') or name.startswith('Missing'):
+ return ('4', '0', name)
+ elif name.startswith('Unsupported'):
+ return ('5', '0', name)
+ else:
+ return ('6', '0', name)
+ else:
+ # Default sorting for non-error modules
+ if kind == 'class':
+ return ('0', name.lower())
+ elif kind == 'function':
+ return ('1', name.lower())
+ else:
+ return ('2', name.lower())
+
+ return sorted(members, key=get_sort_key)
+
+def is_public(member: Dict[str, Any], module: Dict[str, Any], full_data: Dict[str, Any], is_root: bool = False) -> bool:
+ """Check if a member should be included in public documentation."""
+ name = member.get('name', '')
+ path = member.get('path', '')
+
+ # Skip private members except __init__ and __post_init__
+ if name.startswith('_') and name not in {'__init__', '__post_init__'}:
+ return False
+
+ # Specifically exclude SkipTestError and logger/get_logger from test modules
+ if name in {'SkipTestError', 'logger'} and 'tests' in path:
+ return False
+
+ if name == 'get_logger' and path.startswith('validmind.tests'):
+ return False
+
+ # Check if the member is an alias that's imported from another module
+ if member.get('kind') == 'alias' and member.get('target_path'):
+ # If the module has __all__, only include aliases listed there
+ if module and '__all__' in module.get('members', {}):
+ module_all = get_all_members(module.get('members', {}))
+ return name in module_all
+
+ # Otherwise, skip aliases (imported functions) unless at root level
+ if not is_root:
+ return False
+
+ # At root level, only show items from __all__
+ if is_root:
+ root_all = get_all_members(full_data['validmind'].get('members', {}))
+ return name in root_all
+
+ # If module has __all__, only include members listed there
+ if module and '__all__' in module.get('members', {}):
+ module_all = get_all_members(module.get('members', {}))
+ return name in module_all
+
+ return True
+
+def ensure_dir(path):
+ """Create directory if it doesn't exist."""
+ Path(path).mkdir(parents=True, exist_ok=True)
+
+def clean_anchor_text(heading: str) -> str:
+ """Safely clean heading text for anchor generation.
+
+ Handles:
+ - ()
+ - class
+ - Other HTML formatting
+ """
+ # First check if this is a class heading
+ if 'class' in heading or 'class' in heading:
+ # Remove the HTML span for class
+ class_name = re.sub(r'class\s*', '', heading)
+ return 'class-' + class_name.strip().lower()
+
+ # For other headings, remove any HTML spans
+ cleaned = re.sub(r'\(\)', '', heading)
+ cleaned = re.sub(r'[^<]*', '', cleaned)
+ return cleaned.strip().lower()
+
+def collect_documented_items(module: Dict[str, Any], path: List[str], full_data: Dict[str, Any], is_root: bool = False) -> Dict[str, List[Dict[str, str]]]:
+ """Collect all documented items from a module and its submodules."""
+ result = {}
+
+ # Skip if no members
+ if not module.get('members'):
+ return result
+
+ # Determine if this is the root module
+ is_root = module.get('name') == 'validmind' or is_root
+
+ # Build the current file path
+ file_path = '/'.join(path)
+ module_name = module.get('name', 'root')
+
+ # For root module, parse validmind.qmd to get headings
+ if is_root:
+ module_items = []
+ qmd_filename = f"{path[-1]}.qmd"
+ qmd_path = written_qmd_files.get(qmd_filename)
+
+ if qmd_path and os.path.exists(qmd_path):
+ with open(qmd_path, 'r') as f:
+ content = f.read()
+
+ # Track current class for nesting methods
+ current_class = None
+
+ # Parse headings - only update the heading level checks
+ for line in content.split('\n'):
+ if line.startswith('## '): # Main function/class level
+ heading = line[3:].strip()
+ anchor = clean_anchor_text(heading)
+ item = {
+ 'text': heading,
+ 'file': f"validmind/validmind.qmd#{anchor}"
+ }
+
+ # Detect class by presence of class span or prefix span
+ is_class = 'class' in heading or 'class' in heading
+ prefix_class = '' in heading
+
+ if is_class or prefix_class:
+ item['contents'] = []
+ current_class = item
+ module_items.append(item)
+ elif line.startswith('### ') and current_class: # Method level
+ heading = line[4:].strip()
+ anchor = clean_anchor_text(heading)
+ method_item = {
+ 'text': heading,
+ 'file': f"validmind/validmind.qmd#{anchor}"
+ }
+ current_class['contents'].append(method_item)
+
+ # Clean up empty contents lists
+ for item in module_items:
+ if 'contents' in item and not item['contents']:
+ del item['contents']
+
+ if module_items:
+ result['root'] = module_items
+
+ # Process submodules
+ for member in sort_members(module['members'], module.get('name') == 'errors'):
+ if member['kind'] == 'module' and is_public(member, module, full_data, is_root):
+ submodule_path = path + [member['name']]
+ submodule_items = collect_documented_items(member, submodule_path, full_data, False)
+ result.update(submodule_items)
+
+ # Also check for nested modules in the submodule
+ if member.get('members'):
+ for submember in sort_members(member['members'], member.get('name') == 'errors'):
+ if submember['kind'] == 'module' and is_public(submember, member, full_data, False):
+ subsubmodule_path = submodule_path + [submember['name']]
+ subsubmodule_items = collect_documented_items(submember, subsubmodule_path, full_data, False)
+ result.update(subsubmodule_items)
+
+ return result
+
+# Add at module level
+written_qmd_files = {}
+
+def find_class_in_all_modules(class_name: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+ """Recursively search for a class in all modules of the data structure."""
+ if not isinstance(data, dict):
+ return None
+
+ # Check if this is the class we're looking for
+ if data.get('kind') == 'class' and data.get('name') == class_name:
+ return data
+
+ # Special handling for common test suite classes
+ if class_name.endswith(('Suite', 'Performance', 'Metrics', 'Diagnosis', 'Validation', 'Description')):
+ # These are likely test suite classes, check specifically in test_suites module if available
+ if 'validmind' in data and 'test_suites' in data['validmind'].get('members', {}):
+ test_suites = data['validmind']['members']['test_suites']
+ if class_name in test_suites.get('members', {}):
+ return test_suites['members'][class_name]
+
+ # Check members if this is a module
+ if 'members' in data:
+ for member_name, member in data['members'].items():
+ # Direct match in members
+ if member_name == class_name and member.get('kind') == 'class':
+ return member
+
+ # Recursive search in this member
+ result = find_class_in_all_modules(class_name, member)
+ if result:
+ return result
+
+ return None
+
+def process_module(module: Dict[str, Any], path: List[str], env: Environment, full_data: Dict[str, Any]):
+ """Process a module and its submodules."""
+ # Parse docstrings first
+ parse_docstrings(module)
+
+ module_dir = os.path.join('docs', *path[:-1])
+ ensure_dir(module_dir)
+
+ # Extract __all__ list if present (preserving order)
+ if module.get('members') and '__all__' in module.get('members', {}):
+ module['all_list'] = get_all_list(module['members'])
+
+ # Special handling for test_suites module
+ is_test_suites = path and path[-1] == "test_suites"
+ if is_test_suites:
+ # Ensure all class aliases are properly resolved
+ for member_name, member in module.get('members', {}).items():
+ if member.get('kind') == 'alias' and member.get('target_path'):
+ # Try to resolve and cache the target now
+ resolve_alias(member, full_data)
+
+ # Enhanced debugging for vm_models
+ if path and path[-1] == 'vm_models':
+ # Handle special case for vm_models module
+ # Look for result module and copy necessary classes
+ result_module = None
+ for name, member in module.get('members', {}).items():
+ if name == 'result' and member.get('kind') == 'module':
+ result_module = member
+
+ # Copy ResultTable and TestResult to vm_models members if needed
+ if 'ResultTable' in member.get('members', {}):
+ module['members']['ResultTable'] = member['members']['ResultTable']
+
+ if 'TestResult' in member.get('members', {}):
+ module['members']['TestResult'] = member['members']['TestResult']
+ break
+
+ if not result_module:
+ # Fallback: try to find the classes directly in the full data structure
+ result_table = find_class_in_all_modules('ResultTable', full_data)
+ if result_table:
+ module['members']['ResultTable'] = result_table
+
+ test_result = find_class_in_all_modules('TestResult', full_data)
+ if test_result:
+ module['members']['TestResult'] = test_result
+
+ # Check if this is a test module
+ is_test_module = 'tests' in path
+
+ # Get appropriate template based on module name
+ if path[-1] == 'errors':
+ # Use the specialized errors template for the errors module
+ template = env.get_template('errors.qmd.jinja2')
+
+ # Render with the errors template
+ output = template.render(
+ module=module,
+ members=module.get('members', {}), # Pass members directly
+ full_data=full_data,
+ is_errors_module=True
+ )
+ else:
+ # Use the standard module template for all other modules
+ template = env.get_template('module.qmd.jinja2')
+
+ # Generate module documentation
+ output = template.render(
+ module=module,
+ full_data=full_data,
+ is_root=(len(path) <= 1),
+ resolve_alias=resolve_alias,
+ is_test_module=is_test_module # Pass this flag to template
+ )
+
+ # Write output
+ filename = f"{path[-1]}.qmd"
+ output_path = os.path.join(module_dir, filename)
+ with open(output_path, 'w') as f:
+ f.write(output)
+
+ # Track with full relative path as key
+ rel_path = os.path.join(*path[1:], filename) if len(path) > 1 else filename
+ full_path = os.path.join("docs", os.path.relpath(output_path, "docs"))
+ written_qmd_files[rel_path] = full_path
+
+ # Generate version.qmd for root module
+ if module.get('name') == 'validmind' and module.get('members', {}).get('__version__'):
+ version_template = env.get_template('version.qmd.jinja2')
+ version_output = version_template.render(
+ module=module,
+ full_data=full_data
+ )
+ # Removed the underscores from the filename as Quarto treats files with underscores differently
+ version_path = os.path.join('docs/validmind', 'version.qmd')
+ with open(version_path, 'w') as f:
+ f.write(version_output)
+ written_qmd_files['version.qmd'] = version_path
+
+ # Process submodules
+ members = module.get('members', {})
+ for name, member in members.items():
+ if member.get('kind') == 'module':
+ if is_public(member, module, full_data, is_root=len(path) <= 1):
+ process_module(member, path + [name], env, full_data)
+
+def lint_markdown_files(output_dir: str):
+ """Clean up whitespace and formatting in all generated markdown files."""
+ for path in Path(output_dir).rglob('*.qmd'):
+ with open(path) as f:
+ content = f.read()
+
+ # Split content into front matter and body
+ parts = content.split('---', 2)
+ if len(parts) >= 3:
+ # Preserve front matter and format the rest
+ front_matter = parts[1]
+ body = parts[2]
+ formatted_body = mdformat.text(body, options={
+ "wrap": "no",
+ "number": False,
+ "normalize_whitespace": True
+ })
+ formatted = f"---{front_matter}---\n\n{formatted_body}"
+ else:
+ # No front matter, format everything
+ formatted = mdformat.text(content, options={
+ "wrap": "no",
+ "number": False,
+ "normalize_whitespace": True
+ })
+
+ with open(path, 'w') as f:
+ f.write(formatted)
+
+def parse_docstrings(data: Dict[str, Any]):
+ """Recursively parse all docstrings in the data structure."""
+ if isinstance(data, dict):
+ if 'docstring' in data:
+ if isinstance(data['docstring'], dict):
+ original = data['docstring'].get('value', '')
+ elif isinstance(data['docstring'], str):
+ original = data['docstring']
+ else:
+ original = str(data['docstring'])
+
+ try:
+ # Pre-process all docstrings to normalize newlines
+ sections = original.split('\n\n')
+ # Join lines in the first section (description) with spaces
+ if sections:
+ sections[0] = ' '.join(sections[0].split('\n'))
+ # Keep other sections as-is
+ original = '\n\n'.join(sections)
+
+ parsed = parse(original, style=Style.GOOGLE)
+
+ data['docstring'] = {
+ 'value': original,
+ 'parsed': parsed
+ }
+ except Exception as e:
+ print(f"\nParsing failed for {data.get('name', 'unknown')}:")
+ print(f"Error: {str(e)}")
+ print(f"Original:\n{original}")
+
+ if 'members' in data:
+ for member in data['members'].values():
+ parse_docstrings(member)
+
+def get_inherited_members(base: Dict[str, Any], full_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+ """Get all inherited members from a base class."""
+ # Handle case where a class object is passed instead of a base name
+ if isinstance(base, dict) and 'bases' in base:
+ all_members = []
+ for base_item in base['bases']:
+ if isinstance(base_item, dict) and 'name' in base_item:
+ base_members = get_inherited_members(base_item['name'], full_data)
+ all_members.extend(base_members)
+ return all_members
+
+ # Get the base class name
+ base_name = base if isinstance(base, str) else base.get('name', '')
+ if not base_name:
+ return []
+
+ # Handle built-in exceptions
+ if base_name == 'Exception' or base_name.startswith('builtins.'):
+ return [
+ {'name': 'with_traceback', 'kind': 'builtin', 'base': 'builtins.BaseException'},
+ {'name': 'add_note', 'kind': 'builtin', 'base': 'builtins.BaseException'}
+ ]
+
+ # Look for the base class in the errors module
+ errors_module = full_data.get('validmind', {}).get('members', {}).get('errors', {}).get('members', {})
+ base_class = errors_module.get(base_name)
+
+ if not base_class:
+ return []
+
+ # Return the base class and its description method if it exists
+ members = [{'name': base_name, 'kind': 'class', 'base': base_name}]
+
+ # Add all public methods
+ for name, member in base_class.get('members', {}).items():
+ # Skip private methods (including __init__)
+ if name.startswith('_'):
+ continue
+
+ if member['kind'] in ('function', 'method', 'property'):
+ # Add the method to the list of inherited members
+ method_info = {
+ 'name': name,
+ 'kind': 'method',
+ 'base': base_name,
+ 'parameters': member.get('parameters', []), # Include parameters
+ 'returns': member.get('returns', None), # Include return type
+ 'docstring': member.get('docstring', {}).get('value', ''),
+ }
+
+ members.append(method_info)
+
+ # Add built-in methods from Exception
+ members.extend([
+ {'name': 'with_traceback', 'kind': 'builtin', 'base': 'builtins.BaseException'},
+ {'name': 'add_note', 'kind': 'builtin', 'base': 'builtins.BaseException'}
+ ])
+
+ return members
+
+def get_child_files(files_dict: Dict[str, str], module_name: str) -> List[Dict[str, Any]]:
+ """Get all child QMD files for a given module."""
+ prefix = f'docs/validmind/{module_name}/'
+ directory_structure = {}
+
+ # First pass: organize files by directory
+ for filename, path in files_dict.items():
+ if path.startswith(prefix) and path != f'docs/validmind/{module_name}.qmd':
+ # Remove the prefix to get the relative path
+ rel_path = path.replace('docs/', '')
+ parts = Path(rel_path).parts[2:] # Skip 'validmind' and module_name
+
+ # Handle directory-level QMD and its children
+ if len(parts) == 1: # Direct child
+ dir_name = Path(parts[0]).stem
+ if dir_name not in directory_structure:
+ directory_structure[dir_name] = {
+ 'text': dir_name,
+ 'file': f'validmind/{rel_path}' # Add validmind/ prefix
+ }
+ else: # Nested file
+ dir_name = parts[0]
+ if dir_name not in directory_structure:
+ directory_structure[dir_name] = {
+ 'text': dir_name,
+ 'file': f'validmind/validmind/{module_name}/{dir_name}.qmd' # Add validmind/ prefix
+ }
+
+ # Add to contents if it's a child file
+ if 'contents' not in directory_structure[dir_name]:
+ directory_structure[dir_name]['contents'] = []
+
+ directory_structure[dir_name]['contents'].append({
+ 'text': Path(parts[-1]).stem,
+ 'file': f'validmind/{rel_path}' # Add validmind/ prefix
+ })
+
+ # Sort children within each directory
+ for dir_info in directory_structure.values():
+ if 'contents' in dir_info:
+ dir_info['contents'].sort(key=lambda x: x['text'])
+
+ # Return sorted list of directories
+ return sorted(directory_structure.values(), key=lambda x: x['text'])
+
+def has_subfiles(files_dict, module_name):
+ """Check if a module has child QMD files."""
+ prefix = f'docs/validmind/{module_name}/'
+ return any(path.startswith(prefix) for path in files_dict.values())
+
+def find_qmd_files(base_path: str) -> Dict[str, str]:
+ """Find all .qmd files and their associated paths."""
+ # Convert the written_qmd_files paths to be relative to docs/
+ relative_paths = {}
+ for filename, path in written_qmd_files.items():
+ if path.startswith('docs/'):
+ relative_paths[filename] = path
+ else:
+ relative_paths[filename] = f'docs/{path}'
+ return relative_paths
+
+def generate_docs(json_path: str, template_dir: str, output_dir: str):
+ """Generate documentation from JSON data using templates."""
+ # Load JSON data
+ with open(json_path) as f:
+ data = json.load(f)
+
+ # Set up Jinja environment
+ env = Environment(
+ loader=FileSystemLoader(template_dir),
+ trim_blocks=True,
+ lstrip_blocks=True
+ )
+
+ # Add custom filters and globals
+ env.filters['sort_members'] = sort_members
+ env.filters['has_subfiles'] = has_subfiles
+ env.filters['get_child_files'] = get_child_files
+ env.globals['is_public'] = is_public
+ env.globals['resolve_alias'] = resolve_alias
+ env.globals['get_all_members'] = get_all_members
+ env.globals['get_all_list'] = get_all_list
+ env.globals['get_inherited_members'] = get_inherited_members
+
+ # Start processing from root module
+ if 'validmind' in data:
+ # First pass: Generate module documentation
+ process_module(data['validmind'], ['validmind'], env, data)
+
+ qmd_files = find_qmd_files(output_dir)
+
+ # Add to template context
+ env.globals['qmd_files'] = qmd_files
+
+ # Second pass: Collect all documented items
+ documented_items = collect_documented_items(
+ module=data['validmind'],
+ path=['validmind'],
+ full_data=data,
+ is_root=True
+ )
+
+ # Generate sidebar with collected items
+ sidebar_template = env.get_template('sidebar.qmd.jinja2')
+ sidebar_output = sidebar_template.render(
+ module=data['validmind'],
+ full_data=data,
+ is_root=True,
+ resolve_alias=resolve_alias,
+ documented_items=documented_items
+ )
+
+ # Write sidebar
+ sidebar_path = os.path.join(output_dir, '_sidebar.yml')
+ with open(sidebar_path, 'w') as f:
+ f.write(sidebar_output)
+
+ # Clean up markdown formatting
+ lint_markdown_files(output_dir)
+ else:
+ print("Error: No 'validmind' module found in JSON")
+
+def parse_docstring(docstring):
+ """Parse a docstring into its components."""
+ if not docstring:
+ return None
+ try:
+ # Pre-process docstring to reconstruct original format
+ lines = docstring.split('\n')
+ processed_lines = []
+ in_args = False
+ current_param = []
+
+ for line in lines:
+ line = line.strip()
+ # Check if we're in the Args section
+ if line.startswith('Args:'):
+ in_args = True
+ processed_lines.append(line)
+ continue
+
+ if in_args and line:
+ # Fix mangled parameter lines like "optional): The test suite name..."
+ if line.startswith('optional)'):
+ # Extract the actual parameter name from the description
+ desc_parts = line.split(':', 1)[1].strip().split('(')
+ if len(desc_parts) > 1:
+ param_name = desc_parts[1].split(',')[0].strip()
+ desc = desc_parts[0].strip()
+ line = f" {param_name} (str, optional): {desc}"
+ processed_lines.append(line)
+ else:
+ processed_lines.append(line)
+
+ processed_docstring = '\n'.join(processed_lines)
+ return parse(processed_docstring, style=Style.GOOGLE)
+ except Exception as e:
+ # Fallback to just returning the raw docstring
+ return {'value': docstring}
+
+if __name__ == '__main__':
+ generate_docs(
+ json_path='docs/validmind.json',
+ template_dir='docs/templates',
+ output_dir='docs'
+ )
\ No newline at end of file
diff --git a/tests/test_api_client.py b/tests/test_api_client.py
index 3920cdc2f..7e2a9d7b6 100644
--- a/tests/test_api_client.py
+++ b/tests/test_api_client.py
@@ -201,6 +201,7 @@ def test_log_test_result(self, mock_post):
"inputs": ["input1"],
"passed": True,
"summary": [{"key": "value"}],
+ "config": None,
}
mock_post.return_value = MockAsyncResponse(200, json={"cuid": "abc1234"})
diff --git a/tests/unit_tests/data_validation/test_DatasetDescription.py b/tests/unit_tests/data_validation/test_DatasetDescription.py
index 16b9f2b9a..6f7b1e7ef 100644
--- a/tests/unit_tests/data_validation/test_DatasetDescription.py
+++ b/tests/unit_tests/data_validation/test_DatasetDescription.py
@@ -22,13 +22,13 @@ def setUp(self):
[True, False, True, True, False, True, False], dtype=bool
), # Explicitly boolean
"text": [
- "hello",
- "world",
+ "hello@gmail.com",
+ "this is a longer text",
"hello world",
- "test",
- "hello",
- "test",
- "world",
+ "this is a longer text",
+ "this is a longer text",
+ "another example of text",
+ "this is a longer text",
], # Text
"all_null": [
None,
@@ -129,7 +129,7 @@ def test_column_types_and_stats(self):
# Check text column
self.assertEqual(column_info["text"]["Type"], "Text")
- self.assertEqual(column_info["text"]["Distinct"], 4) # 4 unique strings
+ self.assertEqual(column_info["text"]["Distinct"],4) # 4 unique strings
self.assertEqual(column_info["text"]["Missing"], 0) # No missing values
self.assertEqual(column_info["text"]["Count"], 7) # All present
diff --git a/validmind/__init__.py b/validmind/__init__.py
index 3099934ce..c99f3a537 100644
--- a/validmind/__init__.py
+++ b/validmind/__init__.py
@@ -43,7 +43,7 @@
warnings.simplefilter("ignore", category=NumbaPendingDeprecationWarning)
from .__version__ import __version__ # noqa: E402
-from .api_client import init, log_metric, reload
+from .api_client import init, log_metric, log_text, reload
from .client import ( # noqa: E402
get_test_suite,
init_dataset,
@@ -99,19 +99,19 @@ def check_version():
"__version__",
# main library API
"init",
- "reload",
"init_dataset",
"init_model",
"init_r_model",
+ "get_test_suite",
+ "log_metric",
"preview_template",
+ "print_env",
+ "reload",
"run_documentation_tests",
# log metric function (for direct/bulk/retroactive logging of metrics)
- "log_metric",
# test suite functions (less common)
- "get_test_suite",
"run_test_suite",
# helper functions (for troubleshooting)
- "print_env",
# decorators (for building tests
"tags",
"tasks",
@@ -125,4 +125,5 @@ def check_version():
"tests",
"unit_metrics",
"test_suites",
+ "log_text",
]
diff --git a/validmind/__version__.py b/validmind/__version__.py
index 7953a978c..ca466009f 100644
--- a/validmind/__version__.py
+++ b/validmind/__version__.py
@@ -1 +1 @@
-__version__ = "2.8.12"
+__version__ = "2.8.20"
diff --git a/validmind/ai/test_descriptions.py b/validmind/ai/test_descriptions.py
index 2f57270a1..3a9a05ebe 100644
--- a/validmind/ai/test_descriptions.py
+++ b/validmind/ai/test_descriptions.py
@@ -70,7 +70,7 @@ def generate_description(
figures: List[Figure] = None,
title: Optional[str] = None,
):
- """Generate the description for the test results"""
+ """Generate the description for the test results."""
from validmind.api_client import generate_test_result_description
if not tables and not figures and not metric:
@@ -138,6 +138,10 @@ def wrapped():
logger.warning(
f"Test result {test_id} is too large to generate a description"
)
+ elif "Too many images" in str(e):
+ logger.warning(
+ f"Test result {test_id} has too many figures to generate a description"
+ )
else:
logger.warning(f"Failed to generate description for {test_id}: {e}")
logger.warning(f"Using default description for {test_id}")
@@ -156,7 +160,7 @@ def get_result_description(
should_generate: bool = True,
title: Optional[str] = None,
):
- """Get Metadata Dictionary for a Test or Metric Result
+ """Get the metadata dictionary for a test or metric result.
Generates an LLM interpretation of the test results or uses the default
description and returns a metadata object that can be logged with the test results.
@@ -170,15 +174,15 @@ def get_result_description(
Note: Either the tables or figures must be provided to generate the description.
Args:
- test_id (str): The test ID
- test_description (str): The default description for the test
- tables (Any): The test tables or results to interpret
- figures (List[Figure]): The figures to attach to the test suite result
- metric (Union[int, float]): Unit metrics attached to the test result
- should_generate (bool): Whether to generate the description or not (Default: True)
+ test_id (str): The test ID.
+ test_description (str): The default description for the test.
+ tables (Any): The test tables or results to interpret.
+ figures (List[Figure]): The figures to attach to the test suite result.
+ metric (Union[int, float]): Unit metrics attached to the test result.
+ should_generate (bool): Whether to generate the description or not. Defaults to True.
Returns:
- str: The description to be logged with the test results
+ str: The description to be logged with the test results.
"""
# Check the feature flag first, then the environment variable
llm_descriptions_enabled = (
diff --git a/validmind/ai/utils.py b/validmind/ai/utils.py
index 6f39604c1..648d26076 100644
--- a/validmind/ai/utils.py
+++ b/validmind/ai/utils.py
@@ -24,7 +24,7 @@ class DescriptionFuture:
the tests can continue to be run in parallel while the description is
retrieved asynchronously.
- The value will be retrieved later and if its not ready yet, it should
+ The value will be retrieved later and, if it is not ready yet, it should
block until it is.
"""
@@ -42,7 +42,7 @@ def get_description(self):
def get_client_and_model():
- """Get model and client to use for generating interpretations
+ """Get model and client to use for generating interpretations.
On first call, it will look in the environment for the API key endpoint, model etc.
and store them in a global variable to avoid loading them up again.
diff --git a/validmind/api_client.py b/validmind/api_client.py
index cb6228616..c5755daaa 100644
--- a/validmind/api_client.py
+++ b/validmind/api_client.py
@@ -18,11 +18,12 @@
import aiohttp
import requests
from aiohttp import FormData
+from ipywidgets import HTML, Accordion
from .client_config import client_config
from .errors import MissingAPICredentialsError, MissingModelIdError, raise_api_error
from .logging import get_logger, init_sentry, send_single_error
-from .utils import NumpyEncoder, run_async
+from .utils import NumpyEncoder, is_html, md_to_html, run_async
from .vm_models import Figure
logger = get_logger(__name__)
@@ -38,7 +39,7 @@
@atexit.register
def _close_session():
- """Closes the async client session at exit"""
+ """Closes the async client session at exit."""
global __api_session
if __api_session and not __api_session.closed:
@@ -78,7 +79,7 @@ def _get_api_headers() -> Dict[str, str]:
def _get_session() -> aiohttp.ClientSession:
- """Initializes the async client session"""
+ """Initializes the async client session."""
global __api_session
if not __api_session or __api_session.closed:
@@ -156,7 +157,7 @@ async def _post(
def _ping() -> Dict[str, Any]:
- """Validates that we can connect to the ValidMind API (does not use the async session)"""
+ """Validates that we can connect to the ValidMind API (does not use the async session)."""
r = requests.get(
url=_get_url("ping"),
headers=_get_api_headers(),
@@ -243,7 +244,7 @@ def init(
def reload():
- """Reconnect to the ValidMind API and reload the project configuration"""
+ """Reconnect to the ValidMind API and reload the project configuration."""
try:
_ping()
@@ -258,13 +259,13 @@ async def aget_metadata(content_id: str) -> Dict[str, Any]:
"""Gets a metadata object from ValidMind API.
Args:
- content_id (str): Unique content identifier for the metadata
+ content_id (str): Unique content identifier for the metadata.
Raises:
- Exception: If the API call fails
+ Exception: If the API call fails.
Returns:
- dict: Metadata object
+ dict: Metadata object.
"""
return await _get(f"get_metadata/{content_id}")
@@ -277,15 +278,15 @@ async def alog_metadata(
"""Logs free-form metadata to ValidMind API.
Args:
- content_id (str): Unique content identifier for the metadata
+ content_id (str): Unique content identifier for the metadata.
text (str, optional): Free-form text to assign to the metadata. Defaults to None.
_json (dict, optional): Free-form key-value pairs to assign to the metadata. Defaults to None.
Raises:
- Exception: If the API call fails
+ Exception: If the API call fails.
Returns:
- dict: The response from the API
+ dict: The response from the API.
"""
metadata_dict = {"content_id": content_id}
if text is not None:
@@ -304,16 +305,16 @@ async def alog_metadata(
async def alog_figure(figure: Figure) -> Dict[str, Any]:
- """Logs a figure
+ """Logs a figure.
Args:
- figure (Figure): The Figure object wrapper
+ figure (Figure): The Figure object wrapper.
Raises:
- Exception: If the API call fails
+ Exception: If the API call fails.
Returns:
- dict: The response from the API
+ dict: The response from the API.
"""
try:
return await _post(
@@ -330,22 +331,24 @@ async def alog_test_result(
result: Dict[str, Any],
section_id: str = None,
position: int = None,
+ unsafe: bool = False,
+ config: Dict[str, bool] = None,
) -> Dict[str, Any]:
- """Logs test results information
+ """Logs test results information.
This method will be called automatically from any function running tests but
can also be called directly if the user wants to run tests on their own.
Args:
- result (dict): A dictionary representing the test result
- section_id (str, optional): The section ID add a test driven block to the documentation
- position (int): The position in the section to add the test driven block
+ result (dict): A dictionary representing the test result.
+ section_id (str, optional): The section ID add a test driven block to the documentation.
+ position (int): The position in the section to add the test driven block.
Raises:
- Exception: If the API call fails
+ Exception: If the API call fails.
Returns:
- dict: The response from the API
+ dict: The response from the API.
"""
request_params = {}
if section_id:
@@ -357,7 +360,7 @@ async def alog_test_result(
"log_test_results",
params=request_params,
data=json.dumps(
- result,
+ {**result, "config": config},
cls=NumpyEncoder,
allow_nan=False,
),
@@ -405,6 +408,39 @@ def log_input(input_id: str, type: str, metadata: Dict[str, Any]) -> Dict[str, A
return run_async(alog_input, input_id, type, metadata)
+def log_text(
+ content_id: str, text: str, _json: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+ """Logs free-form text to ValidMind API.
+
+ Args:
+ content_id (str): Unique content identifier for the text.
+ text (str): The text to log. Will be converted to HTML with MathML support.
+ _json (dict, optional): Additional metadata to associate with the text. Defaults to None.
+
+ Raises:
+ ValueError: If content_id or text are empty or not strings.
+ Exception: If the API call fails.
+
+ Returns:
+ ipywidgets.Accordion: An accordion widget containing the logged text as HTML.
+ """
+ if not content_id or not isinstance(content_id, str):
+ raise ValueError("`content_id` must be a non-empty string")
+ if not text or not isinstance(text, str):
+ raise ValueError("`text` must be a non-empty string")
+
+ if not is_html(text):
+ text = md_to_html(text, mathml=True)
+
+ log_text = run_async(alog_metadata, content_id, text, _json)
+
+ return Accordion(
+ children=[HTML(log_text["text"])],
+ titles=[f"Text Block: '{log_text['content_id']}'"],
+ )
+
+
async def alog_metric(
key: str,
value: Union[int, float],
@@ -413,7 +449,7 @@ async def alog_metric(
recorded_at: Optional[str] = None,
thresholds: Optional[Dict[str, Any]] = None,
):
- """See log_metric for details"""
+ """See log_metric for details."""
if not key or not isinstance(key, str):
raise ValueError("`key` must be a non-empty string")
@@ -458,7 +494,7 @@ def log_metric(
recorded_at: Optional[str] = None,
thresholds: Optional[Dict[str, Any]] = None,
):
- """Logs a unit metric
+ """Logs a unit metric.
Unit metrics are key-value pairs where the key is the metric name and the value is
a scalar (int or float). These key-value pairs are associated with the currently
@@ -468,18 +504,25 @@ def log_metric(
Args:
key (str): The metric key
- value (float): The metric value
- inputs (list, optional): A list of input IDs that were used to compute the metric.
- params (dict, optional): Dictionary of parameters used to compute the metric.
- recorded_at (str, optional): The timestamp of the metric. Server will use
- current time if not provided.
- thresholds (dict, optional): Dictionary of thresholds for the metric.
+ value (Union[int, float]): The metric value
+ inputs (List[str], optional): List of input IDs
+ params (Dict[str, Any], optional): Parameters used to generate the metric
+ recorded_at (str, optional): Timestamp when the metric was recorded
+ thresholds (Dict[str, Any], optional): Thresholds for the metric
"""
- run_async(alog_metric, key, value, inputs, params, recorded_at, thresholds)
+ return run_async(
+ alog_metric,
+ key=key,
+ value=value,
+ inputs=inputs,
+ params=params,
+ recorded_at=recorded_at,
+ thresholds=thresholds,
+ )
def get_ai_key() -> Dict[str, Any]:
- """Calls the api to get an api key for our LLM proxy"""
+ """Calls the API to get an API key for our LLM proxy."""
r = requests.get(
url=_get_url("ai/key"),
headers=_get_api_headers(),
diff --git a/validmind/client.py b/validmind/client.py
index ef94dc117..fe0517085 100644
--- a/validmind/client.py
+++ b/validmind/client.py
@@ -6,8 +6,12 @@
Client interface for all data and model validation functions
"""
+from typing import Any, Callable, Dict, List, Optional, Union
+
+import numpy as np
import pandas as pd
import polars as pl
+import torch
from .api_client import log_input as log_input
from .client_config import client_config
@@ -42,20 +46,22 @@
def init_dataset(
- dataset,
- model=None,
- index=None,
- index_name: str = None,
+ dataset: Union[
+ pd.DataFrame, pl.DataFrame, "np.ndarray", "torch.utils.data.TensorDataset"
+ ],
+ model: Optional[VMModel] = None,
+ index: Optional[Any] = None,
+ index_name: Optional[str] = None,
date_time_index: bool = False,
- columns: list = None,
- text_column: str = None,
- target_column: str = None,
- feature_columns: list = None,
- extra_columns: dict = None,
- class_labels: dict = None,
- type: str = None,
- input_id: str = None,
- __log=True,
+ columns: Optional[List[str]] = None,
+ text_column: Optional[str] = None,
+ target_column: Optional[str] = None,
+ feature_columns: Optional[List[str]] = None,
+ extra_columns: Optional[Dict[str, Any]] = None,
+ class_labels: Optional[Dict[str, Any]] = None,
+ type: Optional[str] = None,
+ input_id: Optional[str] = None,
+ __log: bool = True,
) -> VMDataset:
"""
Initializes a VM Dataset, which can then be passed to other functions
@@ -69,25 +75,30 @@ def init_dataset(
- Torch TensorDataset
Args:
- dataset : dataset from various python libraries
- model (VMModel): ValidMind model object
- targets (vm.vm.DatasetTargets): A list of target variables
- target_column (str): The name of the target column in the dataset
- feature_columns (list): A list of names of feature columns in the dataset
- extra_columns (dictionary): A dictionary containing the names of the
- prediction_column and group_by_columns in the dataset
- class_labels (dict): A list of class labels for classification problems
- type (str): The type of dataset (one of DATASET_TYPES)
- input_id (str): The input ID for the dataset (e.g. "my_dataset"). By default,
+ dataset: Dataset from various Python libraries.
+ model (VMModel): ValidMind model object.
+ index (Any, optional): Index for the dataset.
+ index_name (str, optional): Name of the index column.
+ date_time_index (bool): Whether the index is a datetime index.
+ columns (List[str], optional): List of column names.
+ text_column (str, optional): Name of the text column.
+ target_column (str, optional): The name of the target column in the dataset.
+ feature_columns (List[str], optional): A list of names of feature columns in the dataset.
+ extra_columns (Dict[str, Any], optional): A dictionary containing the names of the
+ prediction_column and group_by_columns in the dataset.
+ class_labels (Dict[str, Any], optional): A list of class labels for classification problems.
+ type (str, optional): The type of dataset (one of DATASET_TYPES) - DEPRECATED.
+ input_id (str, optional): The input ID for the dataset (e.g. "my_dataset"). By default,
this will be set to `dataset` but if you are passing this dataset as a
test input using some other key than `dataset`, then you should set
this to the same key.
+ __log (bool): Whether to log the input. Defaults to True.
Raises:
- ValueError: If the dataset type is not supported
+ ValueError: If the dataset type is not supported.
Returns:
- vm.vm.Dataset: A VM Dataset instance
+ vm.vm.Dataset: A VM Dataset instance.
"""
# Show deprecation notice if type is passed
if type is not None:
@@ -171,12 +182,12 @@ def init_dataset(
def init_model(
- model: object = None,
+ model: Optional[object] = None,
input_id: str = "model",
- attributes: dict = None,
- predict_fn: callable = None,
- __log=True,
- **kwargs,
+ attributes: Optional[Dict[str, Any]] = None,
+ predict_fn: Optional[Callable] = None,
+ __log: bool = True,
+ **kwargs: Any,
) -> VMModel:
"""
Initializes a VM Model, which can then be passed to other functions
@@ -184,35 +195,21 @@ def init_model(
also ensures we are creating a model supported libraries.
Args:
- model: A trained model or VMModel instance
+ model: A trained model or VMModel instance.
input_id (str): The input ID for the model (e.g. "my_model"). By default,
this will be set to `model` but if you are passing this model as a
test input using some other key than `model`, then you should set
this to the same key.
- attributes (dict): A dictionary of model attributes
- predict_fn (callable): A function that takes an input and returns a prediction
- **kwargs: Additional arguments to pass to the model
+ attributes (dict): A dictionary of model attributes.
+ predict_fn (callable): A function that takes an input and returns a prediction.
+ **kwargs: Additional arguments to pass to the model.
Raises:
- ValueError: If the model type is not supported
+ ValueError: If the model type is not supported.
Returns:
- vm.VMModel: A VM Model instance
+ vm.VMModel: A VM Model instance.
"""
- # vm_model = model if isinstance(model, VMModel) else None
- # metadata = None
-
- # if not vm_model:
- # class_obj = get_model_class(model=model, predict_fn=predict_fn)
- # if not class_obj:
- # if not attributes:
- # raise UnsupportedModelError(
- # f"Model class {str(model.__class__)} is not supported at the moment."
- # )
- # elif not is_model_metadata(attributes):
- # raise UnsupportedModelError(
- # f"Model attributes {str(attributes)} are missing required keys 'architecture' and 'language'."
- # )
vm_model = model if isinstance(model, VMModel) else None
class_obj = get_model_class(model=model, predict_fn=predict_fn)
@@ -276,26 +273,18 @@ def init_r_model(
input_id: str = "model",
) -> VMModel:
"""
- Initializes a VM Model for an R model
-
- R models must be saved to disk and the filetype depends on the model type...
- Currently we support the following model types:
-
- - LogisticRegression `glm` model in R: saved as an RDS file with `saveRDS`
- - LinearRegression `lm` model in R: saved as an RDS file with `saveRDS`
- - XGBClassifier: saved as a .json or .bin file with `xgb.save`
- - XGBRegressor: saved as a .json or .bin file with `xgb.save`
+ Initialize a VM Model from an R model.
LogisticRegression and LinearRegression models are converted to sklearn models by extracting
the coefficients and intercept from the R model. XGB models are loaded using the xgboost
- since xgb models saved in .json or .bin format can be loaded directly with either Python or R
+ since xgb models saved in .json or .bin format can be loaded directly with either Python or R.
Args:
- model_path (str): The path to the R model saved as an RDS or XGB file
- model_type (str): The type of the model (one of R_MODEL_TYPES)
+ model_path (str): The path to the R model saved as an RDS or XGB file.
+ input_id (str): The input ID for the model. Defaults to "model".
Returns:
- vm.vm.Model: A VM Model instance
+ VMModel: A VM Model instance.
"""
# TODO: proper check for supported models
@@ -329,12 +318,12 @@ def init_r_model(
def get_test_suite(
- test_suite_id: str = None,
- section: str = None,
- *args,
- **kwargs,
+ test_suite_id: Optional[str] = None,
+ section: Optional[str] = None,
+ *args: Any,
+ **kwargs: Any,
) -> TestSuite:
- """Gets a TestSuite object for the current project or a specific test suite
+ """Gets a TestSuite object for the current project or a specific test suite.
This function provides an interface to retrieve the TestSuite instance for the
current project or a specific TestSuite instance identified by test_suite_id.
@@ -348,8 +337,11 @@ def get_test_suite(
section (str, optional): The section of the documentation template from which
to retrieve the test suite. This only applies if test_suite_id is None.
Defaults to None.
- args: Additional arguments to pass to the TestSuite
- kwargs: Additional keyword arguments to pass to the TestSuite
+ args: Additional arguments to pass to the TestSuite.
+ kwargs: Additional keyword arguments to pass to the TestSuite.
+
+ Returns:
+ TestSuite: The TestSuite instance.
"""
if test_suite_id is None:
if client_config.documentation_template is None:
@@ -365,31 +357,36 @@ def get_test_suite(
def run_test_suite(
- test_suite_id, send=True, fail_fast=False, config=None, inputs=None, **kwargs
-):
- """High Level function for running a test suite
+ test_suite_id: str,
+ send: bool = True,
+ fail_fast: bool = False,
+ config: Optional[Dict[str, Any]] = None,
+ inputs: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+) -> TestSuite:
+ """High Level function for running a test suite.
This function provides a high level interface for running a test suite. A test suite is
a collection of tests. This function will automatically find the correct test suite
class based on the test_suite_id, initialize each of the tests, and run them.
Args:
- test_suite_id (str): The test suite name (e.g. 'classifier_full_suite')
+ test_suite_id (str): The test suite name. For example, 'classifier_full_suite'.
config (dict, optional): A dictionary of parameters to pass to the tests in the
test suite. Defaults to None.
send (bool, optional): Whether to post the test results to the API. send=False
is useful for testing. Defaults to True.
fail_fast (bool, optional): Whether to stop running tests after the first failure. Defaults to False.
- inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite e.g. `model`, `dataset`
- `models` etc. These inputs will be accessible by any test in the test suite. See the test
- documentation or `vm.describe_test()` for more details on the inputs required for each.
- **kwargs: backwards compatibility for passing in test inputs using keyword arguments
+ inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite, such as `model`, `dataset`
+ `models`, etc. These inputs will be accessible by any test in the test suite. See the test
+ documentation or `vm.describe_test()` for more details on the inputs required for each. Defaults to None.
+ **kwargs: backwards compatibility for passing in test inputs using keyword arguments.
Raises:
- ValueError: If the test suite name is not found or if there is an error initializing the test suite
+ ValueError: If the test suite name is not found or if there is an error initializing the test suite.
Returns:
- TestSuite: the TestSuite instance
+ TestSuite: The TestSuite instance.
"""
try:
Suite: TestSuite = get_test_suite_by_id(test_suite_id)
@@ -414,14 +411,14 @@ class based on the test_suite_id, initialize each of the tests, and run them.
return suite
-def preview_template():
- """Preview the documentation template for the current project
+def preview_template() -> None:
+ """Preview the documentation template for the current project.
This function will display the documentation template for the current project. If
the project has not been initialized, then an error will be raised.
Raises:
- ValueError: If the project has not been initialized
+ ValueError: If the project has not been initialized.
"""
if client_config.documentation_template is None:
raise MissingDocumentationTemplate(
@@ -432,9 +429,14 @@ def preview_template():
def run_documentation_tests(
- section=None, send=True, fail_fast=False, inputs=None, config=None, **kwargs
-):
- """Collect and run all the tests associated with a template
+ section: Optional[str] = None,
+ send: bool = True,
+ fail_fast: bool = False,
+ inputs: Optional[Dict[str, Any]] = None,
+ config: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+) -> Union[TestSuite, Dict[str, TestSuite]]:
+ """Collect and run all the tests associated with a template.
This function will analyze the current project's documentation template and collect
all the tests associated with it into a test suite. It will then run the test
@@ -444,15 +446,15 @@ def run_documentation_tests(
section (str or list, optional): The section(s) to preview. Defaults to None.
send (bool, optional): Whether to send the results to the ValidMind API. Defaults to True.
fail_fast (bool, optional): Whether to stop running tests after the first failure. Defaults to False.
- inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite
- config: A dictionary of test parameters to override the defaults
- **kwargs: backwards compatibility for passing in test inputs using keyword arguments
+ inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite.
+ config: A dictionary of test parameters to override the defaults.
+ **kwargs: backwards compatibility for passing in test inputs using keyword arguments.
Returns:
TestSuite or dict: The completed TestSuite instance or a dictionary of TestSuites if section is a list.
Raises:
- ValueError: If the project has not been initialized
+ ValueError: If the project has not been initialized.
"""
if client_config.documentation_template is None:
raise MissingDocumentationTemplate(
@@ -487,24 +489,30 @@ def run_documentation_tests(
def _run_documentation_section(
- template, section, send=True, fail_fast=False, config=None, inputs=None, **kwargs
-):
- """Run all tests in a template section
+ template: str,
+ section: str,
+ send: bool = True,
+ fail_fast: bool = False,
+ config: Optional[Dict[str, Any]] = None,
+ inputs: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+) -> TestSuite:
+ """Run all tests in a template section.
This function will collect all tests used in a template section into a TestSuite and then
run the TestSuite as usual.
Args:
- template: A valid flat template
- section: The section of the template to run (if not provided, run all sections)
- send: Whether to send the results to the ValidMind API
+ template: A valid flat template.
+ section: The section of the template to run (if not provided, run all sections).
+ send: Whether to send the results to the ValidMind API.
fail_fast (bool, optional): Whether to stop running tests after the first failure. Defaults to False.
- config: A dictionary of test parameters to override the defaults
- inputs: A dictionary of test inputs to pass to the TestSuite
- **kwargs: backwards compatibility for passing in test inputs using keyword arguments
+ config: A dictionary of test parameters to override the defaults.
+ inputs: A dictionary of test inputs to pass to the TestSuite.
+ **kwargs: backwards compatibility for passing in test inputs using keyword arguments.
Returns:
- The completed TestSuite instance
+ The completed TestSuite instance.
"""
test_suite = get_template_test_suite(template, section)
diff --git a/validmind/client_config.py b/validmind/client_config.py
index a237d45e7..df11fb5e0 100644
--- a/validmind/client_config.py
+++ b/validmind/client_config.py
@@ -13,7 +13,7 @@
@dataclass
class ClientConfig:
"""
- Configuration class for the ValidMind API client. This is instantiated
+ Configuration class for the ValidMind API client. This class is instantiated
when initializing the API client.
"""
@@ -25,7 +25,7 @@ class ClientConfig:
def __post_init__(self):
"""
- Set additional attributes when initializing the class
+ Set additional attributes when initializing the class.
"""
# check if running on notebook and set running_on_colab
try:
@@ -36,7 +36,7 @@ def __post_init__(self):
self.running_on_colab = False
def can_generate_llm_test_descriptions(self):
- """Returns True if the client can generate LLM based test descriptions"""
+ """Returns True if the client can generate LLM-based test descriptions."""
return self.feature_flags.get("llm_test_descriptions", True)
diff --git a/validmind/datasets/classification/__init__.py b/validmind/datasets/classification/__init__.py
index bea25dd83..b18241295 100644
--- a/validmind/datasets/classification/__init__.py
+++ b/validmind/datasets/classification/__init__.py
@@ -5,6 +5,8 @@
"""
Entrypoint for classification datasets.
"""
+from typing import List
+
import pandas as pd
__all__ = [
@@ -13,7 +15,7 @@
]
-def simple_preprocess_booleans(df, columns):
+def simple_preprocess_booleans(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame:
"""
Preprocess boolean columns.
@@ -36,7 +38,9 @@ def simple_preprocess_booleans(df, columns):
return df
-def simple_preprocess_categoricals(df, columns):
+def simple_preprocess_categoricals(
+ df: pd.DataFrame, columns: List[str]
+) -> pd.DataFrame:
"""
Preprocess categorical columns.
@@ -56,7 +60,7 @@ def simple_preprocess_categoricals(df, columns):
return df
-def simple_preprocess_numericals(df, columns):
+def simple_preprocess_numericals(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame:
"""
Preprocess numerical columns.
diff --git a/validmind/datasets/credit_risk/lending_club.py b/validmind/datasets/credit_risk/lending_club.py
index d6bd535b3..283c4fd22 100644
--- a/validmind/datasets/credit_risk/lending_club.py
+++ b/validmind/datasets/credit_risk/lending_club.py
@@ -5,6 +5,7 @@
import logging
import os
import warnings
+from typing import Any, Dict, Optional, Tuple
import numpy as np
import pandas as pd
@@ -101,12 +102,15 @@
}
-def load_data(source="online", verbose=True):
+def load_data(source: str = "online", verbose: bool = True) -> pd.DataFrame:
"""
Load data from either an online source or offline files, automatically dropping specified columns for offline data.
- :param source: 'online' for online data, 'offline' for offline files. Defaults to 'online'.
- :return: DataFrame containing the loaded data.
+ Args:
+ source: 'online' for online data, 'offline' for offline files. Defaults to 'online'.
+
+ Returns:
+ DataFrame: DataFrame containing the loaded data.
"""
if source == "online":
@@ -136,7 +140,7 @@ def load_data(source="online", verbose=True):
return df
-def _clean_data(df, verbose=True):
+def _clean_data(df: pd.DataFrame, verbose: bool = True) -> pd.DataFrame:
df = df.copy()
# Drop columns not relevant for application scorecards
@@ -182,7 +186,7 @@ def _clean_data(df, verbose=True):
return df
-def preprocess(df, verbose=True):
+def preprocess(df: pd.DataFrame, verbose: bool = True) -> pd.DataFrame:
df = df.copy()
# Convert the target variable to integer type for modeling.
@@ -245,7 +249,7 @@ def preprocess(df, verbose=True):
return df
-def _preprocess_term(df):
+def _preprocess_term(df: pd.DataFrame) -> pd.DataFrame:
df = df.copy()
# Remove ' months' and convert to integer
@@ -254,7 +258,7 @@ def _preprocess_term(df):
return df
-def _preprocess_emp_length(df):
+def _preprocess_emp_length(df: pd.DataFrame) -> pd.DataFrame:
df = df.copy()
# Mapping string values to numbers
@@ -281,7 +285,7 @@ def _preprocess_emp_length(df):
return df
-def feature_engineering(df, verbose=True):
+def feature_engineering(df: pd.DataFrame, verbose: bool = True) -> pd.DataFrame:
df = df.copy()
# WoE encoding of numerical and categorical features
@@ -295,7 +299,7 @@ def feature_engineering(df, verbose=True):
return df
-def woe_encoding(df, verbose=True):
+def woe_encoding(df: pd.DataFrame, verbose: bool = True) -> pd.DataFrame:
df = df.copy()
woe = _woebin(df, verbose=verbose)
@@ -316,7 +320,7 @@ def woe_encoding(df, verbose=True):
return df
-def _woe_to_bins(woe):
+def _woe_to_bins(woe: Dict[str, Any]) -> Dict[str, Any]:
# Select and rename columns
transformed_df = woe[
[
@@ -350,7 +354,7 @@ def _woe_to_bins(woe):
return bins
-def _woebin(df, verbose=True):
+def _woebin(df: pd.DataFrame, verbose: bool = True) -> Dict[str, Any]:
"""
This function performs automatic binning using WoE.
df: A pandas dataframe
@@ -380,7 +384,13 @@ def _woebin(df, verbose=True):
return bins_df
-def split(df, validation_size=None, test_size=0.2, add_constant=False, verbose=True):
+def split(
+ df: pd.DataFrame,
+ validation_split: Optional[float] = None,
+ test_size: float = 0.2,
+ add_constant: bool = False,
+ verbose: bool = True,
+) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
"""
Split dataset into train, validation (optional), and test sets.
@@ -404,7 +414,7 @@ def split(df, validation_size=None, test_size=0.2, add_constant=False, verbose=T
if add_constant:
test_df = sm.add_constant(test_df)
- if validation_size is None:
+ if validation_split is None:
if add_constant:
train_val_df = sm.add_constant(train_val_df)
@@ -423,7 +433,7 @@ def split(df, validation_size=None, test_size=0.2, add_constant=False, verbose=T
return train_val_df, test_df
# Calculate validation size as proportion of remaining data
- val_size = validation_size / (1 - test_size)
+ val_size = validation_split / (1 - test_size)
train_df, validation_df = train_test_split(
train_val_df, test_size=val_size, random_state=42
)
@@ -451,7 +461,7 @@ def split(df, validation_size=None, test_size=0.2, add_constant=False, verbose=T
return train_df, validation_df, test_df
-def compute_scores(probabilities):
+def compute_scores(probabilities: np.ndarray) -> np.ndarray:
target_score = score_params["target_score"]
target_odds = score_params["target_odds"]
pdo = score_params["pdo"]
@@ -465,7 +475,9 @@ def compute_scores(probabilities):
return scores
-def get_demo_test_config(x_test=None, y_test=None):
+def get_demo_test_config(
+ x_test: Optional[np.ndarray] = None, y_test: Optional[np.ndarray] = None
+) -> Dict[str, Any]:
"""Get demo test configuration.
Args:
diff --git a/validmind/datasets/nlp/cnn_dailymail.py b/validmind/datasets/nlp/cnn_dailymail.py
index 2dc021a6f..80ced3ef8 100644
--- a/validmind/datasets/nlp/cnn_dailymail.py
+++ b/validmind/datasets/nlp/cnn_dailymail.py
@@ -4,6 +4,7 @@
import os
import textwrap
+from typing import Optional, Tuple
import pandas as pd
from datasets import load_dataset
@@ -22,13 +23,18 @@
dataset_path = os.path.join(current_path, "datasets")
-def load_data(source="online", dataset_size=None):
+def load_data(
+ source: str = "online", dataset_size: Optional[str] = None
+) -> Tuple[pd.DataFrame, pd.DataFrame]:
"""
Load data from either online source or offline files.
- :param source: 'online' for online data, 'offline' for offline data. Defaults to 'online'.
- :param dataset_size: Applicable if source is 'offline'. '300k' or '500k' for dataset size. Defaults to None.
- :return: DataFrame containing the loaded data.
+ Args:
+ source: 'online' for online data, 'offline' for offline data. Defaults to 'online'.
+ dataset_size: Applicable if source is 'offline'. '300k' or '500k' for dataset size. Defaults to None.
+
+ Returns:
+ Tuple containing (train_df, test_df) DataFrames with the loaded data.
"""
if source == "online":
# Load online data without predictions
diff --git a/validmind/datasets/regression/__init__.py b/validmind/datasets/regression/__init__.py
index f4d7f99c6..110fd7199 100644
--- a/validmind/datasets/regression/__init__.py
+++ b/validmind/datasets/regression/__init__.py
@@ -5,20 +5,25 @@
"""
Entrypoint for regression datasets
"""
+from typing import List
+
import pandas as pd
-__all__ = [
+__all__: List[str] = [
"fred",
"lending_club",
]
-def identify_frequencies(df):
+def identify_frequencies(df: pd.DataFrame) -> pd.DataFrame:
"""
Identify the frequency of each series in the DataFrame.
- :param df: Time-series DataFrame
- :return: DataFrame with two columns: 'Variable' and 'Frequency'
+ Args:
+ df: Time-series DataFrame.
+
+ Returns:
+ DataFrame with two columns: "Variable" and "Frequency".
"""
frequencies = []
for column in df.columns:
@@ -36,7 +41,19 @@ def identify_frequencies(df):
return freq_df
-def resample_to_common_frequency(df, common_frequency="MS"):
+def resample_to_common_frequency(
+ df: pd.DataFrame, common_frequency: str = "MS"
+) -> pd.DataFrame:
+ """
+ Resample time series data to a common frequency.
+
+ Args:
+ df: Time-series DataFrame.
+ common_frequency: Target frequency for resampling. Defaults to "MS" (month start).
+
+ Returns:
+ DataFrame with data resampled to the common frequency.
+ """
# Make sure the index is a datetime index
if not isinstance(df.index, pd.DatetimeIndex):
df.index = pd.to_datetime(df.index)
diff --git a/validmind/errors.py b/validmind/errors.py
index a7db65c49..60556abab 100644
--- a/validmind/errors.py
+++ b/validmind/errors.py
@@ -15,6 +15,8 @@
class BaseError(Exception):
+ """Common base class for all non-exit exceptions."""
+
def __init__(self, message=""):
self.message = message
super().__init__(self.message)
@@ -52,7 +54,7 @@ class MissingCacheResultsArgumentsError(BaseError):
class MissingOrInvalidModelPredictFnError(BaseError):
"""
- When the pytorch model is missing a predict function or its predict
+ When the PyTorch model is missing a predict function or its predict
method does not have the expected arguments.
"""
@@ -71,7 +73,7 @@ class InvalidAPICredentialsError(APIRequestError):
def description(self, *args, **kwargs):
return (
self.message
- or "Invalid API credentials. Please ensure that you have provided the correct values for api_key and api_secret."
+ or "Invalid API credentials. Please ensure that you have provided the correct values for API_KEY and API_SECRET."
)
@@ -115,7 +117,7 @@ class InvalidTestResultsError(APIRequestError):
class InvalidTestParametersError(BaseError):
"""
- When an invalid parameters for the test.
+ When invalid parameters are provided for the test.
"""
pass
@@ -123,7 +125,15 @@ class InvalidTestParametersError(BaseError):
class InvalidInputError(BaseError):
"""
- When an invalid input object.
+ When an invalid input object is provided.
+ """
+
+ pass
+
+
+class InvalidParameterError(BaseError):
+ """
+ When an invalid parameter is provided.
"""
pass
@@ -131,7 +141,7 @@ class InvalidInputError(BaseError):
class InvalidTextObjectError(APIRequestError):
"""
- When an invalid Metadat (Text) object is sent to the API.
+ When an invalid Metadata (Text) object is sent to the API.
"""
pass
@@ -155,7 +165,7 @@ class InvalidXGBoostTrainedModelError(BaseError):
class LoadTestError(BaseError):
"""
- Exception raised when an error occurs while loading a test
+ Exception raised when an error occurs while loading a test.
"""
def __init__(self, message: str, original_error: Optional[Exception] = None):
@@ -323,7 +333,7 @@ class SkipTestError(BaseError):
def raise_api_error(error_string):
"""
Safely try to parse JSON from the response message in case the API
- returns a non-JSON string or if the API returns a non-standard error
+ returns a non-JSON string or if the API returns a non-standard error.
"""
try:
json_response = json.loads(error_string)
diff --git a/validmind/input_registry.py b/validmind/input_registry.py
index f54034abc..5c92ca306 100644
--- a/validmind/input_registry.py
+++ b/validmind/input_registry.py
@@ -29,7 +29,7 @@ def get(self, key):
if not input_obj:
raise InvalidInputError(
f"There's no such input with given ID '{key}'. "
- "Please pass valid input ID"
+ "Please pass valid input ID."
)
return input_obj
diff --git a/validmind/logging.py b/validmind/logging.py
index 15c16c936..1cb81ec73 100644
--- a/validmind/logging.py
+++ b/validmind/logging.py
@@ -2,11 +2,12 @@
# See the LICENSE file in the root of this repository for details.
# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-"""ValidMind logging module."""
+"""ValidMind logging module"""
import logging
import os
import time
+from typing import Any, Awaitable, Callable, Dict, Optional, TypeVar
import sentry_sdk
from sentry_sdk.utils import event_from_exception, exc_info_from_error
@@ -16,8 +17,8 @@
__dsn = "https://48f446843657444aa1e2c0d716ef864b@o1241367.ingest.sentry.io/4505239625465856"
-def _get_log_level():
- """Get the log level from the environment variable"""
+def _get_log_level() -> int:
+ """Get the log level from the environment variable."""
log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
if log_level_str not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]:
@@ -26,8 +27,10 @@ def _get_log_level():
return logging.getLevelName(log_level_str)
-def get_logger(name="validmind", log_level=None):
- """Get a logger for the given module name"""
+def get_logger(
+ name: str = "validmind", log_level: Optional[int] = None
+) -> logging.Logger:
+ """Get a logger for the given module name."""
formatter = logging.Formatter(
fmt="%(asctime)s - %(levelname)s(%(name)s): %(message)s"
)
@@ -52,18 +55,21 @@ def get_logger(name="validmind", log_level=None):
return logger
-def init_sentry(server_config):
- """Initialize Sentry SDK for sending logs back to ValidMind
+def init_sentry(server_config: Dict[str, Any]) -> None:
+ """Initialize Sentry SDK for sending logs back to ValidMind.
- This will usually only be called by the api_client module to initialize the
- sentry connection after the user calls `validmind.init()`. This is because the DSN
+ This will usually only be called by the API client module to initialize the
+ Sentry connection after the user calls `validmind.init()`. This is because the DSN
and other config options will be returned by the API.
Args:
- config (dict): The config dictionary returned by the API
- - send_logs (bool): Whether to send logs to Sentry (gets removed)
- - dsn (str): The Sentry DSN
- ...: Other config options for Sentry
+ server_config (Dict[str, Any]): The config dictionary returned by the API.
+ - send_logs (bool): Whether to send logs to Sentry (gets removed).
+ - dsn (str): The Sentry DSN.
+ ...: Other config options for Sentry.
+
+ Returns:
+ None.
"""
if os.getenv("VM_NO_TELEMETRY", False):
return
@@ -88,19 +94,27 @@ def init_sentry(server_config):
logger.debug(f"Sentry error: {str(e)}")
-def log_performance(name=None, logger=None, force=False):
- """Decorator to log the time it takes to run a function
+F = TypeVar("F", bound=Callable[..., Any])
+AF = TypeVar("AF", bound=Callable[..., Awaitable[Any]])
+
+
+def log_performance(
+ name: Optional[str] = None,
+ logger: Optional[logging.Logger] = None,
+ force: bool = False,
+) -> Callable[[F], F]:
+ """Decorator to log the time it takes to run a function.
Args:
name (str, optional): The name of the function. Defaults to None.
logger (logging.Logger, optional): The logger to use. Defaults to None.
- force (bool, optional): Whether to force logging even if env var is off
+ force (bool, optional): Whether to force logging even if env var is off.
Returns:
- function: The decorated function
+ Callable: The decorated function.
"""
- def decorator(func):
+ def decorator(func: F) -> F:
# check if log level is set to debug
if _get_log_level() != logging.DEBUG and not force:
return func
@@ -113,7 +127,7 @@ def decorator(func):
if name is None:
name = func.__name__
- def wrapped(*args, **kwargs):
+ def wrapped(*args: Any, **kwargs: Any) -> Any:
time1 = time.perf_counter()
return_val = func(*args, **kwargs)
time2 = time.perf_counter()
@@ -127,18 +141,13 @@ def wrapped(*args, **kwargs):
return decorator
-async def log_performance_async(func, name=None, logger=None, force=False):
- """Decorator to log the time it takes to run an async function
-
- Args:
- func (function): The function to decorate
- name (str, optional): The name of the function. Defaults to None.
- logger (logging.Logger, optional): The logger to use. Defaults to None.
- force (bool, optional): Whether to force logging even if env var is off
-
- Returns:
- function: The decorated function
- """
+async def log_performance_async(
+ func: AF,
+ name: Optional[str] = None,
+ logger: Optional[logging.Logger] = None,
+ force: bool = False,
+) -> AF:
+ """Async version of log_performance decorator"""
# check if log level is set to debug
if _get_log_level() != logging.DEBUG and not force:
return func
@@ -149,7 +158,7 @@ async def log_performance_async(func, name=None, logger=None, force=False):
if name is None:
name = func.__name__
- async def wrap(*args, **kwargs):
+ async def wrap(*args: Any, **kwargs: Any) -> Any:
time1 = time.perf_counter()
return_val = await func(*args, **kwargs)
time2 = time.perf_counter()
@@ -161,11 +170,11 @@ async def wrap(*args, **kwargs):
return wrap
-def send_single_error(error: Exception):
- """Send a single error to Sentry
+def send_single_error(error: Exception) -> None:
+ """Send a single error to Sentry.
Args:
- error (Exception): The exception to send
+ error (Exception): The exception to send.
"""
event, hint = event_from_exception(exc_info_from_error(error))
client = sentry_sdk.Client(__dsn, release=f"validmind-python@{__version__}")
diff --git a/validmind/models/foundation.py b/validmind/models/foundation.py
index 7ef694887..2b4979ecc 100644
--- a/validmind/models/foundation.py
+++ b/validmind/models/foundation.py
@@ -26,9 +26,9 @@ class FoundationModel(FunctionModel):
Attributes:
predict_fn (callable): The predict function that should take a prompt as input
- and return the result from the model
+ and return the result from the model
prompt (Prompt): The prompt object that defines the prompt template and the
- variables (if any)
+ variables (if any)
name (str, optional): The name of the model. Defaults to name of the predict_fn
"""
diff --git a/validmind/models/function.py b/validmind/models/function.py
index d373b3b16..a8c6067a1 100644
--- a/validmind/models/function.py
+++ b/validmind/models/function.py
@@ -2,6 +2,8 @@
# See the LICENSE file in the root of this repository for details.
# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+from typing import Any, Dict, List
+
from validmind.vm_models.model import VMModel
@@ -18,7 +20,12 @@ def __setitem__(self, key, value):
def __delitem__(self, _):
raise TypeError("Cannot delete keys from Input")
- def get_new(self):
+ def get_new(self) -> Dict[str, Any]:
+ """Get the newly added key-value pairs.
+
+ Returns:
+ Dict[str, Any]: Dictionary containing only the newly added key-value pairs.
+ """
return {k: self[k] for k in self._new}
@@ -41,13 +48,13 @@ def __post_init__(self):
self.name = self.name or self.predict_fn.__name__
- def predict(self, X):
+ def predict(self, X) -> List[Any]:
"""Compute predictions for the input (X)
Args:
X (pandas.DataFrame): The input features to predict on
Returns:
- list: The predictions
+ List[Any]: The predictions
"""
return [self.predict_fn(x) for x in X.to_dict(orient="records")]
diff --git a/validmind/template.py b/validmind/template.py
index 757c9e962..315b9449a 100644
--- a/validmind/template.py
+++ b/validmind/template.py
@@ -2,7 +2,9 @@
# See the LICENSE file in the root of this repository for details.
# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-from ipywidgets import HTML, Accordion, VBox
+from typing import Any, Dict, List, Optional, Type, Union
+
+from ipywidgets import HTML, Accordion, VBox, Widget
from .html_templates.content_blocks import (
failed_content_block_html,
@@ -29,8 +31,10 @@
def _convert_sections_to_section_tree(
- sections, parent_id="_root_", start_section_id=None
-):
+ sections: List[Dict[str, Any]],
+ parent_id: str = "_root_",
+ start_section_id: Optional[str] = None,
+) -> List[Dict[str, Any]]:
section_tree = []
for section in sections:
@@ -53,7 +57,7 @@ def _convert_sections_to_section_tree(
return sorted(section_tree, key=lambda x: x.get("order", 0))
-def _create_content_widget(content):
+def _create_content_widget(content: Dict[str, Any]) -> Widget:
content_type = CONTENT_TYPE_MAP[content["content_type"]]
if content["content_type"] not in ["metric", "test"]:
@@ -75,7 +79,9 @@ def _create_content_widget(content):
)
-def _create_sub_section_widget(sub_sections, section_number):
+def _create_sub_section_widget(
+ sub_sections: List[Dict[str, Any]], section_number: str
+) -> Union[HTML, Accordion]:
if not sub_sections:
return HTML("
Empty Section
")
@@ -111,7 +117,7 @@ def _create_sub_section_widget(sub_sections, section_number):
return accordion
-def _create_section_widget(tree):
+def _create_section_widget(tree: List[Dict[str, Any]]) -> Accordion:
widget = Accordion()
for i, section in enumerate(tree):
sub_widget = None
@@ -139,11 +145,11 @@ def _create_section_widget(tree):
return widget
-def preview_template(template):
- """Preview a template in Jupyter Notebook
+def preview_template(template: str) -> None:
+ """Preview a template in Jupyter Notebook.
Args:
- template (dict): The template to preview
+ template (dict): The template to preview.
"""
if not is_notebook():
logger.warning("preview_template() only works in Jupyter Notebook")
@@ -154,7 +160,7 @@ def preview_template(template):
)
-def _get_section_tests(section):
+def _get_section_tests(section: Dict[str, Any]) -> List[str]:
"""
Get all the tests in a section and its subsections.
@@ -179,15 +185,15 @@ def _get_section_tests(section):
return tests
-def _create_test_suite_section(section):
+def _create_test_suite_section(section: Dict[str, Any]) -> Dict[str, Any]:
"""Create a section object for a test suite that contains the tests in a section
- in the template
+ in the template.
Args:
- section: a section of a template (in tree form)
+ section: A section of a template (in tree form).
Returns:
- A TestSuite section dict
+ A TestSuite section dict.
"""
if section_tests := _get_section_tests(section):
return {
@@ -197,16 +203,18 @@ def _create_test_suite_section(section):
}
-def _create_template_test_suite(template, section=None):
+def _create_template_test_suite(
+ template: str, section: Optional[str] = None
+) -> Type[TestSuite]:
"""
Create and run a test suite from a template.
Args:
- template: A valid flat template
- section: The section of the template to run (if not provided, run all sections)
+ template: A valid flat template.
+ section: The section of the template to run. Runs all sections if not provided.
Returns:
- A dynamically-create TestSuite Class
+ A dynamically-created TestSuite Class.
"""
section_tree = _convert_sections_to_section_tree(
sections=template["sections"],
@@ -229,17 +237,17 @@ def _create_template_test_suite(template, section=None):
)
-def get_template_test_suite(template, section=None):
- """Get a TestSuite instance containing all tests in a template
+def get_template_test_suite(template: str, section: Optional[str] = None) -> TestSuite:
+ """Get a TestSuite instance containing all tests in a template.
This function will collect all tests used in a template into a dynamically-created
- TestSuite object
+ TestSuite object.
Args:
template: A valid flat template
section: The section of the template to run (if not provided, run all sections)
Returns:
- The TestSuite instance
+ The TestSuite instance.
"""
return _create_template_test_suite(template, section)()
diff --git a/validmind/test_suites/__init__.py b/validmind/test_suites/__init__.py
index 0c4b3adae..cd09d3968 100644
--- a/validmind/test_suites/__init__.py
+++ b/validmind/test_suites/__init__.py
@@ -141,7 +141,7 @@ def list_suites(pretty: bool = True):
return format_dataframe(pd.DataFrame(table))
-def describe_suite(test_suite_id: str, verbose=False):
+def describe_suite(test_suite_id: str, verbose: bool = False) -> pd.DataFrame:
"""
Describes a Test Suite by ID
@@ -150,7 +150,7 @@ def describe_suite(test_suite_id: str, verbose=False):
verbose: If True, describe all plans and tests in the Test Suite
Returns:
- pandas.DataFrame: A formatted table with the Test Suite description
+ pd.DataFrame: A formatted table with the Test Suite description
"""
test_suite = get_by_id(test_suite_id)
diff --git a/validmind/tests/_store.py b/validmind/tests/_store.py
index c0da5179e..569094d6f 100644
--- a/validmind/tests/_store.py
+++ b/validmind/tests/_store.py
@@ -5,6 +5,8 @@
"""Module for storing loaded tests and test providers"""
+from typing import Any, Callable, Optional
+
from .test_providers import TestProvider, ValidMindTestProvider
@@ -65,19 +67,26 @@ class TestStore:
def __init__(self):
self.tests = {}
- def get_test(self, test_id: str):
+ def get_test(self, test_id: str) -> Optional[Callable[..., Any]]:
"""Get a test by test ID
Args:
test_id (str): The test ID
Returns:
- object: The test class or function
+ Optional[Callable[..., Any]]: The test function if found, None otherwise
"""
return self.tests.get(test_id)
- def register_test(self, test_id: str, test: object = None):
- """Register a test"""
+ def register_test(
+ self, test_id: str, test: Optional[Callable[..., Any]] = None
+ ) -> None:
+ """Register a test
+
+ Args:
+ test_id (str): The test ID
+ test (Optional[Callable[..., Any]], optional): The test function. Defaults to None.
+ """
self.tests[test_id] = test
diff --git a/validmind/tests/comparison.py b/validmind/tests/comparison.py
index 6f94f8865..d1d167047 100644
--- a/validmind/tests/comparison.py
+++ b/validmind/tests/comparison.py
@@ -146,7 +146,9 @@ def _combine_tables(results: List[TestResult]) -> List[pd.DataFrame]:
return [_combine_single_table(results, i) for i in range(len(results[0].tables))]
-def _build_input_param_string(result: TestResult, results: List[TestResult]) -> str:
+def _build_input_param_string(
+ result: TestResult, results: List[TestResult], show_params: bool
+) -> str:
"""Build a string repr of unique inputs + params for a figure title"""
parts = []
unique_inputs = _get_unique_inputs(results)
@@ -162,19 +164,29 @@ def _build_input_param_string(result: TestResult, results: List[TestResult]) ->
input_val = _get_input_key(input_obj)
parts.append(f"{input_name}={input_val}")
- # TODO: revisit this when we can create a value/title to show for params
- # unique_params = _get_unique_params(results)
- # # if theres only one unique value for a param, don't show it
- # # however, if there is only one unique value for all params then show it as
- # # long as there is no existing inputs in the parts list
- # if result.params:
- # should_show = (
- # all(len(unique_params[param_name]) == 1 for param_name in unique_params)
- # and not parts
- # )
- # for param_name, param_value in result.params.items():
- # if should_show or len(unique_params[param_name]) > 1:
- # parts.append(f"{param_name}={param_value}")
+ # Handle params if show_params is enabled
+ if show_params and result.params:
+ unique_params = _get_unique_params(results)
+ # If there's only one unique value for a param, don't show it
+ # unless there is only one unique value for all params and no inputs shown
+ should_show = (
+ all(len(unique_params[param_name]) == 1 for param_name in unique_params)
+ and not parts
+ )
+ for param_name, param_value in result.params.items():
+ if should_show or len(unique_params[param_name]) > 1:
+ # Convert the param_value to a string representation
+ if isinstance(param_value, list):
+ # For lists, join elements with commas
+ str_value = ",".join(str(v) for v in param_value)
+ elif hasattr(param_value, "__str__"):
+ # Use string representation if available
+ str_value = str(param_value)
+ else:
+ # Default fallback
+ str_value = repr(param_value)
+
+ parts.append(f"{param_name}={str_value}")
return ", ".join(parts)
@@ -207,7 +219,7 @@ def _update_figure_title(figure: Any, input_param_str: str) -> None:
raise ValueError(f"Unsupported figure type: {type(figure)}")
-def _combine_figures(results: List[TestResult]) -> List[Any]:
+def _combine_figures(results: List[TestResult], show_params: bool) -> List[Any]:
"""Combine figures from multiple test results (gets raw figure objects, not vm Figures)"""
combined_figures = []
@@ -216,7 +228,7 @@ def _combine_figures(results: List[TestResult]) -> List[Any]:
# update the figure object in-place with the new title
_update_figure_title(
figure=figure.figure,
- input_param_str=_build_input_param_string(result, results),
+ input_param_str=_build_input_param_string(result, results, show_params),
)
combined_figures.append(figure)
@@ -279,35 +291,53 @@ def get_comparison_test_configs(
A list of test configurations.
"""
- # Convert list of dicts to dict of lists if necessary
+ # Convert list of dicts to dict of lists if necessary for input_grid
def list_to_dict(grid_list):
return {k: [d[k] for d in grid_list] for k in grid_list[0].keys()}
+ # Handle input_grid the same way as before
if isinstance(input_grid, list):
input_grid = list_to_dict(input_grid)
- if isinstance(param_grid, list):
- param_grid = list_to_dict(param_grid)
-
test_configs = []
- if input_grid and param_grid:
- input_combinations = _cartesian_product(input_grid)
- param_combinations = _cartesian_product(param_grid)
- test_configs = [
- {"inputs": i, "params": p}
- for i, p in product(input_combinations, param_combinations)
- ]
+ # Check if param_grid is a list of dictionaries
+ is_param_grid_list = isinstance(param_grid, list)
+
+ # Special handling for list-based param_grid
+ if is_param_grid_list:
+ if input_grid:
+ # Generate all combinations of input_grid and each param dictionary
+ input_combinations = _cartesian_product(input_grid)
+ test_configs = [
+ {"inputs": i, "params": p}
+ for i in input_combinations
+ for p in param_grid
+ ]
+ else:
+ # Each dictionary in param_grid is a specific test configuration
+ test_configs = [{"inputs": inputs or {}, "params": p} for p in param_grid]
+
+ # Dictionary-based param_grid
+ elif param_grid:
+ if input_grid:
+ input_combinations = _cartesian_product(input_grid)
+ param_combinations = _cartesian_product(param_grid)
+ test_configs = [
+ {"inputs": i, "params": p}
+ for i, p in product(input_combinations, param_combinations)
+ ]
+ else:
+ param_combinations = _cartesian_product(param_grid)
+ test_configs = [
+ {"inputs": inputs or {}, "params": p} for p in param_combinations
+ ]
+ # Just input_grid, no param_grid
elif input_grid:
input_combinations = _cartesian_product(input_grid)
test_configs = [
{"inputs": i, "params": params or {}} for i in input_combinations
]
- elif param_grid:
- param_combinations = _cartesian_product(param_grid)
- test_configs = [
- {"inputs": inputs or {}, "params": p} for p in param_combinations
- ]
return test_configs
@@ -333,12 +363,14 @@ def _combine_raw_data(results: List[TestResult]) -> RawData:
def combine_results(
results: List[TestResult],
+ show_params: bool,
) -> Tuple[List[Any], Dict[str, List[Any]], Dict[str, List[Any]]]:
"""
Combine multiple test results into a single set of outputs.
Args:
results: A list of TestResult objects to combine.
+ show_params: Whether to show parameter values in figure titles.
Returns:
A tuple containing:
@@ -353,7 +385,7 @@ def combine_results(
# handle tables (if any)
combined_outputs.extend(_combine_tables(results))
# handle figures (if any)
- combined_outputs.extend(_combine_figures(results))
+ combined_outputs.extend(_combine_figures(results, show_params))
# handle threshold tests (i.e. tests that have pass/fail bool status)
if results[0].passed is not None:
combined_outputs.append(all(result.passed for result in results))
diff --git a/validmind/tests/data_validation/ClassImbalance.py b/validmind/tests/data_validation/ClassImbalance.py
index bab192e74..14d4eacf3 100644
--- a/validmind/tests/data_validation/ClassImbalance.py
+++ b/validmind/tests/data_validation/ClassImbalance.py
@@ -14,7 +14,9 @@
from validmind.vm_models import VMDataset
-@tags("tabular_data", "binary_classification", "multiclass_classification")
+@tags(
+ "tabular_data", "binary_classification", "multiclass_classification", "data_quality"
+)
@tasks("classification")
def ClassImbalance(
dataset: VMDataset, min_percent_threshold: int = 10
diff --git a/validmind/tests/data_validation/DatasetDescription.py b/validmind/tests/data_validation/DatasetDescription.py
index f8f3d0699..64fe81db7 100644
--- a/validmind/tests/data_validation/DatasetDescription.py
+++ b/validmind/tests/data_validation/DatasetDescription.py
@@ -6,12 +6,10 @@
from collections import Counter
import numpy as np
-from ydata_profiling.config import Settings
-from ydata_profiling.model.typeset import ProfilingTypeSet
from validmind import RawData, tags, tasks
-from validmind.errors import UnsupportedColumnTypeError
from validmind.logging import get_logger
+from validmind.utils import infer_datatypes
from validmind.vm_models import VMDataset
DEFAULT_HISTOGRAM_BINS = 10
@@ -20,25 +18,6 @@
logger = get_logger(__name__)
-def infer_datatypes(df):
- column_type_mappings = {}
- typeset = ProfilingTypeSet(Settings())
- variable_types = typeset.infer_type(df)
-
- for column, type in variable_types.items():
- if str(type) == "Unsupported":
- if df[column].isnull().all():
- column_type_mappings[column] = {"id": column, "type": "Null"}
- else:
- raise UnsupportedColumnTypeError(
- f"Unsupported type for column {column}. Please review all values in this dataset column."
- )
- else:
- column_type_mappings[column] = {"id": column, "type": str(type)}
-
- return list(column_type_mappings.values())
-
-
def get_numerical_histograms(df, column):
"""
Returns a collection of histograms for a numerical column, each one
@@ -50,7 +29,7 @@ def get_numerical_histograms(df, column):
# bins='sturges'. Cannot use 'auto' until we review and fix its performance
# on datasets with too many unique values
#
- # 'sturges': R’s default method, only accounts for data size. Only optimal
+ # 'sturges': R's default method, only accounts for data size. Only optimal
# for gaussian data and underestimates number of bins for large non-gaussian datasets.
default_hist = np.histogram(values_cleaned, bins="sturges")
diff --git a/validmind/tests/data_validation/DescriptiveStatistics.py b/validmind/tests/data_validation/DescriptiveStatistics.py
index 1303a53e5..3e9e929e4 100644
--- a/validmind/tests/data_validation/DescriptiveStatistics.py
+++ b/validmind/tests/data_validation/DescriptiveStatistics.py
@@ -44,7 +44,7 @@ def get_summary_statistics_categorical(df, categorical_fields):
return summary_stats
-@tags("tabular_data", "time_series_data")
+@tags("tabular_data", "time_series_data", "data_quality")
@tasks("classification", "regression")
def DescriptiveStatistics(dataset: VMDataset):
"""
diff --git a/validmind/tests/data_validation/Skewness.py b/validmind/tests/data_validation/Skewness.py
index c472159fd..2c7550f75 100644
--- a/validmind/tests/data_validation/Skewness.py
+++ b/validmind/tests/data_validation/Skewness.py
@@ -2,10 +2,8 @@
# See the LICENSE file in the root of this repository for details.
# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-from ydata_profiling.config import Settings
-from ydata_profiling.model.typeset import ProfilingTypeSet
-
from validmind import tags, tasks
+from validmind.utils import infer_datatypes
@tags("data_quality", "tabular_data")
@@ -49,8 +47,11 @@ def Skewness(dataset, max_threshold=1):
- Subjective threshold for risk grading, requiring expert input and recurrent iterations for refinement.
"""
- typeset = ProfilingTypeSet(Settings())
- dataset_types = typeset.infer_type(dataset.df)
+ # Use the imported infer_datatypes function
+ dataset_types = infer_datatypes(dataset.df)
+
+ # Convert the list of dictionaries to a dictionary for easy access
+ dataset_types_dict = {item["id"]: item["type"] for item in dataset_types}
skewness = dataset.df.skew(numeric_only=True)
@@ -58,7 +59,7 @@ def Skewness(dataset, max_threshold=1):
passed = True
for col in skewness.index:
- if str(dataset_types[col]) != "Numeric":
+ if dataset_types_dict.get(col) != "Numeric":
continue
col_skewness = skewness[col]
diff --git a/validmind/tests/decorator.py b/validmind/tests/decorator.py
index 9ca1af087..26aa78f90 100644
--- a/validmind/tests/decorator.py
+++ b/validmind/tests/decorator.py
@@ -7,6 +7,7 @@
import inspect
import os
from functools import wraps
+from typing import Any, Callable, List, Optional, TypeVar, Union
from validmind.logging import get_logger
@@ -15,8 +16,10 @@
logger = get_logger(__name__)
+F = TypeVar("F", bound=Callable[..., Any])
-def _get_save_func(func, test_id):
+
+def _get_save_func(func: Callable[..., Any], test_id: str) -> Callable[..., None]:
"""Helper function to save a decorated function to a file
Useful when a custom test function has been created inline in a notebook or
@@ -29,7 +32,7 @@ def _get_save_func(func, test_id):
# remove decorator line
source = source.split("\n", 1)[1]
- def save(root_folder=".", imports=None):
+ def save(root_folder: str = ".", imports: Optional[List[str]] = None) -> None:
parts = test_id.split(".")
if len(parts) > 1:
@@ -84,7 +87,7 @@ def save(root_folder=".", imports=None):
return save
-def test(func_or_id):
+def test(func_or_id: Union[Callable[..., Any], str, None]) -> Callable[[F], F]:
"""Decorator for creating and registering custom tests
This decorator registers the function it wraps as a test function within ValidMind
@@ -109,14 +112,14 @@ def test(func_or_id):
as the metric's description.
Args:
- func: The function to decorate
- test_id: The identifier for the metric. If not provided, the function name is used.
+ func_or_id (Union[Callable[..., Any], str, None]): Either the function to decorate
+ or the test ID. If None, the function name is used.
Returns:
- The decorated function.
+ Callable[[F], F]: The decorated function.
"""
- def decorator(func):
+ def decorator(func: F) -> F:
test_id = func_or_id or f"validmind.custom_metrics.{func.__name__}"
test_func = load_test(test_id, func, reload=True)
test_store.register_test(test_id, test_func)
@@ -136,28 +139,28 @@ def decorator(func):
return decorator
-def tasks(*tasks):
+def tasks(*tasks: str) -> Callable[[F], F]:
"""Decorator for specifying the task types that a test is designed for.
Args:
*tasks: The task types that the test is designed for.
"""
- def decorator(func):
+ def decorator(func: F) -> F:
func.__tasks__ = list(tasks)
return func
return decorator
-def tags(*tags):
+def tags(*tags: str) -> Callable[[F], F]:
"""Decorator for specifying tags for a test.
Args:
*tags: The tags to apply to the test.
"""
- def decorator(func):
+ def decorator(func: F) -> F:
func.__tags__ = list(tags)
return func
diff --git a/validmind/tests/load.py b/validmind/tests/load.py
index a1731f27d..697c64219 100644
--- a/validmind/tests/load.py
+++ b/validmind/tests/load.py
@@ -7,7 +7,7 @@
import inspect
import json
from pprint import pformat
-from typing import List
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from uuid import uuid4
import pandas as pd
@@ -32,7 +32,10 @@
}
-def _inspect_signature(test_func: callable):
+def _inspect_signature(
+ test_func: Callable[..., Any],
+) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Dict[str, Any]]]:
+ """Inspect a test function's signature to get inputs and parameters"""
inputs = {}
params = {}
@@ -56,7 +59,9 @@ def _inspect_signature(test_func: callable):
return inputs, params
-def load_test(test_id: str, test_func: callable = None, reload: bool = False):
+def load_test(
+ test_id: str, test_func: Optional[Callable[..., Any]] = None, reload: bool = False
+) -> Callable[..., Any]:
"""Load a test by test ID
Test IDs are in the format `namespace.path_to_module.TestClassOrFuncName[:tag]`.
@@ -67,6 +72,8 @@ def load_test(test_id: str, test_func: callable = None, reload: bool = False):
test_id (str): The test ID in the format `namespace.path_to_module.TestName[:tag]`
test_func (callable, optional): The test function to load. If not provided, the
test will be loaded from the test provider. Defaults to None.
+ reload (bool, optional): If True, reload the test even if it's already loaded.
+ Defaults to False.
"""
# remove tag if present
test_id = test_id.split(":", 1)[0]
@@ -109,7 +116,8 @@ def load_test(test_id: str, test_func: callable = None, reload: bool = False):
return test_store.get_test(test_id)
-def _list_test_ids():
+def _list_test_ids() -> List[str]:
+ """List all available test IDs"""
test_ids = []
for namespace, test_provider in test_provider_store.test_providers.items():
@@ -120,7 +128,7 @@ def _list_test_ids():
return test_ids
-def _load_tests(test_ids):
+def _load_tests(test_ids: List[str]) -> Dict[str, Callable[..., Any]]:
"""Load a set of tests, handling missing dependencies."""
tests = {}
@@ -138,12 +146,12 @@ def _load_tests(test_ids):
logger.debug(str(e))
if e.extra:
- logger.info(
+ logger.debug(
f"Skipping `{test_id}` as it requires extra dependencies: {e.required_dependencies}."
f" Please run `pip install validmind[{e.extra}]` to view and run this test."
)
else:
- logger.info(
+ logger.debug(
f"Skipping `{test_id}` as it requires missing dependencies: {e.required_dependencies}."
" Please install the missing dependencies to view and run this test."
)
@@ -151,7 +159,8 @@ def _load_tests(test_ids):
return tests
-def _test_description(test_description: str, num_lines: int = 5):
+def _test_description(test_description: str, num_lines: int = 5) -> str:
+ """Format a test description"""
description = test_description.strip("\n").strip()
if len(description.split("\n")) > num_lines:
@@ -160,7 +169,10 @@ def _test_description(test_description: str, num_lines: int = 5):
return description
-def _pretty_list_tests(tests, truncate=True):
+def _pretty_list_tests(
+ tests: Dict[str, Callable[..., Any]], truncate: bool = True
+) -> None:
+ """Pretty print a list of tests"""
table = [
{
"ID": test_id,
@@ -171,6 +183,8 @@ def _pretty_list_tests(tests, truncate=True):
),
"Required Inputs": list(test.inputs.keys()),
"Params": test.params,
+ "Tags": test.__tags__,
+ "Tasks": test.__tasks__,
}
for test_id, test in tests.items()
]
@@ -178,10 +192,8 @@ def _pretty_list_tests(tests, truncate=True):
return format_dataframe(pd.DataFrame(table))
-def list_tags():
- """
- List unique tags from all test classes.
- """
+def list_tags() -> List[str]:
+ """List all unique available tags"""
unique_tags = set()
@@ -191,7 +203,7 @@ def list_tags():
return list(unique_tags)
-def list_tasks_and_tags(as_json=False):
+def list_tasks_and_tags(as_json: bool = False) -> Union[str, Dict[str, List[str]]]:
"""
List all task types and their associated tags, with one row per task type and
all tags for a task type in one row.
@@ -218,11 +230,8 @@ def list_tasks_and_tags(as_json=False):
)
-def list_tasks():
- """
- List unique tasks from all test classes.
- """
-
+def list_tasks() -> List[str]:
+ """List all unique available tasks"""
unique_tasks = set()
for test in _load_tests(list_tests(pretty=False)).values():
@@ -231,7 +240,13 @@ def list_tasks():
return list(unique_tasks)
-def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
+def list_tests(
+ filter: Optional[str] = None,
+ task: Optional[str] = None,
+ tags: Optional[List[str]] = None,
+ pretty: bool = True,
+ truncate: bool = True,
+) -> Union[List[str], None]:
"""List all tests in the tests directory.
Args:
@@ -245,9 +260,6 @@ def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
formatted table. Defaults to True.
truncate (bool, optional): If True, truncates the test description to the first
line. Defaults to True. (only used if pretty=True)
-
- Returns:
- list or pandas.DataFrame: A list of all tests or a formatted table.
"""
test_ids = _list_test_ids()
@@ -286,7 +298,9 @@ def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
return _pretty_list_tests(tests, truncate=truncate)
-def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
+def describe_test(
+ test_id: Optional[TestID] = None, raw: bool = False, show: bool = True
+) -> Union[str, HTML, Dict[str, Any]]:
"""Get or show details about the test
This function can be used to see test details including the test name, description,
diff --git a/validmind/tests/model_validation/ragas/AnswerCorrectness.py b/validmind/tests/model_validation/ragas/AnswerCorrectness.py
index e7fdc6309..6352bf990 100644
--- a/validmind/tests/model_validation/ragas/AnswerCorrectness.py
+++ b/validmind/tests/model_validation/ragas/AnswerCorrectness.py
@@ -123,8 +123,10 @@ def AnswerCorrectness(
score_column = "answer_correctness"
- fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
- fig_box = px.box(x=result_df[score_column].to_list())
+ fig_histogram = px.histogram(
+ x=result_df[score_column].to_list(), nbins=10, title="Answer Correctness"
+ )
+ fig_box = px.box(x=result_df[score_column].to_list(), title="Answer Correctness")
return (
{
diff --git a/validmind/tests/model_validation/ragas/ContextEntityRecall.py b/validmind/tests/model_validation/ragas/ContextEntityRecall.py
index 2c3516c70..fa5fb3ae9 100644
--- a/validmind/tests/model_validation/ragas/ContextEntityRecall.py
+++ b/validmind/tests/model_validation/ragas/ContextEntityRecall.py
@@ -118,8 +118,10 @@ def ContextEntityRecall(
score_column = "context_entity_recall"
- fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
- fig_box = px.box(x=result_df[score_column].to_list())
+ fig_histogram = px.histogram(
+ x=result_df[score_column].to_list(), nbins=10, title="Context Entity Recall"
+ )
+ fig_box = px.box(x=result_df[score_column].to_list(), title="Context Entity Recall")
return (
{
diff --git a/validmind/tests/model_validation/ragas/ContextPrecision.py b/validmind/tests/model_validation/ragas/ContextPrecision.py
index 6be615425..035e76f25 100644
--- a/validmind/tests/model_validation/ragas/ContextPrecision.py
+++ b/validmind/tests/model_validation/ragas/ContextPrecision.py
@@ -114,8 +114,10 @@ def ContextPrecision(
score_column = "llm_context_precision_with_reference"
- fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
- fig_box = px.box(x=result_df[score_column].to_list())
+ fig_histogram = px.histogram(
+ x=result_df[score_column].to_list(), nbins=10, title="Context Precision"
+ )
+ fig_box = px.box(x=result_df[score_column].to_list(), title="Context Precision")
return (
{
diff --git a/validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py b/validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py
index 916641589..9b9d18ea5 100644
--- a/validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py
+++ b/validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py
@@ -109,8 +109,10 @@ def ContextPrecisionWithoutReference(
score_column = "llm_context_precision_without_reference"
- fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
- fig_box = px.box(x=result_df[score_column].to_list())
+ fig_histogram = px.histogram(
+ x=result_df[score_column].to_list(), nbins=10, title="Context Precision"
+ )
+ fig_box = px.box(x=result_df[score_column].to_list(), title="Context Precision")
return (
{
diff --git a/validmind/tests/model_validation/ragas/ContextRecall.py b/validmind/tests/model_validation/ragas/ContextRecall.py
index 7503297b8..e6b0317f4 100644
--- a/validmind/tests/model_validation/ragas/ContextRecall.py
+++ b/validmind/tests/model_validation/ragas/ContextRecall.py
@@ -114,8 +114,10 @@ def ContextRecall(
score_column = "context_recall"
- fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
- fig_box = px.box(x=result_df[score_column].to_list())
+ fig_histogram = px.histogram(
+ x=result_df[score_column].to_list(), nbins=10, title="Context Recall"
+ )
+ fig_box = px.box(x=result_df[score_column].to_list(), title="Context Recall")
return (
{
diff --git a/validmind/tests/model_validation/ragas/Faithfulness.py b/validmind/tests/model_validation/ragas/Faithfulness.py
index 989774bdf..034b5fb61 100644
--- a/validmind/tests/model_validation/ragas/Faithfulness.py
+++ b/validmind/tests/model_validation/ragas/Faithfulness.py
@@ -119,8 +119,10 @@ def Faithfulness(
score_column = "faithfulness"
- fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
- fig_box = px.box(x=result_df[score_column].to_list())
+ fig_histogram = px.histogram(
+ x=result_df[score_column].to_list(), nbins=10, title="Faithfulness"
+ )
+ fig_box = px.box(x=result_df[score_column].to_list(), title="Faithfulness")
return (
{
diff --git a/validmind/tests/model_validation/ragas/ResponseRelevancy.py b/validmind/tests/model_validation/ragas/ResponseRelevancy.py
index 6d6bb9f3b..a7eabd1db 100644
--- a/validmind/tests/model_validation/ragas/ResponseRelevancy.py
+++ b/validmind/tests/model_validation/ragas/ResponseRelevancy.py
@@ -133,8 +133,10 @@ def ResponseRelevancy(
score_column = "answer_relevancy"
- fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
- fig_box = px.box(x=result_df[score_column].to_list())
+ fig_histogram = px.histogram(
+ x=result_df[score_column].to_list(), nbins=10, title="Response Relevancy"
+ )
+ fig_box = px.box(x=result_df[score_column].to_list(), title="Response Relevancy")
return (
{
diff --git a/validmind/tests/model_validation/ragas/SemanticSimilarity.py b/validmind/tests/model_validation/ragas/SemanticSimilarity.py
index b4ca3049e..42d62a877 100644
--- a/validmind/tests/model_validation/ragas/SemanticSimilarity.py
+++ b/validmind/tests/model_validation/ragas/SemanticSimilarity.py
@@ -112,8 +112,10 @@ def SemanticSimilarity(
score_column = "semantic_similarity"
- fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
- fig_box = px.box(x=result_df[score_column].to_list())
+ fig_histogram = px.histogram(
+ x=result_df[score_column].to_list(), nbins=10, title="Semantic Similarity"
+ )
+ fig_box = px.box(x=result_df[score_column].to_list(), title="Semantic Similarity")
return (
{
diff --git a/validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py b/validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py
index a8c96c72f..73edf7044 100644
--- a/validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py
+++ b/validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py
@@ -2,6 +2,8 @@
# See the LICENSE file in the root of this repository for details.
# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+from typing import Dict, List, Optional, Union
+
import numpy as np
import pandas as pd
import plotly.graph_objects as go
@@ -12,7 +14,12 @@
from validmind.vm_models import VMDataset, VMModel
-def find_optimal_threshold(y_true, y_prob, method="youden", target_recall=None):
+def find_optimal_threshold(
+ y_true: np.ndarray,
+ y_prob: np.ndarray,
+ method: str = "youden",
+ target_recall: Optional[float] = None,
+) -> Dict[str, Union[str, float]]:
"""
Find the optimal classification threshold using various methods.
@@ -80,8 +87,11 @@ def find_optimal_threshold(y_true, y_prob, method="youden", target_recall=None):
@tags("model_validation", "threshold_optimization", "classification_metrics")
@tasks("classification")
def ClassifierThresholdOptimization(
- dataset: VMDataset, model: VMModel, methods=None, target_recall=None
-):
+ dataset: VMDataset,
+ model: VMModel,
+ methods: Optional[List[str]] = None,
+ target_recall: Optional[float] = None,
+) -> Dict[str, Union[pd.DataFrame, go.Figure]]:
"""
Analyzes and visualizes different threshold optimization methods for binary classification models.
diff --git a/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py b/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py
index 759024718..0ef87f5f2 100644
--- a/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py
+++ b/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py
@@ -73,6 +73,7 @@ def _prepare_results(
columns={"shape": "training records", f"{metric}": f"training {metric}"},
inplace=True,
)
+ results["test records"] = results_test["shape"]
results[f"test {metric}"] = results_test[metric]
# Adjust gap calculation based on metric directionality
@@ -292,7 +293,8 @@ def OverfitDiagnosis(
{
"Feature": feature_column,
"Slice": row["slice"],
- "Number of Records": row["training records"],
+ "Number of Training Records": row["training records"],
+ "Number of Test Records": row["test records"],
f"Training {metric.upper()}": row[f"training {metric}"],
f"Test {metric.upper()}": row[f"test {metric}"],
"Gap": row["gap"],
diff --git a/validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py b/validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py
index 56165fdf6..bebaf4b00 100644
--- a/validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py
+++ b/validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py
@@ -3,10 +3,12 @@
# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
import warnings
+from typing import Dict, List, Optional, Union
from warnings import filters as _warnings_filters
import matplotlib.pyplot as plt
import numpy as np
+import pandas as pd
import shap
from validmind import RawData, tags, tasks
@@ -18,7 +20,10 @@
logger = get_logger(__name__)
-def select_shap_values(shap_values, class_of_interest):
+def select_shap_values(
+ shap_values: Union[np.ndarray, List[np.ndarray]],
+ class_of_interest: Optional[int] = None,
+) -> np.ndarray:
"""Selects SHAP values for binary or multiclass classification.
For regression models, returns the SHAP values directly as there are no classes.
@@ -41,32 +46,30 @@ def select_shap_values(shap_values, class_of_interest):
"""
if not isinstance(shap_values, list):
# For regression, return the SHAP values as they are
- # TODO: shap_values is always an array of all predictions, how is the if above supposed to work?
- # logger.info("Returning SHAP values as-is.")
- return shap_values
-
- num_classes = len(shap_values)
-
- # Default to class 1 for binary classification where no class is specified
- if num_classes == 2 and class_of_interest is None:
- logger.debug("Using SHAP values for class 1 (positive class).")
- return shap_values[1]
+ selected_values = shap_values
+ else:
+ num_classes = len(shap_values)
+ # Default to class 1 for binary classification where no class is specified
+ if num_classes == 2 and class_of_interest is None:
+ selected_values = shap_values[1]
+ # Otherwise, use the specified class_of_interest
+ elif class_of_interest is not None and 0 <= class_of_interest < num_classes:
+ selected_values = shap_values[class_of_interest]
+ else:
+ raise ValueError(
+ f"Invalid class_of_interest: {class_of_interest}. Must be between 0 and {num_classes - 1}."
+ )
- # Otherwise, use the specified class_of_interest
- if (
- class_of_interest is None
- or class_of_interest < 0
- or class_of_interest >= num_classes
- ):
- raise ValueError(
- f"Invalid class_of_interest: {class_of_interest}. Must be between 0 and {num_classes - 1}."
- )
+ # Add type conversion here to ensure proper float array
+ if hasattr(selected_values, "dtype"):
+ selected_values = np.array(selected_values, dtype=np.float64)
- logger.debug(f"Using SHAP values for class {class_of_interest}.")
- return shap_values[class_of_interest]
+ return selected_values
-def generate_shap_plot(type_, shap_values, x_test):
+def generate_shap_plot(
+ type_: str, shap_values: np.ndarray, x_test: Union[np.ndarray, pd.DataFrame]
+) -> plt.Figure:
"""Plots two types of SHAP global importance (SHAP).
Args:
@@ -117,8 +120,8 @@ def SHAPGlobalImportance(
dataset: VMDataset,
kernel_explainer_samples: int = 10,
tree_or_linear_explainer_samples: int = 200,
- class_of_interest: int = None,
-):
+ class_of_interest: Optional[int] = None,
+) -> Dict[str, Union[plt.Figure, Dict[str, float]]]:
"""
Evaluates and visualizes global feature importance using SHAP values for model explanation and risk identification.
diff --git a/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py b/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py
index 468b82bb2..f8f0b6667 100644
--- a/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py
+++ b/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py
@@ -47,7 +47,7 @@ def _compute_metrics(
None: The computed metrics are appended to the `results` dictionary in-place.
"""
results["Slice"].append(str(region))
- results["Shape"].append(df_region.shape[0])
+ results["Number of Records"].append(df_region.shape[0])
results["Feature"].append(feature_column)
# Check if df_region is an empty dataframe and if so, append 0 to all metrics
@@ -222,7 +222,7 @@ def WeakspotsDiagnosis(
thresholds = thresholds or DEFAULT_THRESHOLDS
thresholds = {k.title(): v for k, v in thresholds.items()}
- results_headers = ["Slice", "Shape", "Feature"]
+ results_headers = ["Slice", "Number of Records", "Feature"]
results_headers.extend(metrics.keys())
figures = []
@@ -236,19 +236,20 @@ def WeakspotsDiagnosis(
feature_columns
+ [datasets[1].target_column, datasets[1].prediction_column(model)]
]
-
+ results_1 = pd.DataFrame()
+ results_2 = pd.DataFrame()
for feature in feature_columns:
bins = 10
if feature in datasets[0].feature_columns_categorical:
bins = len(df_1[feature].unique())
df_1["bin"] = pd.cut(df_1[feature], bins=bins)
- results_1 = {k: [] for k in results_headers}
- results_2 = {k: [] for k in results_headers}
+ r1 = {k: [] for k in results_headers}
+ r2 = {k: [] for k in results_headers}
for region, df_region in df_1.groupby("bin"):
_compute_metrics(
- results=results_1,
+ results=r1,
metrics=metrics,
region=region,
df_region=df_region,
@@ -260,7 +261,7 @@ def WeakspotsDiagnosis(
(df_2[feature] > region.left) & (df_2[feature] <= region.right)
]
_compute_metrics(
- results=results_2,
+ results=r2,
metrics=metrics,
region=region,
df_region=df_2_region,
@@ -271,8 +272,8 @@ def WeakspotsDiagnosis(
for metric in metrics.keys():
fig, df = _plot_weak_spots(
- results_1=results_1,
- results_2=results_2,
+ results_1=r1,
+ results_2=r2,
feature_column=feature,
metric=metric,
threshold=thresholds[metric],
@@ -284,6 +285,8 @@ def WeakspotsDiagnosis(
# rely on visual assessment for this test for now.
if not df[df[list(thresholds.keys())].lt(thresholds).any(axis=1)].empty:
passed = False
+ results_1 = pd.concat([results_1, pd.DataFrame(r1)])
+ results_2 = pd.concat([results_2, pd.DataFrame(r2)])
return (
pd.concat(
@@ -291,7 +294,9 @@ def WeakspotsDiagnosis(
pd.DataFrame(results_1).assign(Dataset=datasets[0].input_id),
pd.DataFrame(results_2).assign(Dataset=datasets[1].input_id),
]
- ).sort_values(["Feature", "Dataset"]),
+ )
+ .reset_index(drop=True)
+ .sort_values(["Feature", "Dataset"]),
*figures,
passed,
)
diff --git a/validmind/tests/output.py b/validmind/tests/output.py
index d5afc3f3c..52ee23d1b 100644
--- a/validmind/tests/output.py
+++ b/validmind/tests/output.py
@@ -9,6 +9,7 @@
import numpy as np
import pandas as pd
+from validmind.utils import is_html, md_to_html
from validmind.vm_models.figure import (
Figure,
is_matplotlib_figure,
@@ -77,30 +78,72 @@ def process(self, item: Any, result: TestResult) -> None:
class TableOutputHandler(OutputHandler):
def can_handle(self, item: Any) -> bool:
- return isinstance(item, (list, pd.DataFrame, dict, ResultTable))
+ return isinstance(item, (list, pd.DataFrame, dict, ResultTable, tuple))
+
+ def _convert_simple_type(self, data: Any) -> pd.DataFrame:
+ """Convert a simple data type to a DataFrame."""
+ if isinstance(data, dict):
+ return pd.DataFrame([data])
+ elif data is None:
+ return pd.DataFrame()
+ else:
+ raise ValueError(f"Cannot convert {type(data)} to DataFrame")
+
+ def _convert_list(self, data_list: List) -> pd.DataFrame:
+ """Convert a list to a DataFrame."""
+ if not data_list:
+ return pd.DataFrame()
+
+ try:
+ return pd.DataFrame(data_list)
+ except Exception as e:
+ # If conversion fails, try to handle common cases
+ if all(
+ isinstance(item, (int, float, str, bool, type(None)))
+ for item in data_list
+ ):
+ return pd.DataFrame({"Values": data_list})
+ else:
+ raise ValueError(f"Could not convert list to DataFrame: {e}")
+
+ def _convert_to_dataframe(self, table_data: Any) -> pd.DataFrame:
+ """Convert various data types to a pandas DataFrame."""
+ # Handle special cases by type
+ if isinstance(table_data, pd.DataFrame):
+ return table_data
+ elif isinstance(table_data, (dict, str, type(None))):
+ return self._convert_simple_type(table_data)
+ elif isinstance(table_data, tuple):
+ return self._convert_list(list(table_data))
+ elif isinstance(table_data, list):
+ return self._convert_list(table_data)
+ else:
+ # If we reach here, we don't know how to handle this type
+ raise ValueError(
+ f"Invalid table format: must be a list of dictionaries or a DataFrame, got {type(table_data)}"
+ )
def process(
self,
- item: Union[List[Dict[str, Any]], pd.DataFrame, Dict[str, Any], ResultTable],
+ item: Union[
+ List[Dict[str, Any]], pd.DataFrame, Dict[str, Any], ResultTable, str, tuple
+ ],
result: TestResult,
) -> None:
+ # Convert to a dictionary of tables if not already
tables = item if isinstance(item, dict) else {"": item}
for table_name, table_data in tables.items():
- # if already a ResultTable, add it directly
+ # If already a ResultTable, add it directly
if isinstance(table_data, ResultTable):
result.add_table(table_data)
continue
- if not isinstance(table_data, (list, pd.DataFrame)):
- raise ValueError(
- "Invalid table format: must be a list of dictionaries or a DataFrame"
- )
-
- if isinstance(table_data, list):
- table_data = pd.DataFrame(table_data)
+ # Convert the data to a DataFrame using our helper method
+ df = self._convert_to_dataframe(table_data)
- result.add_table(ResultTable(data=table_data, title=table_name or None))
+ # Add the resulting DataFrame as a table to the resul
+ result.add_table(ResultTable(data=df, title=table_name or None))
class RawDataOutputHandler(OutputHandler):
@@ -111,6 +154,17 @@ def process(self, item: Any, result: TestResult) -> None:
result.raw_data = item
+class StringOutputHandler(OutputHandler):
+ def can_handle(self, item: Any) -> bool:
+ return isinstance(item, str)
+
+ def process(self, item: Any, result: TestResult) -> None:
+ if not is_html(item):
+ item = md_to_html(item, mathml=True)
+
+ result.description = item
+
+
def process_output(item: Any, result: TestResult) -> None:
"""Process a single test output item and update the TestResult."""
handlers = [
@@ -119,6 +173,7 @@ def process_output(item: Any, result: TestResult) -> None:
FigureOutputHandler(),
TableOutputHandler(),
RawDataOutputHandler(),
+ StringOutputHandler(),
]
for handler in handlers:
diff --git a/validmind/tests/run.py b/validmind/tests/run.py
index 66dd40e7d..2a32a3a81 100644
--- a/validmind/tests/run.py
+++ b/validmind/tests/run.py
@@ -76,7 +76,7 @@ def _get_run_metadata(**metadata: Dict[str, Any]) -> Dict[str, Any]:
def _get_test_kwargs(
test_func: callable, inputs: Dict[str, Any], params: Dict[str, Any]
-):
+) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""Insepect function signature to build kwargs to pass the inputs and params
that the test function expects
@@ -93,7 +93,7 @@ def _get_test_kwargs(
params (dict): Test parameters e.g. {"param1": 1, "param2": 2}
Returns:
- tuple: Tuple of input and param kwargs
+ Tuple[Dict[str, Any], Dict[str, Any]]: Tuple of input and param kwargs
"""
input_kwargs = {} # map function inputs (`dataset` etc) to actual objects
@@ -222,6 +222,7 @@ def _run_comparison_test(
params: Union[Dict[str, Any], None],
param_grid: Union[Dict[str, List[Any]], List[Dict[str, Any]], None],
title: Optional[str] = None,
+ show_params: bool = True,
):
"""Run a comparison test i.e. a test that compares multiple outputs of a test across
different input and/or param combinations"""
@@ -242,6 +243,7 @@ def _run_comparison_test(
show=False,
generate_description=False,
title=title,
+ show_params=show_params,
)
for config in run_test_configs
]
@@ -253,7 +255,9 @@ def _run_comparison_test(
else:
test_doc = describe_test(test_id, raw=True)["Description"]
- combined_outputs, combined_inputs, combined_params = combine_results(results)
+ combined_outputs, combined_inputs, combined_params = combine_results(
+ results, show_params
+ )
return build_test_result(
outputs=combined_outputs,
@@ -265,7 +269,12 @@ def _run_comparison_test(
)
-def _run_test(test_id: TestID, inputs: Dict[str, Any], params: Dict[str, Any]):
+def _run_test(
+ test_id: TestID,
+ inputs: Dict[str, Any],
+ params: Dict[str, Any],
+ title: Optional[str] = None,
+):
"""Run a standard test and return a TestResult object"""
test_func = load_test(test_id)
input_kwargs, param_kwargs = _get_test_kwargs(
@@ -282,6 +291,7 @@ def _run_test(test_id: TestID, inputs: Dict[str, Any], params: Dict[str, Any]):
test_doc=getdoc(test_func),
inputs=input_kwargs,
params=param_kwargs,
+ title=title,
)
@@ -297,6 +307,7 @@ def run_test( # noqa: C901
generate_description: bool = True,
title: Optional[str] = None,
post_process_fn: Union[Callable[[TestResult], None], None] = None,
+ show_params: bool = True,
**kwargs,
) -> TestResult:
"""Run a ValidMind or custom test
@@ -321,6 +332,7 @@ def run_test( # noqa: C901
generate_description (bool, optional): Whether to generate a description. Defaults to True.
title (str, optional): Custom title for the test result
post_process_fn (Callable[[TestResult], None], optional): Function to post-process the test result
+ show_params (bool, optional): Whether to include parameter values in figure titles for comparison tests. Defaults to True.
Returns:
TestResult: A TestResult object containing the test results
@@ -358,6 +370,7 @@ def run_test( # noqa: C901
input_grid=input_grid,
params=params,
param_grid=param_grid,
+ show_params=show_params,
)
elif unit_metrics:
@@ -375,7 +388,7 @@ def run_test( # noqa: C901
)
else:
- result = _run_test(test_id, inputs, params)
+ result = _run_test(test_id, inputs, params, title)
end_time = time.perf_counter()
result.metadata = _get_run_metadata(duration_seconds=end_time - start_time)
@@ -383,15 +396,16 @@ def run_test( # noqa: C901
if post_process_fn:
result = post_process_fn(result)
- result.description = get_result_description(
- test_id=test_id,
- test_description=result.doc,
- tables=result.tables,
- figures=result.figures,
- metric=result.metric,
- should_generate=generate_description,
- title=title,
- )
+ if not result.description:
+ result.description = get_result_description(
+ test_id=test_id,
+ test_description=result.doc,
+ tables=result.tables,
+ figures=result.figures,
+ metric=result.metric,
+ should_generate=generate_description,
+ title=title,
+ )
if show:
result.show()
diff --git a/validmind/tests/test_providers.py b/validmind/tests/test_providers.py
index 6820e247d..47bf8470e 100644
--- a/validmind/tests/test_providers.py
+++ b/validmind/tests/test_providers.py
@@ -7,7 +7,7 @@
import re
import sys
from pathlib import Path
-from typing import List, Protocol
+from typing import Any, Callable, List, Protocol
from validmind.logging import get_logger
@@ -95,45 +95,38 @@ def __init__(self, root_folder: str):
"""
self.root_folder = os.path.abspath(root_folder)
- def list_tests(self):
+ def list_tests(self) -> List[str]:
"""List all tests in the given namespace
Returns:
list: A list of test IDs
"""
- test_ids = []
-
+ test_files = []
for root, _, files in os.walk(self.root_folder):
- for filename in files:
- if not filename.endswith(".py") or filename.startswith("__"):
- continue
-
- path = Path(root) / filename
- if not _is_test_file(path):
+ for file in files:
+ if not file.endswith(".py"):
continue
- rel_path = path.relative_to(self.root_folder)
-
- test_id_parts = [p.stem for p in rel_path.parents if p.stem][::-1]
- test_id_parts.append(path.stem)
- test_ids.append(".".join(test_id_parts))
+ path = Path(os.path.join(root, file))
+ if _is_test_file(path):
+ rel_path = os.path.relpath(path, self.root_folder)
+ test_id = os.path.splitext(rel_path)[0].replace(os.sep, ".")
+ test_files.append(test_id)
- return sorted(test_ids)
+ return test_files
- def load_test(self, test_id: str):
- """
- Load the test identified by the given test_id.
+ def load_test(self, test_id: str) -> Callable[..., Any]:
+ """Load the test function identified by the given test_id
Args:
- test_id (str): The identifier of the test. This corresponds to the relative
- path of the python file from the root folder, with slashes replaced by dots
+ test_id (str): The test ID (does not contain the namespace under which
+ the test is registered)
Returns:
- The test class that matches the last part of the test_id.
+ callable: The test function
Raises:
- LocalTestProviderLoadModuleError: If the test module cannot be imported
- LocalTestProviderLoadTestError: If the test class cannot be found in the module
+ FileNotFoundError: If the test is not found
"""
# Convert test_id to file path
file_path = os.path.join(self.root_folder, f"{test_id.replace('.', '/')}.py")
@@ -162,28 +155,28 @@ def load_test(self, test_id: str):
class ValidMindTestProvider:
- """Test provider for ValidMind tests"""
+ """Provider for built-in ValidMind tests"""
- def __init__(self):
+ def __init__(self) -> None:
# two subproviders: unit_metrics and normal tests
- self.metrics_provider = LocalTestProvider(
+ self.unit_metrics_provider = LocalTestProvider(
os.path.join(os.path.dirname(__file__), "..", "unit_metrics")
)
- self.tests_provider = LocalTestProvider(os.path.dirname(__file__))
+ self.test_provider = LocalTestProvider(os.path.dirname(__file__))
def list_tests(self) -> List[str]:
- """List all tests in the ValidMind test provider"""
+ """List all tests in the given namespace"""
metric_ids = [
- f"unit_metrics.{test}" for test in self.metrics_provider.list_tests()
+ f"unit_metrics.{test}" for test in self.unit_metrics_provider.list_tests()
]
- test_ids = self.tests_provider.list_tests()
+ test_ids = self.test_provider.list_tests()
return metric_ids + test_ids
- def load_test(self, test_id: str) -> callable:
- """Load a ValidMind test or unit metric"""
+ def load_test(self, test_id: str) -> Callable[..., Any]:
+ """Load the test function identified by the given test_id"""
return (
- self.metrics_provider.load_test(test_id.replace("unit_metrics.", ""))
+ self.unit_metrics_provider.load_test(test_id.replace("unit_metrics.", ""))
if test_id.startswith("unit_metrics.")
- else self.tests_provider.load_test(test_id)
+ else self.test_provider.load_test(test_id)
)
diff --git a/validmind/tests/utils.py b/validmind/tests/utils.py
index fa12c1a84..7ef416071 100644
--- a/validmind/tests/utils.py
+++ b/validmind/tests/utils.py
@@ -5,6 +5,7 @@
"""Test Module Utils"""
import inspect
+from typing import Any, Optional, Tuple, Type, Union
import numpy as np
import pandas as pd
@@ -14,7 +15,7 @@
logger = get_logger(__name__)
-def test_description(test_class, truncate=True):
+def test_description(test_class: Type[Any], truncate: bool = True) -> str:
description = inspect.getdoc(test_class).strip()
if truncate and len(description.split("\n")) > 5:
@@ -23,7 +24,11 @@ def test_description(test_class, truncate=True):
return description
-def remove_nan_pairs(y_true, y_pred, dataset_id=None):
+def remove_nan_pairs(
+ y_true: Union[np.ndarray, list],
+ y_pred: Union[np.ndarray, list],
+ dataset_id: Optional[str] = None,
+) -> Tuple[np.ndarray, np.ndarray]:
"""
Remove pairs where either true or predicted values are NaN/None.
Args:
@@ -52,7 +57,11 @@ def remove_nan_pairs(y_true, y_pred, dataset_id=None):
return y_true, y_pred
-def ensure_equal_lengths(y_true, y_pred, dataset_id=None):
+def ensure_equal_lengths(
+ y_true: Union[np.ndarray, list],
+ y_pred: Union[np.ndarray, list],
+ dataset_id: Optional[str] = None,
+) -> Tuple[np.ndarray, np.ndarray]:
"""
Check if true and predicted values have matching lengths, log warning if they don't,
and truncate to the shorter length if necessary. Also removes any NaN/None values.
@@ -82,7 +91,11 @@ def ensure_equal_lengths(y_true, y_pred, dataset_id=None):
return y_true, y_pred
-def validate_prediction(y_true, y_pred, dataset_id=None):
+def validate_prediction(
+ y_true: Union[np.ndarray, list],
+ y_pred: Union[np.ndarray, list],
+ dataset_id: Optional[str] = None,
+) -> Tuple[np.ndarray, np.ndarray]:
"""
Comprehensive validation of true and predicted value pairs.
Handles NaN/None values and length mismatches.
diff --git a/validmind/unit_metrics/__init__.py b/validmind/unit_metrics/__init__.py
index 8f934c329..8ef360291 100644
--- a/validmind/unit_metrics/__init__.py
+++ b/validmind/unit_metrics/__init__.py
@@ -10,7 +10,7 @@
def list_metrics(**kwargs):
"""List all metrics"""
vm_provider = test_provider_store.get_test_provider("validmind")
- vm_metrics_provider = vm_provider.metrics_provider
+ vm_metrics_provider = vm_provider.unit_metrics_provider
prefix = "validmind.unit_metrics."
diff --git a/validmind/utils.py b/validmind/utils.py
index 4a9c07b46..5d8306a05 100644
--- a/validmind/utils.py
+++ b/validmind/utils.py
@@ -12,7 +12,7 @@
import warnings
from datetime import date, datetime, time
from platform import python_version
-from typing import Any, Dict, List
+from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar
import matplotlib.pylab as pylab
import mistune
@@ -20,6 +20,7 @@
import numpy as np
import pandas as pd
import seaborn as sns
+from bs4 import BeautifulSoup
from IPython.core import getipython
from IPython.display import HTML
from IPython.display import display as ipy_display
@@ -59,23 +60,25 @@
logger = get_logger(__name__)
+T = TypeVar("T")
+
def parse_version(version: str) -> tuple[int, ...]:
"""
- Parse a semver version string into a tuple of major, minor, patch integers
+ Parse a semver version string into a tuple of major, minor, patch integers.
Args:
- version (str): The semantic version string to parse
+ version (str): The semantic version string to parse.
Returns:
- tuple[int, ...]: A tuple of major, minor, patch integers
+ tuple[int, ...]: A tuple of major, minor, patch integers.
"""
return tuple(int(x) for x in version.split(".")[:3])
def is_notebook() -> bool:
"""
- Checks if the code is running in a Jupyter notebook or IPython shell
+ Checks if the code is running in a Jupyter notebook or IPython shell.
https://stackoverflow.com/questions/15411967/how-can-i-check-if-code-is-executed-in-the-ipython-notebook
"""
@@ -209,9 +212,7 @@ def is_dataframe(self, obj):
def get_full_typename(o: Any) -> Any:
- """We determine types based on type names so we don't have to import
- (and therefore depend on) PyTorch, TensorFlow, etc.
- """
+ """We determine types based on type names so we don't have to import."""
instance_name = o.__class__.__module__ + "." + o.__class__.__name__
if instance_name in ["builtins.module", "__builtin__.module"]:
return o.__name__
@@ -313,9 +314,9 @@ def format_key_values(key_values: Dict[str, Any]) -> Dict[str, Any]:
def summarize_data_quality_results(results):
"""
- TODO: generalize this to work with metrics and test results
+ TODO: generalize this to work with metrics and test results.
- Summarize the results of the data quality test suite
+ Summarize the results of the data quality test suite.
"""
test_results = []
for result in results:
@@ -354,25 +355,31 @@ def format_number(number):
def format_dataframe(df: pd.DataFrame) -> pd.DataFrame:
- """Format a pandas DataFrame for display purposes"""
+ """Format a pandas DataFrame for display purposes."""
df = df.style.set_properties(**{"text-align": "left"}).hide(axis="index")
return df.set_table_styles([dict(selector="th", props=[("text-align", "left")])])
-def run_async(func, *args, name=None, **kwargs):
- """Helper function to run functions asynchronously
+def run_async(
+ func: Callable[..., Awaitable[T]],
+ *args: Any,
+ name: Optional[str] = None,
+ **kwargs: Any,
+) -> T:
+ """Helper function to run functions asynchronously.
This takes care of the complexity of running the logging functions asynchronously. It will
- detect the type of environment we are running in (ipython notebook or not) and run the
+ detect the type of environment we are running in (IPython notebook or not) and run the
function accordingly.
Args:
- func (function): The function to run asynchronously
- *args: The arguments to pass to the function
- **kwargs: The keyword arguments to pass to the function
+ func: The function to run asynchronously.
+ *args: The arguments to pass to the function.
+ name: Optional name for the task.
+ **kwargs: The keyword arguments to pass to the function.
Returns:
- The result of the function
+ The result of the function.
"""
try:
if asyncio.get_event_loop().is_running() and is_notebook():
@@ -390,8 +397,19 @@ def run_async(func, *args, name=None, **kwargs):
return asyncio.get_event_loop().run_until_complete(func(*args, **kwargs))
-def run_async_check(func, *args, **kwargs):
- """Helper function to run functions asynchronously if the task doesn't already exist"""
+def run_async_check(
+ func: Callable[..., Awaitable[T]], *args: Any, **kwargs: Any
+) -> Optional[asyncio.Task[T]]:
+ """Helper function to run functions asynchronously if the task doesn't already exist.
+
+ Args:
+ func: The function to run asynchronously.
+ *args: The arguments to pass to the function.
+ **kwargs: The keyword arguments to pass to the function.
+
+ Returns:
+ Optional[asyncio.Task[T]]: The task if created or found, None otherwise.
+ """
if __loop:
return # we don't need this if we are using our own loop
@@ -408,16 +426,16 @@ def run_async_check(func, *args, **kwargs):
pass
-def fuzzy_match(string: str, search_string: str, threshold=0.7):
- """Check if a string matches another string using fuzzy matching
+def fuzzy_match(string: str, search_string: str, threshold: float = 0.7) -> bool:
+ """Check if a string matches another string using fuzzy matching.
Args:
- string (str): The string to check
- search_string (str): The string to search for
- threshold (float): The similarity threshold to use (Default: 0.7)
+ string (str): The string to check.
+ search_string (str): The string to search for.
+ threshold (float): The similarity threshold to use (Default: 0.7).
Returns:
- True if the string matches the search string, False otherwise
+ bool: True if the string matches the search string, False otherwise.
"""
score = difflib.SequenceMatcher(None, string, search_string).ratio()
@@ -448,7 +466,7 @@ def test_id_to_name(test_id: str) -> str:
def get_model_info(model):
- """Attempts to extract all model info from a model object instance"""
+ """Attempts to extract all model info from a model object instance."""
architecture = model.name
framework = model.library
framework_version = model.library_version
@@ -472,7 +490,7 @@ def get_model_info(model):
def get_dataset_info(dataset):
- """Attempts to extract all dataset info from a dataset object instance"""
+ """Attempts to extract all dataset info from a dataset object instance."""
num_rows, num_cols = dataset.df.shape
schema = dataset.df.dtypes.apply(lambda x: x.name).to_dict()
description = (
@@ -491,7 +509,7 @@ def preview_test_config(config):
"""Preview test configuration in a collapsible HTML section.
Args:
- config (dict): Test configuration dictionary
+ config (dict): Test configuration dictionary.
"""
try:
@@ -515,7 +533,7 @@ def preview_test_config(config):
def display(widget_or_html, syntax_highlighting=True, mathjax=True):
- """Display widgets with extra goodies (syntax highlighting, MathJax, etc.)"""
+ """Display widgets with extra goodies (syntax highlighting, MathJax, etc.)."""
if isinstance(widget_or_html, str):
ipy_display(HTML(widget_or_html))
# if html we can auto-detect if we actually need syntax highlighting or MathJax
@@ -532,7 +550,7 @@ def display(widget_or_html, syntax_highlighting=True, mathjax=True):
def md_to_html(md: str, mathml=False) -> str:
- """Converts Markdown to HTML using mistune with plugins"""
+ """Converts Markdown to HTML using mistune with plugins."""
# use mistune with math plugin to convert to html
html = mistune.create_markdown(
plugins=["math", "table", "strikethrough", "footnotes"]
@@ -559,6 +577,63 @@ def md_to_html(md: str, mathml=False) -> str:
return html
+def is_html(text: str) -> bool:
+ """Check if a string is HTML.
+
+ Uses more robust heuristics to determine if a string contains HTML content.
+
+ Args:
+ text (str): The string to check
+
+ Returns:
+ bool: True if the string likely contains HTML, False otherwise
+ """
+ # Strip whitespace first
+ text = text.strip()
+
+ # Basic check: Must at least start with < and end with >
+ if not (text.startswith("<") and text.endswith(">")):
+ return False
+
+ # Look for common HTML tags
+ common_html_patterns = [
+ r"", # HTML tag
+ r"", # Body tag
+ r"", # Div tag
+ r"
""".strip()
self._widgets.append(widgets.HTML(value=results_link))
def _add_description(self):
+ """Add the test suite description to the summary."""
self._widgets.append(
widgets.HTML(
value=f'
- ![\"Screenshot](\"compliance-summary.png\")
\n",
+ " ![\"Screenshot](\"link-validator-evidence.png\")
\n",
+ " ![\"Screenshot](\"selecting-class-imbalance-results.png\")
\n",
+ " ![\"Screenshot](\"inserted-class-imbalance-results.png\")
\n",
+ " ![\"Screenshot](\"class-imbalance-results-detail.png\")
\n",
+ " ![\"Screenshot](\"link-finding.png\")
\n",
+ " ![\"Screenshot](\"select-finding.png\")
\n",
+ " ![\"Screenshot](\"inserted-finding.png\")
\n",
+ "