diff --git a/.github/actions/demo-notebook/action.yml b/.github/actions/demo-notebook/action.yml index 898123341b..aeab9d5597 100644 --- a/.github/actions/demo-notebook/action.yml +++ b/.github/actions/demo-notebook/action.yml @@ -1,14 +1,25 @@ name: Execute demo notebooks -description: Installs python3, validmind, checks dependencies then executes ONLY the Intro for Model Developers notebook with development heap tracking +description: Installs python3, validmind, checks dependencies then executes ONLY the training notebook series with development heap tracking inputs: - env_file: - description: "Load the created .env file" + dev_env: + description: "Load the created dev.env file" + required: true + valid_env: + description: "Load the created valid.env file" required: true runs: using: "composite" steps: + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-pip-${{ runner.os }}-${{ hashFiles('.github/actions/demo-notebook/action.yml') }} + restore-keys: | + python-pip-${{ runner.os }}- + - name: Install python3 for Jupyter Notebooks shell: bash run: | @@ -29,23 +40,46 @@ runs: run: | python -m ipykernel install --user --name validmind --display-name "ValidMind Library" - - name: Ensure .env file is available + - name: Ensure dev.env file is available shell: bash - id: find_env + id: find_dev_env run: | - if [ ! -f "${{ inputs.env_file }}" ]; then - echo "Error: .env file not found at ${{ inputs.env_file }}" + if [ ! -f "${{ inputs.dev_env }}" ]; then + echo "Error: dev.env file not found at ${{ inputs.dev_env }}" exit 1 fi - name: Execute ONLY the ValidMind for model development series with heap development shell: bash - if: ${{ steps.find_env.outcome == 'success' }} + if: ${{ steps.find_dev_env.outcome == 'success' }} run: | cd site - source ../${{ inputs.env_file }} + cp ../${{ inputs.dev_env }} ../.env + source ../.env quarto render --profile exe-demo notebooks/EXECUTED/model_development &> render_errors.log || { - echo "Execute for ValidMind for model development series failed"; - cat render_errors.log; - exit 1; + echo "Execute for ValidMind for model development series failed"; + cat render_errors.log; + exit 1; + } + + - name: Ensure valid.env file is available + shell: bash + id: find_valid_env + run: | + if [ ! -f "${{ inputs.valid_env }}" ]; then + echo "Error: valid.env file not found at ${{ inputs.valid_env }}" + exit 1 + fi + + - name: Execute ONLY the ValidMind for model validation series with heap development + shell: bash + if: ${{ steps.find_valid_env.outcome == 'success' }} + run: | + cd site + cp ../${{ inputs.valid_env }} ../.env + source ../.env + quarto render --profile exe-demo notebooks/EXECUTED/model_validation &> render_errors.log || { + echo "Execute for ValidMind for model validation series failed"; + cat render_errors.log; + exit 1; } diff --git a/.github/actions/prod-notebook/action.yml b/.github/actions/prod-notebook/action.yml index fd86151175..850503dc1f 100644 --- a/.github/actions/prod-notebook/action.yml +++ b/.github/actions/prod-notebook/action.yml @@ -1,14 +1,25 @@ -name: Execute prod notebooks -description: Installs python3, validmind, checks dependencies then executes ONLY the Intro for Model Developers notebook with production heap tracking +name: Execute production notebooks +description: Installs python3, validmind, checks dependencies then executes ONLY the training notebook series with production heap tracking inputs: - env_file: - description: "Load the created .env file" + dev_env: + description: "Load the created dev.env file" + required: true + valid_env: + description: "Load the created valid.env file" required: true runs: using: "composite" steps: + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-pip-${{ runner.os }}-${{ hashFiles('.github/actions/prod-notebook/action.yml') }} + restore-keys: | + python-pip-${{ runner.os }}- + - name: Install python3 for Jupyter Notebooks shell: bash run: | @@ -29,23 +40,46 @@ runs: run: | python -m ipykernel install --user --name validmind --display-name "ValidMind Library" - - name: Ensure .env file is available + - name: Ensure dev.env file is available shell: bash - id: find_env + id: find_dev_env run: | - if [ ! -f "${{ inputs.env_file }}" ]; then - echo "Error: .env file not found at ${{ inputs.env_file }}" + if [ ! -f "${{ inputs.dev_env }}" ]; then + echo "Error: dev.env file not found at ${{ inputs.dev_env }}" exit 1 fi - name: Execute ONLY the ValidMind for model development series with heap production shell: bash - if: ${{ steps.find_env.outcome == 'success' }} + if: ${{ steps.find_dev_env.outcome == 'success' }} run: | cd site - source ../${{ inputs.env_file }} + cp ../${{ inputs.dev_env }} ../.env + source ../.env quarto render --profile exe-prod notebooks/EXECUTED/model_development &> render_errors.log || { - echo "Execute for ValidMind for model development series failed"; - cat render_errors.log; - exit 1; - } \ No newline at end of file + echo "Execute for ValidMind for model development series failed"; + cat render_errors.log; + exit 1; + } + + - name: Ensure valid.env file is available + shell: bash + id: find_valid_env + run: | + if [ ! -f "${{ inputs.valid_env }}" ]; then + echo "Error: valid.env file not found at ${{ inputs.valid_env }}" + exit 1 + fi + + - name: Execute ONLY the ValidMind for model validation series with heap production + shell: bash + if: ${{ steps.find_valid_env.outcome == 'success' }} + run: | + cd site + cp ../${{ inputs.valid_env }} ../.env + source ../.env + quarto render --profile exe-prod notebooks/EXECUTED/model_validation &> render_errors.log || { + echo "Execute for ValidMind for model validation series failed"; + cat render_errors.log; + exit 1; + } diff --git a/.github/actions/staging-notebook/action.yml b/.github/actions/staging-notebook/action.yml index d5a2b8dff3..39334124d0 100644 --- a/.github/actions/staging-notebook/action.yml +++ b/.github/actions/staging-notebook/action.yml @@ -1,14 +1,25 @@ name: Execute staging notebooks -description: Installs python3, validmind, checks dependencies then executes ONLY the Intro for Model Developers notebook with staging heap tracking +description: Installs python3, validmind, checks dependencies then executes ONLY the training notebook series with staging heap tracking inputs: - env_file: - description: "Load the created .env file" + dev_env: + description: "Load the created dev.env file" + required: true + valid_env: + description: "Load the created valid.env file" required: true runs: using: "composite" steps: + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-pip-${{ runner.os }}-${{ hashFiles('.github/actions/staging-notebook/action.yml') }} + restore-keys: | + python-pip-${{ runner.os }}- + - name: Install python3 for Jupyter Notebooks shell: bash run: | @@ -29,23 +40,46 @@ runs: run: | python -m ipykernel install --user --name validmind --display-name "ValidMind Library" - - name: Ensure .env file is available + - name: Ensure dev.env file is available shell: bash - id: find_env + id: find_dev_env run: | - if [ ! -f "${{ inputs.env_file }}" ]; then - echo "Error: .env file not found at ${{ inputs.env_file }}" + if [ ! -f "${{ inputs.dev_env }}" ]; then + echo "Error: dev.env file not found at ${{ inputs.dev_env }}" exit 1 fi - name: Execute ONLY the ValidMind for model development series with heap staging shell: bash - if: ${{ steps.find_env.outcome == 'success' }} + if: ${{ steps.find_dev_env.outcome == 'success' }} run: | cd site - source ../${{ inputs.env_file }} + cp ../${{ inputs.dev_env }} ../.env + source ../.env quarto render --profile exe-staging notebooks/EXECUTED/model_development &> render_errors.log || { - echo "Execute for iValidMind for model development series failed"; - cat render_errors.log; - exit 1; - } \ No newline at end of file + echo "Execute for ValidMind for model development series failed"; + cat render_errors.log; + exit 1; + } + + - name: Ensure valid.env file is available + shell: bash + id: find_valid_env + run: | + if [ ! -f "${{ inputs.valid_env }}" ]; then + echo "Error: valid.env file not found at ${{ inputs.valid_env }}" + exit 1 + fi + + - name: Execute ONLY the ValidMind for model validation series with heap staging + shell: bash + if: ${{ steps.find_valid_env.outcome == 'success' }} + run: | + cd site + cp ../${{ inputs.valid_env }} ../.env + source ../.env + quarto render --profile exe-staging notebooks/EXECUTED/model_validation &> render_errors.log || { + echo "Execute for ValidMind for model validation series failed"; + cat render_errors.log; + exit 1; + } diff --git a/.github/workflows/deploy-docs-prod.yaml b/.github/workflows/deploy-docs-prod.yaml index 84c4c2e750..76fa1e2148 100644 --- a/.github/workflows/deploy-docs-prod.yaml +++ b/.github/workflows/deploy-docs-prod.yaml @@ -48,23 +48,38 @@ jobs: exit 1; } - - name: Create .env file - id: create_env + # If yes then create the dev.env file for use in execution step + - name: Create dev.env file + if: steps.filter.outputs.notebooks == 'true' + id: create_dev_env run: | - touch .env - echo VM_API_HOST=${{ secrets.PLATFORM_API_HOST }} >> .env - echo VM_API_KEY=${{ secrets.PLATFORM_API_KEY }} >> .env - echo VM_API_SECRET=${{ secrets.PLATFORM_API_SECRET }} >> .env - echo VM_API_MODEL=${{ secrets.PLATFORM_DEV_MODEL }} >> .env - cat .env + touch dev.env + echo VM_API_HOST=${{ secrets.PLATFORM_API_HOST }} >> dev.env + echo VM_API_KEY=${{ secrets.PLATFORM_API_KEY }} >> dev.env + echo VM_API_SECRET=${{ secrets.PLATFORM_API_SECRET }} >> dev.env + echo VM_API_MODEL=${{ secrets.PLATFORM_DEV_MODEL }} >> dev.env + cat dev.env - # Only execute the prod notebook if .env file is created - - name: Execute prod ValidMind for model development series - if: ${{ steps.create_env.outcome == 'success' }} + # If yes then create the valid.env file for use in execution step + - name: Create valid.env file + if: steps.filter.outputs.notebooks == 'true' + id: create_valid_env + run: | + touch valid.env + echo VM_API_HOST=${{ secrets.PLATFORM_API_HOST }} >> valid.env + echo VM_API_KEY=${{ secrets.PLATFORM_API_KEY }} >> valid.env + echo VM_API_SECRET=${{ secrets.PLATFORM_API_SECRET }} >> valid.env + echo VM_API_MODEL=${{ secrets.PLATFORM_VALID_MODEL }} >> valid.env + cat valid.env + + # Only execute the production notebooks for training if .env files are created + - name: Execute production ValidMind for model development and validation series + if: ${{ steps.create_dev_env.outcome == 'success' && steps.create_valid_env.outcome == 'success' }} uses: ./.github/actions/prod-notebook id: execute-prod-notebook with: - env_file: .env + dev_env: dev.env + valid_env: valid.env # Prod bucket is in us-east-1 - name: Configure AWS credentials diff --git a/.github/workflows/deploy-docs-staging.yaml b/.github/workflows/deploy-docs-staging.yaml index 523824c51d..aa295f2d2a 100644 --- a/.github/workflows/deploy-docs-staging.yaml +++ b/.github/workflows/deploy-docs-staging.yaml @@ -38,23 +38,38 @@ jobs: exit 1; } - - name: Create .env file - id: create_env + # If yes then create the dev.env file for use in execution step + - name: Create dev.env file + if: steps.filter.outputs.notebooks == 'true' + id: create_dev_env run: | - touch .env - echo VM_API_HOST=${{ secrets.PLATFORM_API_HOST }} >> .env - echo VM_API_KEY=${{ secrets.PLATFORM_API_KEY }} >> .env - echo VM_API_SECRET=${{ secrets.PLATFORM_API_SECRET }} >> .env - echo VM_API_MODEL=${{ secrets.PLATFORM_DEV_MODEL }} >> .env - cat .env + touch dev.env + echo VM_API_HOST=${{ secrets.PLATFORM_API_HOST }} >> dev.env + echo VM_API_KEY=${{ secrets.PLATFORM_API_KEY }} >> dev.env + echo VM_API_SECRET=${{ secrets.PLATFORM_API_SECRET }} >> dev.env + echo VM_API_MODEL=${{ secrets.PLATFORM_DEV_MODEL }} >> dev.env + cat dev.env - # Only execute the staging notebook if .env file is created - - name: Execute staging ValidMind for model development series - if: ${{ steps.create_env.outcome == 'success' }} + # If yes then create the valid.env file for use in execution step + - name: Create valid.env file + if: steps.filter.outputs.notebooks == 'true' + id: create_valid_env + run: | + touch valid.env + echo VM_API_HOST=${{ secrets.PLATFORM_API_HOST }} >> valid.env + echo VM_API_KEY=${{ secrets.PLATFORM_API_KEY }} >> valid.env + echo VM_API_SECRET=${{ secrets.PLATFORM_API_SECRET }} >> valid.env + echo VM_API_MODEL=${{ secrets.PLATFORM_VALID_MODEL }} >> valid.env + cat valid.env + + # Only execute the staging notebooks for training if .env files are created + - name: Execute staging ValidMind for model development and validation series + if: ${{ steps.create_dev_env.outcome == 'success' && steps.create_valid_env.outcome == 'success' }} uses: ./.github/actions/staging-notebook id: execute-staging-notebook with: - env_file: .env + dev_env: dev.env + valid_env: valid.env # Staging bucket is in us-west-2 - name: Configure AWS credentials diff --git a/.github/workflows/validate-docs-site.yaml b/.github/workflows/validate-docs-site.yaml index 13e500f8c9..2bdd873180 100644 --- a/.github/workflows/validate-docs-site.yaml +++ b/.github/workflows/validate-docs-site.yaml @@ -57,27 +57,40 @@ jobs: ref: ${{ github.head_ref }} filters: | notebooks: - - 'site/notebooks/**' + - 'site/notebooks/EXECUTED/**' - # If yes then create the .env file for use in execution step - - name: Create .env file + # If yes then create the dev.env file for use in execution step + - name: Create dev.env file if: steps.filter.outputs.notebooks == 'true' - id: create_env + id: create_dev_env run: | - touch .env - echo VM_API_HOST=${{ secrets.PLATFORM_API_HOST }} >> .env - echo VM_API_KEY=${{ secrets.PLATFORM_API_KEY }} >> .env - echo VM_API_SECRET=${{ secrets.PLATFORM_API_SECRET }} >> .env - echo VM_API_MODEL=${{ secrets.PLATFORM_DEV_MODEL }} >> .env - cat .env - - # Only execute the demo notebook if .env file is created - - name: Execute demo ValidMind for model development series - if: ${{ vars.ENABLE_DEMO_NOTEBOOK == 'true' && steps.create_env.outcome == 'success' }} + touch dev.env + echo VM_API_HOST=${{ secrets.PLATFORM_API_HOST }} >> dev.env + echo VM_API_KEY=${{ secrets.PLATFORM_API_KEY }} >> dev.env + echo VM_API_SECRET=${{ secrets.PLATFORM_API_SECRET }} >> dev.env + echo VM_API_MODEL=${{ secrets.PLATFORM_DEV_MODEL }} >> dev.env + cat dev.env + + # If yes then create the valid.env file for use in execution step + - name: Create valid.env file + if: steps.filter.outputs.notebooks == 'true' + id: create_valid_env + run: | + touch valid.env + echo VM_API_HOST=${{ secrets.PLATFORM_API_HOST }} >> valid.env + echo VM_API_KEY=${{ secrets.PLATFORM_API_KEY }} >> valid.env + echo VM_API_SECRET=${{ secrets.PLATFORM_API_SECRET }} >> valid.env + echo VM_API_MODEL=${{ secrets.PLATFORM_VALID_MODEL }} >> valid.env + cat valid.env + + # Only execute the demo notebooks for training if .env files are created + - name: Execute demo ValidMind for model development and validation series + if: ${{ vars.ENABLE_DEMO_NOTEBOOK == 'true' && steps.create_dev_env.outcome == 'success' && steps.create_valid_env.outcome == 'success' }} uses: ./.github/actions/demo-notebook id: execute-demo-notebook with: - env_file: .env + dev_env: dev.env + valid_env: valid.env - name: Test for warnings or errors run: | diff --git a/site/Makefile b/site/Makefile index 2520957700..e4a0c54b93 100644 --- a/site/Makefile +++ b/site/Makefile @@ -6,11 +6,9 @@ SRC_DIR := $(SRC_ROOT)/validmind-library DEST_DIR_NB := notebooks DEST_DIR_PYTHON := validmind DEST_DIR_TESTS := tests -FILE_PATH := notebooks/EXECUTED/model_development GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) INSTALLATION_BRANCH := main LIBRARY_BRANCH ?= $(or $(BRANCH),main) -PROFILE := exe-demo SRC_ROOT := _source SRC_DIR := $(SRC_ROOT)/validmind-library @@ -34,7 +32,7 @@ help: @echo " docker-site Get source, render site with Docker profile, execute notebooks" @echo " docker-site-lite Get source and render site with Docker profile (skips notebook execution)" @echo " docs-site Get all source files and render the production docs site with Quarto" - @echo " execute Execute a Jupyter Notebook, optional overrides: PROFILE= FILE_PATH" + @echo " execute Execute a Jupyter Notebook or notebook directory" @echo " get-source Get all source files (clean, clone, copy-installation, notebooks, python-docs, test-descriptions)" @echo " notebooks Copy Jupyter notebooks into notebooks/" @echo " python-docs Copy the Python library docs into validmind/" @@ -48,7 +46,6 @@ clean: @rm -rf $(SRC_DIR) @rm -rf $(SRC_ROOT) @rm -rf installation/* - @rm -rf notebooks/EXECUTED/model_development/ # Prompt for a branch or release tag and clone the repository clone: @@ -124,15 +121,48 @@ docker-site-lite: get-source @echo "\nRendering the configurable static HTML site for Docker (skip notebook execution) ..." quarto render --profile docker 2>&1 | grep -v "WARN:" 1>&2 -# Get all source files +# Get all source docs docs-site: get-source @echo "\nRendering the static HTML site ..." quarto render --profile production @$(MAKE) execute PROFILE=exe-prod -# Will default to `exe-demo` profile & the `notebooks/EXECUTED/model_development` if no input provided +# Execute Juptyer Notebooks with a profile & provided credentials execute: - quarto render --profile $(PROFILE) $(FILE_PATH) + @bash -c ' \ + echo "Select PROFILE:"; \ + echo "1) exe-demo"; \ + echo "2) exe-staging"; \ + echo "3) exe-prod"; \ + read -p "Enter choice [1-3] (default: 1): " PROFILE_CHOICE; \ + if [ "$$PROFILE_CHOICE" = "1" ] || [ -z "$$PROFILE_CHOICE" ]; then PROFILE="exe-demo"; \ + elif [ "$$PROFILE_CHOICE" = "2" ]; then PROFILE="exe-staging"; \ + elif [ "$$PROFILE_CHOICE" = "3" ]; then PROFILE="exe-prod"; \ + else echo "Invalid choice"; exit 1; fi; \ + echo "Select FILE_PATH:"; \ + echo "1) notebooks/EXECUTED/model_development"; \ + echo "2) notebooks/EXECUTED/model_validation"; \ + echo "3) Enter custom filepath"; \ + read -p "Enter choice [1-3]: " FILE_CHOICE; \ + if [ "$$FILE_CHOICE" = "1" ]; then FILE_PATH="notebooks/EXECUTED/model_development"; \ + elif [ "$$FILE_CHOICE" = "2" ]; then FILE_PATH="notebooks/EXECUTED/model_validation"; \ + elif [ "$$FILE_CHOICE" = "3" ]; then read -p "Enter custom FILE_PATH: " CUSTOM_FILE_PATH; FILE_PATH="$$CUSTOM_FILE_PATH"; \ + else echo "Invalid choice"; exit 1; fi; \ + read -p "Enter VM_API_HOST (default: https://api.prod.validmind.ai/api/v1/tracking): " INPUT_VM_API_HOST; \ + VM_API_HOST=$${INPUT_VM_API_HOST:-https://api.prod.validmind.ai/api/v1/tracking}; \ + read -p "Enter VM_API_KEY: " VM_API_KEY; \ + read -p "Enter VM_API_SECRET: " VM_API_SECRET; \ + read -p "Enter VM_API_MODEL: " VM_API_MODEL; \ + env_file="temp.env"; \ + echo "VM_API_HOST=$$VM_API_HOST" > $$env_file; \ + echo "VM_API_KEY=$$VM_API_KEY" >> $$env_file; \ + echo "VM_API_SECRET=$$VM_API_SECRET" >> $$env_file; \ + echo "VM_API_MODEL=$$VM_API_MODEL" >> $$env_file; \ + set -a; \ + source $$env_file; \ + set +a; \ + quarto render --profile "$$PROFILE" "$$FILE_PATH"; \ + rm $$env_file ' # Get all source files get-source: clean clone copy-installation notebooks python-docs test-descriptions @@ -146,6 +176,9 @@ notebooks: @echo "Duplicating all files from notebooks/tutorials/model_development/ for execution" @mkdir -p notebooks/EXECUTED/model_development/ @cp -r notebooks/tutorials/model_development/* notebooks/EXECUTED/model_development/ + @echo "Duplicating all files from notebooks/tutorials/model_validation/ for execution" + @mkdir -p notebooks/EXECUTED/model_validation/ + @cp -r notebooks/tutorials/model_validation/* notebooks/EXECUTED/model_validation/ @echo "Copying LICENSE into notebooks ..." @cp -r $(SRC_DIR)/LICENSE $(DEST_DIR_NB)/ @rm -rf $(DEST_DIR_NB)/code_sharing diff --git a/site/_quarto-exe-demo.yml b/site/_quarto-exe-demo.yml index 36e5a37ca9..0f851ada7a 100644 --- a/site/_quarto-exe-demo.yml +++ b/site/_quarto-exe-demo.yml @@ -3,6 +3,7 @@ format: page-layout: full navbar: false code-tools: true + repo-actions: false include-in-header: - environments/heap-development.html execute: diff --git a/site/_quarto-exe-prod.yml b/site/_quarto-exe-prod.yml index 63c2c1ae2a..64d94ff36c 100644 --- a/site/_quarto-exe-prod.yml +++ b/site/_quarto-exe-prod.yml @@ -3,6 +3,7 @@ format: page-layout: full navbar: false code-tools: true + repo-actions: false include-in-header: - environments/heap-production.html execute: diff --git a/site/_quarto-exe-staging.yml b/site/_quarto-exe-staging.yml index 6d29745454..79e37faab1 100644 --- a/site/_quarto-exe-staging.yml +++ b/site/_quarto-exe-staging.yml @@ -3,6 +3,7 @@ format: page-layout: full navbar: false code-tools: true + repo-actions: false include-in-header: - environments/heap-staging.html execute: diff --git a/site/get-started/platform/register-your-first-model.qmd b/site/get-started/platform/register-your-first-model.qmd index 7524c0e962..e67757990d 100644 --- a/site/get-started/platform/register-your-first-model.qmd +++ b/site/get-started/platform/register-your-first-model.qmd @@ -37,7 +37,7 @@ Signing up is FREE — {{< var link.register >}} You can fill in other options according to your own preference. -2. Click **Register Model**. +2. Click **{{< fa plus >}} Register Model**. 3. After the model is registered, confirm that initial model documentation and validation report outlines have been provided based on the documentation template[^2] you selected. diff --git a/site/guide/configuration/_manage-permissions.qmd b/site/guide/configuration/_manage-permissions.qmd index ed4eab9900..ab4996adad 100644 --- a/site/guide/configuration/_manage-permissions.qmd +++ b/site/guide/configuration/_manage-permissions.qmd @@ -1,10 +1,10 @@ 1. Under User Permissions, select **{{< fa screwdriver-wrench >}} Permissions**. 1. Add or delete roles from a specific permission: - - To assign a role to a permission, click **+**, then select the permission from the drop-down menu. + - To assign a role to a permission, click **{{< fa plus >}}**, then select the permission from the drop-down menu. - To remove a role from a permission, click **{{< fa x >}}** next to the specified role. For example, to grant the [{{< fa code >}} Developer]{.bubble} role the permission to update an existing template: 1. Locate **Update Template** under {{< fa clipboard >}} Template. - 1. Click **+** and select [**{{< fa code >}} Developer**]{.bubble} from the drop-down menu. \ No newline at end of file + 1. Click **{{< fa plus >}}** and select [**{{< fa code >}} Developer**]{.bubble} from the drop-down menu. \ No newline at end of file diff --git a/site/guide/configuration/_manage-roles.qmd b/site/guide/configuration/_manage-roles.qmd index e9e8ea21fc..927d281a36 100644 --- a/site/guide/configuration/_manage-roles.qmd +++ b/site/guide/configuration/_manage-roles.qmd @@ -33,9 +33,9 @@ ##### Add role to user - 1. Click **+** to open the Assign Roles menu for that user. + 1. Click **{{< fa plus >}}** to open the Assign Roles menu for that user. - 1. Click **+** next to the role for each role to be assigned. + 1. Click **{{< fa plus >}}** next to the role for each role to be assigned. 1. Click **{{< fa puzzle-piece >}} Assign Roles to User** to save changes. @@ -82,9 +82,9 @@ ##### Add role to user - 1. Click **+** to open the Assign Roles menu for that user. + 1. Click **{{< fa plus >}}** to open the Assign Roles menu for that user. - 1. Click **+** next to the role for each role to be assigned. + 1. Click **{{< fa plus >}}** next to the role for each role to be assigned. 1. Click **{{< fa puzzle-piece >}} Assign Roles to User** to save changes. diff --git a/site/guide/configuration/_manage-users-invite.qmd b/site/guide/configuration/_manage-users-invite.qmd index a5de75a5b4..8ad4b87120 100644 --- a/site/guide/configuration/_manage-users-invite.qmd +++ b/site/guide/configuration/_manage-users-invite.qmd @@ -4,9 +4,9 @@ - Enter the user's **[email]{.smallcaps}** address. - Assign the user to a group from the **[group]{.smallcaps}** dropdown menu. - - Click **+** under [role]{.smallcap} to open the Assign Roles menu for that user: + - Click **{{< fa plus >}}** under [role]{.smallcap} to open the Assign Roles menu for that user: - a. Click **+** next to the role for each role to be assigned. + a. Click **{{< fa plus >}}** next to the role for each role to be assigned. b. Click **Assign Roles to Invitee**. 1. Click **{{< fa envelope >}} Send Invite**. diff --git a/site/guide/configuration/manage-users.qmd b/site/guide/configuration/manage-users.qmd index 00c8a90592..c6d47823ff 100644 --- a/site/guide/configuration/manage-users.qmd +++ b/site/guide/configuration/manage-users.qmd @@ -42,7 +42,7 @@ Invite new users and review your current users from within the user directory. U 3. Next to any individual user, click the {{< fa circle-plus >}} icon to open the Assign Roles menu for that user. -4. Click **+** next to the role for each role to be assigned. +4. Click **{{< fa plus >}}** next to the role for each role to be assigned. 5. Click **{{< fa puzzle-piece >}} Assign Roles to User** to save changes. diff --git a/site/guide/model-documentation/_collaborate-with-others-activity.qmd b/site/guide/model-documentation/_collaborate-with-others-activity.qmd index 040d67a614..29d2cad1d0 100644 --- a/site/guide/model-documentation/_collaborate-with-others-activity.qmd +++ b/site/guide/model-documentation/_collaborate-with-others-activity.qmd @@ -1,12 +1,12 @@ :::: {.content-visible unless-format="revealjs"} -While working with content blocks in documentation,^[[Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd)] you can comment directly on specific portions of the text. +While working with content blocks,^[[Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd)] you can comment directly on specific portions of the text. All users associated with a model, such as model developers and model validators, will see a notification that a comment has been posted in their **Recent Activity** feed, accessible via the **{{< fa home >}} Dashboard**.^[[Customize your dashboard](/guide/configuration/customize-your-dashboard.qmd)] :::: :::: {.content-hidden unless-format="revealjs"} -While working with content blocks in documentation, you can comment directly on specific portions of the text. +While working with content blocks, you can comment directly on specific portions of the text. All users associated with a model, such as model developers and model validators, will see a notification that a comment has been posted in their **Recent Activity** feed, accessible via the [**{{< fa home >}} Dashboard**](/guide/configuration/customize-your-dashboard.qmd){target="_blank"}. diff --git a/site/guide/model-documentation/_collaborate-with-others-comments.qmd b/site/guide/model-documentation/_collaborate-with-others-comments.qmd index 84bfaae0dc..1fbfe4b0a4 100644 --- a/site/guide/model-documentation/_collaborate-with-others-comments.qmd +++ b/site/guide/model-documentation/_collaborate-with-others-comments.qmd @@ -1,6 +1,6 @@ ### Post comments -1. In any section of the model documentation, select the portion of text you want to comment on, then click the **{{< fa message >}}** button that appears. +1. In any text block or test description, select the portion of text you want to comment on, then click the **{{< fa message >}}** button that appears. 2. Enter your comment and click **Comment**. diff --git a/site/guide/model-inventory/register-models-in-inventory.qmd b/site/guide/model-inventory/register-models-in-inventory.qmd index 04c997c4f5..be60a1451e 100644 --- a/site/guide/model-inventory/register-models-in-inventory.qmd +++ b/site/guide/model-inventory/register-models-in-inventory.qmd @@ -31,7 +31,7 @@ If no additional fields are required on registration for your models: 1. Provide the information outlined in the basic model information fields table below. -2. Click **Register Model** to create a new entry in the model inventory. +2. Click **{{< fa plus >}} Register Model** to create a new entry in the model inventory. | Field | Description | |---:|---| @@ -57,7 +57,7 @@ If additional fields are required on model registration for your organization: 2. Fill in your organization's required additional fields. -3. Click **Register Model** to create a new entry in the model inventory. +3. Click **{{< fa plus >}} Register Model** to create a new entry in the model inventory. ## What's next diff --git a/site/guide/model-validation/_add-finding-via-overview.qmd b/site/guide/model-validation/_add-finding-via-overview.qmd new file mode 100644 index 0000000000..d22f5595c1 --- /dev/null +++ b/site/guide/model-validation/_add-finding-via-overview.qmd @@ -0,0 +1,45 @@ + +:::: {.content-visible unless-format="revealjs"} + +#### Add finding via overview + +To log a finding from the documentation overview: + +1. On your model's **{{< fa book-open >}} Documentation** page, click **{{< fa plus >}} Add Finding**. + +1. On the **Add Model Finding** modal that opens, provide information for: + + - Title + - Risk area + - Owner + - Severity + - Due date + - Documentation section + - Description + +1. When you are done, click **Save** to submit the finding. + + Once created, you can then add supporting documentation to your finding.^[[Manage supporting documentation](/guide/model-validation/add-manage-model-findings.qmd#manage-supporting-documentation)] + +:::: + +:::: {.content-hidden unless-format="revealjs"} +To log a finding from the documentation overview: + +1. On your model's **{{< fa book-open >}} Documentation** page, click **{{< fa plus >}} Add Finding**. + +1. On the **Add Model Finding** modal that opens, provide information for: + + - Title + - Risk area + - Owner + - Severity + - Due date + - Documentation section + - Description + +1. When you are done, click **Save** to submit the finding. + + Once created, you can then [add supporting documentation to your finding](/guide/model-validation/add-manage-model-findings.qmd#manage-supporting-documentation){target="_blank"}. + +:::: diff --git a/site/guide/model-validation/_add-finding-via-section.qmd b/site/guide/model-validation/_add-finding-via-section.qmd new file mode 100644 index 0000000000..b44928f8ee --- /dev/null +++ b/site/guide/model-validation/_add-finding-via-section.qmd @@ -0,0 +1,55 @@ +:::: {.content-visible unless-format="revealjs"} +#### Add finding via section + +While working within a section of your documentation, you can easily log a finding associated with that section without leaving the page: + +1. Click on **{{< var vm.product >}} Insights™** to expand the insight panel. + +1. For the section you want to add a Finding for, click **{{< fa triangle-exclamation >}} Add Finding** beneath the Documentation Guidelines. + +1. On the **Add Model Finding** modal that opens, provide information for: + + - Title + - Risk area + - Owner + - Severity + - Due date + - Documentation section + - Description + + The documentation section will be auto-populated with the section you are working from — you are able to select another section if desired. + + ![Adding a finding with the {{< var vm.product >}} Insights™ panel](add-section-finding.gif){fig-alt="An animated gif demonstrating how to add a finding with the ValidMind Insights™ panel" .screenshot} + +1. When you are done, click **Save** to submit the finding. + + Once created, you can then add supporting documentation to your finding.^[[Manage supporting documentation](/guide/model-validation/add-manage-model-findings.qmd#manage-supporting-documentation)] + + +:::: + +:::: {.content-hidden unless-format="revealjs"} +While working within a section of your documentation, you can easily log a finding associated with that section without leaving the page: + +1. Click on **{{< var vm.product >}} Insights™** to expand the insight panel. + +1. For the section you want to add a Finding for, click **{{< fa triangle-exclamation >}} Add Finding** beneath the Documentation Guidelines. + +1. On the **Add Model Finding** modal that opens, provide information for: + + - Title + - Risk area + - Owner + - Severity + - Due date + - Documentation section + - Description + + The documentation section will be auto-populated with the section you are working from — you are able to select another section if desired. + +1. When you are done, click **Save** to submit the finding. + + Once created, you can then [add supporting documentation to your finding](/guide/model-validation/add-manage-model-findings.qmd#manage-supporting-documentation){target="_blank"}. + +:::: + diff --git a/site/guide/model-validation/_assess-compliance-assess.qmd b/site/guide/model-validation/_assess-compliance-assess.qmd index 1f793f8edf..8b41bca324 100644 --- a/site/guide/model-validation/_assess-compliance-assess.qmd +++ b/site/guide/model-validation/_assess-compliance-assess.qmd @@ -27,15 +27,7 @@ :::: :::: {.content-hidden unless-format="revealjs"} -1. Select a model or [find your model by applying a filter or searching for it](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models){target="_blank"}. - -1. In the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**. - - You can now expand any subsection of the validation report you would like to work with. - - For example, locate 2. Validation then select **2.1.1. Assumptions**. - -1. In any section of the documentation where the **[assessment]{.smallcaps}** dropdown menu is available, select one of the available options: +Select one of the available options: - **No compliance** — No tests have been run and no other evidence has been provided. - **Some compliance** — Evidence provided but not properly documented or verifiable. diff --git a/site/guide/model-validation/_view-all-findings.qmd b/site/guide/model-validation/_view-all-findings.qmd new file mode 100644 index 0000000000..feb8dd1cd7 --- /dev/null +++ b/site/guide/model-validation/_view-all-findings.qmd @@ -0,0 +1,74 @@ +:::: {.content-visible unless-format="revealjs"} +#### All model findings + +To view findings for all models: + +1. In the left sidebar, click **{{< fa triangle-exclamation >}} Findings**. + + In the page that opens is a list of all model findings, including information for: + + - Title + - Severity + - Status + - Inventory model + - Risk area + - Assignee + - Due date + - Creation date + - Business unit + + ![List of findings across all models](/guide/model-validation/all-model-findings.png){fig-alt="A screenshot showing a list of findings across all models" .screenshot} + +::: {.callout} +Filter this list to include only findings you want to see,^[[Filter and sort model findings](/guide/model-validation/view-filter-model-findings.qmd#filter-and-sort-model-findings)] or toggle visibilty for column headers.^[[Customize model findings layout](/guide/model-validation/view-filter-model-findings.qmd#customize-model-findings-layout)] + +::: + +1. Click on any model finding for more information, including the proposed remediation plan and any supporting attachments.^[[Manage supporting documentation](/guide/model-validation/add-manage-model-findings.qmd#manage-supporting-documentation)] + + Attachments are listed with the following information: + + - Filename + - Icon representing file type + - Date uploaded + - The user who uploaded the file + - File size + - Notes + + Click on the attachment to view or download the file. expanded sidebar that appears for your model, click **{{< fa wifi >}} Model Activity**. + +:::: + +:::: {.content-hidden unless-format="revealjs"} +To view findings for all models: + +1. In the left sidebar, click **{{< fa triangle-exclamation >}} Findings**. + + In the page that opens is a list of all model findings, including information for: + + - Title + - Severity + - Status + - Inventory model + - Risk area + - Assignee + - Due date + - Creation date + - Business unit + + [Filter this list](/guide/model-validation/view-filter-model-findings.qmd#filter-and-sort-model-findings){target="_blank"} to include only findings you want to see, or [toggle visibilty for column headers](/guide/model-validation/view-filter-model-findings.qmd#customize-model-findings-layout){target="_blank"}. + +1. Click on any model finding for more information, including the proposed remediation plan and [any supporting attachments](/guide/model-validation/add-manage-model-findings.qmd#manage-supporting-documentation){target="_blank"}. + + Attachments are listed with the following information: + + - Filename + - Icon representing file type + - Date uploaded + - The user who uploaded the file + - File size + - Notes + + Click on the attachment to view or download the file. expanded sidebar that appears for your model, click **{{< fa wifi >}} Model Activity**. + +:::: \ No newline at end of file diff --git a/site/guide/model-validation/add-manage-model-findings.qmd b/site/guide/model-validation/add-manage-model-findings.qmd index 840ce15b17..d85c53d022 100644 --- a/site/guide/model-validation/add-manage-model-findings.qmd +++ b/site/guide/model-validation/add-manage-model-findings.qmd @@ -22,59 +22,17 @@ Add findings within the {{< var validmind.platform >}} at the model or documenta {{< include _add-model-findings.qmd >}} -#### Add finding via overview +{{< include _add-finding-via-overview.qmd >}} -To log a finding from the documentation overview: - -1. On your model's **{{< fa book-open >}} Documentation** page, click **{{< fa plus >}} Add Finding**. - -2. On the **Add Model Finding** modal that opens, provide information for: - - - Title - - Risk area - - Owner - - Severity - - Due date - - Documentation section - - Description - -3. When you are done, click **Save** to submit the finding. - - Once created, you can then add supporting documentation to your finding.[^4] - -#### Add finding via section - -While working within a section of your documentation, you can easily log a finding associated with that section without leaving the page: - -1. Click on **{{< var vm.product >}} Insights™** to expand the insight panel. - -2. For the section you want to add a Finding for, click **{{< fa triangle-exclamation >}} Add Finding** beneath the Documentation Guidelines. - -3. On the **Add Model Finding** modal that opens, provide information for: - - - Title - - Risk area - - Owner - - Severity - - Due date - - Documentation section - - Description - - The documentation section will be auto-populated with the section you are working from — you are able to select another section if desired. - - ![Adding a finding with the {{< var vm.product >}} Insights™ panel](add-section-finding.gif){fig-alt="An animated gif demonstrating how to add a finding with the ValidMind Insights™ panel" .screenshot} - -4. When you are done, click **Save** to submit the finding. - - Once created, you can then add supporting documentation to your finding.[^5] +{{< include _add-finding-via-section.qmd >}} ## Add findings on validation reports -When linking findings to validation reports,[^6] you can add findings as you assess a model's compliance: +When linking findings to validation reports,[^4] you can add findings as you assess a model's compliance: 1. In the left sidebar, click **{{< fa cubes >}} Inventory**. -2. Select a model or find your model by applying a filter or searching for it.[^7] +2. Select a model or find your model by applying a filter or searching for it.[^5] 3. In the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**. @@ -98,14 +56,14 @@ When linking findings to validation reports,[^6] you can add findings as you ass As findings get resolved or require other changes during the model validation process, you can update them: -1. Locate the finding you want to update.[^8] +1. Locate the finding you want to update.[^6] 2. Click on a field to make changes. You can make updates to: - Title - Description - Proposed remediation plan - - Supporting documentation[^9] + - Supporting documentation[^7] - Status - Severity - Risk area @@ -144,14 +102,14 @@ Uploaded files must be less than 50 MB each in size. ## Delete model findings ::: {.callout-important title="Finding deletion is permanent."} -- While finding deletion will be logged under your {{< fa wifi >}} Model Activity,[^10] deleted findings cannot be retrieved. -- Deleting a finding will also remove it from any validation reports[^11] it was previously attached to. +- While finding deletion will be logged under your {{< fa wifi >}} Model Activity,[^8] deleted findings cannot be retrieved. +- Deleting a finding will also remove it from any validation reports[^9] it was previously attached to. ::: -If you logged a finding in error or otherwise no longer need to track that finding, you can delete it: +If you logged a finding in error or otherwise no longer need to track that finding, you can delete it: -1. Locate the finding you want to delete.[^12] +1. Locate the finding you want to delete.[^10] 2. On the finding's detail page, click **{{< fa trash-can>}} Delete Finding** in the right sidebar. @@ -168,20 +126,16 @@ If you logged a finding in error or otherwise no longer need to track that findi [^3]: [Manage permissions](/guide/configuration/manage-permissions.qmd) -[^4]: [Manage supporting documentation](#manage-supporting-documentation) - -[^5]: [Manage supporting documentation](#manage-supporting-documentation) - -[^6]: [Assess compliance](/guide/model-validation/assess-compliance.qmd) +[^4]: [Assess compliance](/guide/model-validation/assess-compliance.qmd#link-findings-to-reports) -[^7]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) +[^5]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) -[^8]: [View and filter model findings](view-filter-model-findings.qmd) +[^6]: [View and filter model findings](view-filter-model-findings.qmd) -[^9]: [Manage supporting documentation](#manage-supporting-documentation) +[^7]: [Manage supporting documentation](#manage-supporting-documentation) -[^10]: [View model activity](/guide/model-inventory/view-model-activity.qmd) +[^8]: [View model activity](/guide/model-inventory/view-model-activity.qmd) -[^11]: [Assess compliance](/guide/model-validation/assess-compliance.qmd#link-findings-to-reports) +[^9]: [Assess compliance](/guide/model-validation/assess-compliance.qmd#link-findings-to-reports) -[^12]: [View and filter model findings](view-filter-model-findings.qmd) \ No newline at end of file +[^10]: [View and filter model findings](view-filter-model-findings.qmd) \ No newline at end of file diff --git a/site/guide/model-validation/view-filter-model-findings.qmd b/site/guide/model-validation/view-filter-model-findings.qmd index 85d7a8d7ca..1439ffad99 100644 --- a/site/guide/model-validation/view-filter-model-findings.qmd +++ b/site/guide/model-validation/view-filter-model-findings.qmd @@ -20,43 +20,7 @@ Access a complete list of filterable findings logged across all your models in t You can review findings for all models at once, or only look at findings linked to a specific model. -#### All model findings - -To view findings for all models: - -1. In the left sidebar, click **{{< fa triangle-exclamation >}} Findings**. - - In the page that opens is a list of all model findings, including information for: - - - Title - - Severity - - Status - - Inventory model - - Risk area - - Assignee - - Due date - - Creation date - - Business unit - - ![List of findings across all models](all-model-findings.png){fig-alt="A screenshot showing a list of findings across all models" .screenshot} - -::: {.callout} -Filter this list to include only findings you want to see,[^4] or toggle visibilty for column headers.[^5] - -::: - -2. Click on any model finding for more information, including the proposed remediation plan and any supporting attachments.[^6] - - Attachments are listed with the following information: - - - Filename - - Icon representing file type - - Date uploaded - - The user who uploaded the file - - File size - - Notes - - Click on the attachment to view or download the file. +{{< include _view-all-findings.qmd >}} #### Model-specific findings @@ -64,7 +28,7 @@ To view findings linked to a specific model: 1. In the left sidebar, click **{{< fa cubes >}} Inventory**. -2. Select a model or find your model by applying a filter or searching for it.[^7] +2. Select a model or find your model by applying a filter or searching for it.[^4] 3. In the left sidebar that appears for your model, click **{{< fa triangle-exclamation >}} Model Findings**. @@ -79,19 +43,18 @@ To view findings linked to a specific model: ![List of findings specific to a model](model-specific-findings.png){fig-alt="A screenshot showing a list of findings specific to a model" .screenshot} -4. Click on any model finding for more information, including the proposed remediation plan and any supporting attachments.[^8] +4. Click on any model finding for more information, including the proposed remediation plan and any supporting attachments.[^5] Attachments are listed with the following information: - - Filename + - Filename - Icon representing file type - Date uploaded - The user who uploaded the file - File size - Notes - Click on the attachment to view or download the file. - + Click on the attachment to view or download the file. ## Filter and sort model findings @@ -141,7 +104,7 @@ Customize what is displayed on your list of all model findings: ![Manage Columns on all Findings](manage-finding-columns-demo.gif){width=40% fig-alt="An animation that shows how to manage columns on all model findings" .screenshot} -Once customized, you can save a view for later application.[^9] +Once customized, you can save a view for later application.[^6] ### Manage custom findings views @@ -154,9 +117,9 @@ Saved views are unique to you and do not display for other users in your organiz You can save custom findings configuration as a view to reapply at any time: -1. Customize a findings layout.[^10] +1. Customize a findings layout.[^7] - You can also apply filters or sorting before saving your view to include them in the view.[^11] + You can also apply filters or sorting before saving your view to include them in the view.[^8] {{< include /guide/model-inventory/_save-layout-view.qmd >}} @@ -179,18 +142,12 @@ Deletion of saved views is permanent. [^3]: [Manage permissions](/guide/configuration/manage-permissions.qmd) -[^4]: [Filter and sort model findings](#filter-and-sort-model-findings) - -[^5]: [Customize model findings layout](#customize-model-findings-layout) - -[^6]: [Manage supporting documentation](add-manage-model-findings.qmd#manage-supporting-documentation) - -[^7]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) +[^4]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) -[^8]: [Manage supporting documentation](add-manage-model-findings.qmd#manage-supporting-documentation) +[^5]: [Manage supporting documentation](add-manage-model-findings.qmd#manage-supporting-documentation) -[^9]: [Save new findings view](#save-new-findings-view) +[^6]: [Save new findings view](#save-new-findings-view) -[^10]: [Customize model findings layout](#customize-model-findings-layout) +[^7]: [Customize model findings layout](#customize-model-findings-layout) -[^11]: [Filter and sort model findings](#filter-and-sort-model-findings) \ No newline at end of file +[^8]: [Filter and sort model findings](#filter-and-sort-model-findings) \ No newline at end of file diff --git a/site/guide/model-workflows/set-up-model-workflows.qmd b/site/guide/model-workflows/set-up-model-workflows.qmd index 02e5e144cb..b3485cb732 100644 --- a/site/guide/model-workflows/set-up-model-workflows.qmd +++ b/site/guide/model-workflows/set-up-model-workflows.qmd @@ -64,7 +64,7 @@ When a workflow is edited, the new flow will only apply to new models created af |---:|---| | [display action button when]{.smallcaps} (optional) | Add conditional requirements for displaying this step.[^6] | | [action button label]{.smallcaps} | Text that displays on your action button. | -| [request fields]{.smallcaps} (optional) | Include inventory fields alongside your action if desired. Select the field, then click **+ Add Field** to insert.[^7] | +| [request fields]{.smallcaps} (optional) | Include inventory fields alongside your action if desired. Select the field, then click **{{< fa plus >}} Add Field** to insert.[^7] | | [to set status]{.smallcaps} | Select the status to transition to. | : **User Action** step configuration {.hover tbl-colwidths="[40,60]"} @@ -152,7 +152,7 @@ Workflows cannot be saved until condition branches are connected to other steps. To configure a condition branch: -1. On the Configure Condition Branch modal, click **+ Add Branch**. +1. On the Configure Condition Branch modal, click **{{< fa plus >}} Add Branch**. 2. Enter in the **[path name]{.smallcaps}** and designate the **[conditions]{.smallcaps}**[^12] that apply to this path. 3. Continue with steps 1 and 2 until your conditional branch logic is complete. diff --git a/site/notebooks.zip b/site/notebooks.zip index 972005fa8d..e37aae0467 100644 Binary files a/site/notebooks.zip and b/site/notebooks.zip differ diff --git a/site/notebooks/EXECUTED/model_development/1-set_up_validmind.ipynb b/site/notebooks/EXECUTED/model_development/1-set_up_validmind.ipynb index 46a002a83d..b79b0b1b27 100644 --- a/site/notebooks/EXECUTED/model_development/1-set_up_validmind.ipynb +++ b/site/notebooks/EXECUTED/model_development/1-set_up_validmind.ipynb @@ -9,7 +9,11 @@ "\n", "Learn how to use ValidMind for your end-to-end model documentation process based on common model development scenarios with our series of four introductory notebooks. This first notebook walks you through the initial setup of the ValidMind Library.\n", "\n", - "These notebooks use a binary classification model as an example, but the same principles shown here apply to other model types." + "These notebooks use a binary classification model as an example, but the same principles shown here apply to other model types.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for developers new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Developer Fundamentals
" ] }, { diff --git a/site/notebooks/EXECUTED/model_development/2-start_development_process.ipynb b/site/notebooks/EXECUTED/model_development/2-start_development_process.ipynb index e886dc7ff4..939b907661 100644 --- a/site/notebooks/EXECUTED/model_development/2-start_development_process.ipynb +++ b/site/notebooks/EXECUTED/model_development/2-start_development_process.ipynb @@ -10,7 +10,11 @@ "\n", "You'll become familiar with the individual tests available in ValidMind, as well as how to run them and change parameters as necessary. Using ValidMind's repository of individual tests as building blocks helps you ensure that a model is being built appropriately. \n", "\n", - "**For a full list of out-of-the-box tests,** refer to our [Test descriptions](https://docs.validmind.ai/developer/model-testing/test-descriptions.html) or try the interactive [Test sandbox](https://docs.validmind.ai/developer/model-testing/test-sandbox.html)." + "**For a full list of out-of-the-box tests,** refer to our [Test descriptions](https://docs.validmind.ai/developer/model-testing/test-descriptions.html) or try the interactive [Test sandbox](https://docs.validmind.ai/developer/model-testing/test-sandbox.html).\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for developers new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Developer Fundamentals
" ] }, { @@ -74,6 +78,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "## Setting up" ] }, @@ -686,8 +692,9 @@ " \"Screenshot\n", "

\n", "\n", - "5. Click **+** and then select **Test-Driven Block**:\n", + "5. Click **+** and then select **Test-Driven Block** under FROM LIBRARY:\n", "\n", + " - Click on **VM Library** under TEST-DRIVEN in the left sidebar.\n", " - In the search bar, type in `HighPearsonCorrelation`.\n", " - Select `HighPearsonCorrelation:balanced_raw_dataset` as the test.\n", "\n", diff --git a/site/notebooks/EXECUTED/model_development/3-integrate_custom_tests.ipynb b/site/notebooks/EXECUTED/model_development/3-integrate_custom_tests.ipynb index 038f45c38a..ab7a4c8805 100644 --- a/site/notebooks/EXECUTED/model_development/3-integrate_custom_tests.ipynb +++ b/site/notebooks/EXECUTED/model_development/3-integrate_custom_tests.ipynb @@ -13,7 +13,11 @@ "- The function can be as simple or as complex as you need it to be — it can use external libraries, make API calls, or do anything else that you can do in Python.\n", "- The only requirement is that the function signature and return values can be \"understood\" and handled by the ValidMind Library. As such, custom tests offer added flexibility by extending the default tests provided by ValidMind, enabling you to document any type of model or use case.\n", "\n", - "**For a more in-depth introduction to custom tests,** refer to our [Implement custom tests](../../code_samples/custom_tests/implement_custom_tests.ipynb) notebook." + "**For a more in-depth introduction to custom tests,** refer to our [Implement custom tests](../../code_samples/custom_tests/implement_custom_tests.ipynb) notebook.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for developers new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Developer Fundamentals
" ] }, { @@ -913,10 +917,10 @@ " \"Screenshot\n", "

\n", "\n", - "5. Click **+** and then select **Test-Driven Block**:\n", + "5. Click **+** and then select **Test-Driven Block** under FROM LIBRARY:\n", "\n", - " - In the search bar, type in `ConfusionMatrix`.\n", - " - Select the custom `ConfusionMatrix` tests you logged above:\n", + " - Click on **Custom** under TEST-DRIVEN in the left sidebar.\n", + " - Select the two custom `ConfusionMatrix` tests you logged above:\n", "\n", " \"Screenshot\n", "

\n", diff --git a/site/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.ipynb b/site/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.ipynb index 13a4f1e148..e38978aec3 100644 --- a/site/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.ipynb +++ b/site/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.ipynb @@ -8,7 +8,11 @@ "\n", "Learn how to use ValidMind for your end-to-end model documentation process with our introductory notebook series. In this last notebook, finalize the testing and documentation of your model and have a fully documented sample model ready for review.\n", "\n", - "We'll first use [`run_documentation_tests()`](https://docs.validmind.ai/validmind/validmind.html#run_documentation_tests) previously covered in **[2 — Start the model development process](2-start_development_process.ipynb)** to ensure that your custom test results generated in **[3 — Integrate custom tests](3-integrate_custom_tests.ipynb)** are included in your documentation. Then, we'll view and update the configuration for the entire model documentation template to suit your needs.\n" + "We'll first use [`run_documentation_tests()`](https://docs.validmind.ai/validmind/validmind.html#run_documentation_tests) previously covered in **[2 — Start the model development process](2-start_development_process.ipynb)** to ensure that your custom test results generated in **[3 — Integrate custom tests](3-integrate_custom_tests.ipynb)** are included in your documentation. Then, we'll view and update the configuration for the entire model documentation template to suit your needs.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for developers new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Developer Fundamentals
" ] }, { diff --git a/site/notebooks/EXECUTED/model_development/add-content-block.gif b/site/notebooks/EXECUTED/model_development/add-content-block.gif index daab9d44d8..60d9e5825d 100644 Binary files a/site/notebooks/EXECUTED/model_development/add-content-block.gif and b/site/notebooks/EXECUTED/model_development/add-content-block.gif differ diff --git a/site/notebooks/EXECUTED/model_development/selecting-confusion-matrix-test.png b/site/notebooks/EXECUTED/model_development/selecting-confusion-matrix-test.png index de27323a76..ce32255cc4 100644 Binary files a/site/notebooks/EXECUTED/model_development/selecting-confusion-matrix-test.png and b/site/notebooks/EXECUTED/model_development/selecting-confusion-matrix-test.png differ diff --git a/site/notebooks/EXECUTED/model_development/selecting-high-pearson-correlation-test.png b/site/notebooks/EXECUTED/model_development/selecting-high-pearson-correlation-test.png index 2c4b87c6bb..7f9868d1e1 100644 Binary files a/site/notebooks/EXECUTED/model_development/selecting-high-pearson-correlation-test.png and b/site/notebooks/EXECUTED/model_development/selecting-high-pearson-correlation-test.png differ diff --git a/site/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.ipynb b/site/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.ipynb new file mode 100644 index 0000000000..71f08b5af6 --- /dev/null +++ b/site/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.ipynb @@ -0,0 +1,461 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a9d0996f", + "metadata": {}, + "source": [ + "# ValidMind for model validation 1 — Set up the ValidMind Library for validation\n", + "\n", + "Learn how to use ValidMind for your end-to-end model validation process based on common scenarios with our series of four introductory notebooks. In this first notebook, set up the ValidMind Library in preparation for validating a champion model.\n", + "\n", + "These notebooks use a binary classification model as an example, but the same principles shown here apply to other model types.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for validators new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Validator Fundamentals
" + ] + }, + { + "cell_type": "markdown", + "id": "c747db34", + "metadata": {}, + "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [Introduction](#toc1_) \n", + "- [About ValidMind](#toc2_) \n", + " - [Before you begin](#toc2_1_) \n", + " - [New to ValidMind?](#toc2_2_) \n", + " - [Key concepts](#toc2_3_) \n", + "- [Setting up](#toc3_) \n", + " - [Register a sample model](#toc3_1_) \n", + " - [Assign validator credentials](#toc3_1_1_) \n", + " - [Install the ValidMind Library](#toc3_2_) \n", + " - [Initialize the ValidMind Library](#toc3_3_) \n", + " - [Get your code snippet](#toc3_3_1_) \n", + "- [Getting to know ValidMind](#toc4_) \n", + " - [Preview the validation report template](#toc4_1_) \n", + " - [View validation report in the ValidMind Platform](#toc4_1_1_) \n", + " - [Explore available tests](#toc4_2_) \n", + "- [Upgrade ValidMind](#toc5_) \n", + "- [In summary](#toc6_) \n", + "- [Next steps](#toc7_) \n", + " - [Start the model validation process](#toc7_1_) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "f1d4715f", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Introduction\n", + "\n", + "Model validation aims to independently assess the compliance of *champion models* created by model developers with regulatory guidance by conducting thorough testing and analysis, potentially including the use of challenger models to benchmark performance. Assessments, presented in the form of a validation report, typically include *model findings* and recommendations to address those issues.\n", + "\n", + "A *binary classification model* is a type of predictive model used in churn analysis to identify customers who are likely to leave a service or subscription by analyzing various behavioral, transactional, and demographic factors.\n", + "\n", + "- This model helps businesses take proactive measures to retain at-risk customers by offering personalized incentives, improving customer service, or adjusting pricing strategies.\n", + "- Effective validation of a churn prediction model ensures that businesses can accurately identify potential churners, optimize retention efforts, and enhance overall customer satisfaction while minimizing revenue loss." + ] + }, + { + "cell_type": "markdown", + "id": "14c2d80d", + "metadata": {}, + "source": [ + "\n", + "\n", + "## About ValidMind\n", + "\n", + "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n", + "\n", + "You use the ValidMind Library to automate comparison and other validation tests, and then use the ValidMind Platform to submit compliance assessments of champion models via comprehensive validation reports. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model developers." + ] + }, + { + "cell_type": "markdown", + "id": "151a4ca5", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Before you begin\n", + "\n", + "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n", + "\n", + "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html)." + ] + }, + { + "cell_type": "markdown", + "id": "089c960e", + "metadata": {}, + "source": [ + "\n", + "\n", + "### New to ValidMind?\n", + "\n", + "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n", + "\n", + "
For access to all features available in this notebook, create a free ValidMind account.\n", + "

\n", + "Signing up is FREE — Register with ValidMind
" + ] + }, + { + "cell_type": "markdown", + "id": "5f307177", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Key concepts\n", + "\n", + "**Validation report**: A comprehensive and structured assessment of a model’s development and performance, focusing on verifying its integrity, appropriateness, and alignment with its intended use. It includes analyses of model assumptions, data quality, performance metrics, outcomes of testing procedures, and risk considerations. The validation report supports transparency, regulatory compliance, and informed decision-making by documenting the validator’s independent review and conclusions.\n", + "\n", + "**Validation report template**: Serves as a standardized framework for conducting and documenting model validation activities. It outlines the required sections, recommended analyses, and expected validation tests, ensuring consistency and completeness across validation reports. The template helps guide validators through a systematic review process while promoting comparability and traceability of validation outcomes.\n", + "\n", + "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets.\n", + "\n", + "**Metrics**: A subset of tests that do not have thresholds. In the context of this notebook, metrics and tests can be thought of as interchangeable concepts.\n", + "\n", + "**Custom metrics**: Custom metrics are functions that you define to evaluate your model or dataset. These functions can be registered with the ValidMind Library to be used in the ValidMind Platform.\n", + "\n", + "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n", + "\n", + " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n", + " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n", + " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom metric.\n", + " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom metric. (Learn more: [Run tests with multiple datasets](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html))\n", + "\n", + "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a metric, customize its behavior, or provide additional context.\n", + "\n", + "**Outputs**: Custom metrics can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures." + ] + }, + { + "cell_type": "markdown", + "id": "c42665b8", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Setting up" + ] + }, + { + "cell_type": "markdown", + "id": "0faed42c", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Register a sample model\n", + "\n", + "In a usual model lifecycle, a champion model will have been independently registered in your model inventory and submitted to you for validation by your model development team as part of the effective challenge process. (**Learn more:** [Submit for approval](https://docs.validmind.ai/guide/model-documentation/submit-for-approval.html))\n", + "\n", + "For this series of notebooks, we'll have you register a dummy model in the ValidMind Platform inventory and assign yourself as the validator to familiarize you with the ValidMind interface and circumvent the need for an existing model:\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", + "\n", + "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", + "\n", + " For example, to register a model for use with this notebook, select:\n", + "\n", + " - Documentation template: `Binary classification`\n", + " - Use case: `Marketing/Sales - Attrition/Churn Management`\n", + "\n", + " You can fill in other options according to your preference." + ] + }, + { + "cell_type": "markdown", + "id": "0c350e0d", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Assign validator credentials\n", + "\n", + "In order to log tests as a validator instead of as a developer, on the model details page that appears after you've successfully registered your sample model:\n", + "\n", + "1. Remove yourself as a model owner: \n", + "\n", + " - Click on the **OWNERS** tile.\n", + " - Click the **x** next to your name to remove yourself from that model's role.\n", + " - Click **Save** to apply your changes to that role.\n", + "\n", + "2. Remove yourself as a developer: \n", + "\n", + " - Click on the **DEVELOPERS** tile.\n", + " - Click the **x** next to your name to remove yourself from that model's role.\n", + " - Click **Save** to apply your changes to that role.\n", + "\n", + "3. Add yourself as a validator: \n", + "\n", + " - Click on the **VALIDATORS** tile.\n", + " - Select your name from the drop-down menu.\n", + " - Click **Save** to apply your changes to that role." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Install the ValidMind Library\n", + "\n", + "
Recommended Python versions\n", + "

\n", + "Python 3.8 <= x <= 3.11
\n", + "\n", + "To install the library:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "931d8f7f", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q validmind" + ] + }, + { + "cell_type": "markdown", + "id": "5ec7fcb7", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind Library\n", + "\n", + "ValidMind generates a unique _code snippet_ for each registered model to connect with your validation environment. You initialize the ValidMind Library with this code snippet, which ensures that your test results are uploaded to the correct model when you run the notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Get your code snippet\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n", + "\n", + "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", + "\n", + "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5d87e2d", + "metadata": {}, + "outputs": [], + "source": [ + "# Load your model identifier credentials from an `.env` file\n", + "\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " # api_host=\"...\",\n", + " # api_key=\"...\",\n", + " # api_secret=\"...\",\n", + " # model=\"...\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "b4b5a00f", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Getting to know ValidMind" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Preview the validation report template\n", + "\n", + "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for model validation. A template predefines sections for your validation report and provides a general outline to follow, making the validation process much easier.\n", + "\n", + "You will attach evidence to this template in the form of risk assessment notes, findings, and test results later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13d34bbb", + "metadata": {}, + "outputs": [], + "source": [ + "vm.preview_template()" + ] + }, + { + "cell_type": "markdown", + "id": "a2e86bc8", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### View validation report in the ValidMind Platform\n", + "\n", + "Next, let's head to the ValidMind Platform to see the template in action:\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n", + "\n", + "3. Click on the **Validation Report** for your model and note:\n", + "\n", + " - [x] The risk assessment compliance summary at the top of the report (screenshot below)\n", + " - [x] How the structure of the validation report reflects the previewed template\n", + "\n", + " \"Screenshot\n", + "

" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Explore available tests\n", + "\n", + "Next, let's explore the list of all available tests in the ValidMind Library with [the `vm.tests.list_tests()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) — we'll later narrow down the tests we want to run from this list when we learn to run tests." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de6abc2a", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.list_tests()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Upgrade ValidMind\n", + "\n", + "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n", + "\n", + "Retrieve the information for the currently installed version of ValidMind:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10272aa9", + "metadata": {}, + "outputs": [], + "source": [ + "%pip show validmind" + ] + }, + { + "cell_type": "markdown", + "id": "upgrade-version-d64591ca-3073-4b3e-9586-d3577adda203", + "metadata": {}, + "source": [ + "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n", + "\n", + "```bash\n", + "%pip install --upgrade validmind\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "44657dea", + "metadata": {}, + "source": [ + "You may need to restart your kernel after running the upgrade package for changes to be applied." + ] + }, + { + "cell_type": "markdown", + "id": "39f45f58", + "metadata": {}, + "source": [ + "\n", + "\n", + "## In summary\n", + "\n", + "In this first notebook, you learned how to:\n", + "\n", + "- [x] Register a model within the ValidMind Platform and assign yourself as the validator\n", + "- [x] Install and initialize the ValidMind Library\n", + "- [x] Preview the validation report template for your model\n", + "- [x] Explore the available tests offered by the ValidMind Library\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Next steps\n", + "\n", + "\n", + "\n", + "### Start the model validation process\n", + "\n", + "Now that the ValidMind Library is connected to your model in the ValidMind Library with the correct template applied, we can go ahead and start the model validation process: **[2 — Start the model validation process](2-start_validation_process.ipynb)**" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ValidMind Library", + "language": "python", + "name": "validmind" + }, + "language_info": { + "name": "python", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/site/notebooks/EXECUTED/model_validation/2-start_validation_process.ipynb b/site/notebooks/EXECUTED/model_validation/2-start_validation_process.ipynb new file mode 100644 index 0000000000..51bb9dc97b --- /dev/null +++ b/site/notebooks/EXECUTED/model_validation/2-start_validation_process.ipynb @@ -0,0 +1,877 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# ValidMind for model validation 2 — Start the model validation process\n", + "\n", + "Learn how to use ValidMind for your end-to-end model validation process with our series of four introductory notebooks. In this second notebook, independently verify the data quality tests performed on the dataset used to train the champion model.\n", + "\n", + "You'll learn how to run relevant validation tests with ValidMind, log the results of those tests to the ValidMind Platform, and insert your logged test results as evidence into your validation report. You'll become familiar with the tests available in ValidMind, as well as how to run them. Running tests during model validation is crucial to the effective challenge process, as we want to independently evaluate the evidence and assessments provided by the model development team.\n", + "\n", + "While running our tests in this notebook, we'll focus on:\n", + "\n", + "- Ensuring that data used for training and testing the model is of appropriate data quality\n", + "- Ensuring that the raw data has been preprocessed appropriately and that the resulting final datasets reflects this\n", + "\n", + "**For a full list of out-of-the-box tests,** refer to our [Test descriptions](https://docs.validmind.ai/developer/model-testing/test-descriptions.html) or try the interactive [Test sandbox](https://docs.validmind.ai/developer/model-testing/test-sandbox.html).\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for validators new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Validator Fundamentals
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [Prerequisites](#toc1_) \n", + "- [Setting up](#toc2_) \n", + " - [Initialize the ValidMind Library](#toc2_1_) \n", + "- [Load the sample dataset](#toc3_) \n", + "- [Verifying data quality adjustments](#toc4_) \n", + " - [Identify qualitative tests](#toc4_1_) \n", + " - [Initialize the ValidMind datasets](#toc4_2_) \n", + " - [Run data quality tests](#toc4_3_) \n", + " - [Run tabular data tests](#toc4_3_1_) \n", + " - [Remove highly correlated features](#toc4_4_) \n", + "- [Documenting test results](#toc5_) \n", + " - [Configure and run comparison tests](#toc5_1_) \n", + " - [Log tests with unique identifiers](#toc5_2_) \n", + " - [Add test results to reporting](#toc5_3_) \n", + "- [Split the preprocessed dataset](#toc6_) \n", + " - [Initialize the split datasets](#toc6_1_) \n", + "- [In summary](#toc7_) \n", + "- [Next steps](#toc8_) \n", + " - [Develop potential challenger models](#toc8_1_) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Prerequisites\n", + "\n", + "In order to independently assess the quality of your datasets with notebook, you'll need to first have:\n", + "\n", + "- [x] Registered a model within the ValidMind Platform and granted yourself access to the model as a validator\n", + "- [x] Installed the ValidMind Library in your local environment, allowing you to access all its features\n", + "\n", + "
Need help with the above steps?\n", + "

\n", + "Refer to the first notebook in this series: 1 — Set up the ValidMind Library for validation
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Setting up" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind Library\n", + "\n", + "First, let's connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n", + "\n", + "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", + "\n", + "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Make sure the ValidMind Library is installed\n", + "\n", + "%pip install -q validmind\n", + "\n", + "# Load your model identifier credentials from an `.env` file\n", + "\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " # api_host=\"...\",\n", + " # api_key=\"...\",\n", + " # api_secret=\"...\",\n", + " # model=\"...\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Load the sample dataset\n", + "\n", + "Let's first import the public [Bank Customer Churn Prediction](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction) dataset from Kaggle, which was used to develop the dummy champion model.\n", + "\n", + "We'll use this dataset to review steps that should have been conducted during the initial development and documentation of the model to ensure that the model was built correctly. By independently performing steps taken by the model development team, we can confirm whether the model was built using appropriate and properly processed data.\n", + "\n", + "In our below example, note that:\n", + "\n", + "- The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n", + "- The ValidMind Library provides a wrapper to automatically load the dataset as a Pandas [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.datasets.classification import customer_churn as demo_dataset\n", + "\n", + "print(\n", + " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n", + ")\n", + "\n", + "raw_df = demo_dataset.load_data()\n", + "raw_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Verifying data quality adjustments\n", + "\n", + "Let's say that thanks to the documentation submitted by the model development team ([Learn more ...](https://docs.validmind.ai/developer/validmind-library.html#for-model-development)), we know that the sample dataset was first modified before being used to train the champion model. After performing some data quality assessments on the raw dataset, it was determined that the dataset required rebalancing, and highly correlated features were also removed." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Identify qualitative tests\n", + "\n", + "During model validation, we use the same data processing logic and training procedure to confirm that the model's results can be reproduced independently, so let's start by doing some data quality assessments by running a few individual tests just like the development team did.\n", + "\n", + "Use the [`vm.tests.list_tests()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) introduced by the first notebook in this series in combination with [`vm.tests.list_tags()`](https://docs.validmind.ai/validmind/validmind/tests.html#list_tags) and [`vm.tests.list_tasks()`](https://docs.validmind.ai/validmind/validmind/tests.html#list_tasks) to find which prebuilt tests are relevant for data quality assessment:\n", + "\n", + "- **`tasks`** represent the kind of modeling task associated with a test. Here we'll focus on `classification` tasks.\n", + "- **`tags`** are free-form descriptions providing more details about the test, for example, what category the test falls into. Here we'll focus on the `data_quality` tag." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get the list of available task types\n", + "sorted(vm.tests.list_tasks())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get the list of available tags\n", + "sorted(vm.tests.list_tags())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can pass `tags` and `tasks` as parameters to the `vm.tests.list_tests()` function to filter the tests based on the tags and task types.\n", + "\n", + "For example, to find tests related to tabular data quality for classification models, you can call `list_tests()` like this:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.list_tests(task=\"classification\", tags=[\"tabular_data\", \"data_quality\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
Want to learn more about navigating ValidMind tests?\n", + "

\n", + "Refer to our notebook outlining the utilities available for viewing and understanding available ValidMind tests: Explore tests
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind datasets\n", + "\n", + "With the individual tests we want to run identified, the next step is to connect your data with a ValidMind `Dataset` object. **This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind,** but you only need to do it once per dataset.\n", + "\n", + "Initialize a ValidMind dataset object using the [`init_dataset` function](https://docs.validmind.ai/validmind/validmind.html#init_dataset) from the ValidMind (`vm`) module. For this example, we'll pass in the following arguments:\n", + "\n", + "- **`dataset`** — The raw dataset that you want to provide as input to tests.\n", + "- **`input_id`** — A unique identifier that allows tracking what inputs are used when running each individual test.\n", + "- **`target_column`** — A required argument if tests require access to true values. This is the name of the target column in the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# vm_raw_dataset is now a VMDataset object that you can pass to any ValidMind test\n", + "vm_raw_dataset = vm.init_dataset(\n", + " dataset=raw_df,\n", + " input_id=\"raw_dataset\",\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Run data quality tests\n", + "\n", + "Now that we know how to initialize a ValidMind `dataset` object, we're ready to run some tests!\n", + "\n", + "You run individual tests by calling [the `run_test` function](https://docs.validmind.ai/validmind/validmind/tests.html#run_test) provided by the `validmind.tests` module. For the examples below, we'll pass in the following arguments:\n", + "\n", + "- **`test_id`** — The ID of the test to run, as seen in the `ID` column when you run `list_tests`. \n", + "- **`params`** — A dictionary of parameters for the test. These will override any `default_params` set in the test definition. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Run tabular data tests\n", + "\n", + "The inputs expected by a test can also be found in the test definition — let's take [`validmind.data_validation.DescriptiveStatistics`](https://docs.validmind.ai/tests/data_validation/DescriptiveStatistics.html) as an example.\n", + "\n", + "Note that the output of the [`describe_test()` function](https://docs.validmind.ai/validmind/validmind/tests.html#describe_test) below shows that this test expects a `dataset` as input:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.describe_test(\"validmind.data_validation.DescriptiveStatistics\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, let's run a few tests to assess the quality of the dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result2 = vm.tests.run_test(\n", + " test_id=\"validmind.data_validation.ClassImbalance\",\n", + " inputs={\"dataset\": vm_raw_dataset},\n", + " params={\"min_percent_threshold\": 30},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output above shows that [the class imbalance test](https://docs.validmind.ai/tests/data_validation/ClassImbalance.html) did not pass according to the value we set for `min_percent_threshold` — great, this matches what was reported by the model development team.\n", + "\n", + "To address this issue, we'll re-run the test on some processed data. In this case let's apply a very simple rebalancing technique to the dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "raw_copy_df = raw_df.sample(frac=1) # Create a copy of the raw dataset\n", + "\n", + "# Create a balanced dataset with the same number of exited and not exited customers\n", + "exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 1]\n", + "not_exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 0].sample(n=exited_df.shape[0])\n", + "\n", + "balanced_raw_df = pd.concat([exited_df, not_exited_df])\n", + "balanced_raw_df = balanced_raw_df.sample(frac=1, random_state=42)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With this new balanced dataset, you can re-run the individual test to see if it now passes the class imbalance test requirement.\n", + "\n", + "As this is technically a different dataset, **remember to first initialize a new ValidMind `Dataset` object** to pass in as input as required by `run_test()`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Register new data and now 'balanced_raw_dataset' is the new dataset object of interest\n", + "vm_balanced_raw_dataset = vm.init_dataset(\n", + " dataset=balanced_raw_df,\n", + " input_id=\"balanced_raw_dataset\",\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Pass the initialized `balanced_raw_dataset` as input into the test run\n", + "result = vm.tests.run_test(\n", + " test_id=\"validmind.data_validation.ClassImbalance\",\n", + " inputs={\"dataset\": vm_balanced_raw_dataset},\n", + " params={\"min_percent_threshold\": 30},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Remove highly correlated features\n", + "\n", + "Next, let's also remove highly correlated features from our dataset as outlined by the development team. Removing highly correlated features helps make the model simpler, more stable, and easier to understand.\n", + "\n", + "You can utilize the output from a ValidMind test for further use — in this below example, to retrieve the list of features with the highest correlation coefficients and use them to reduce the final list of features for modeling.\n", + "\n", + "First, we'll run [`validmind.data_validation.HighPearsonCorrelation`](https://docs.validmind.ai/tests/data_validation/HighPearsonCorrelation.html) with the `balanced_raw_dataset` we initialized previously as input as is for comparison with later runs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "corr_result = vm.tests.run_test(\n", + " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n", + " params={\"max_threshold\": 0.3},\n", + " inputs={\"dataset\": vm_balanced_raw_dataset},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output above shows that the test did not pass according to the value we set for `max_threshold` — as reported and expected.\n", + "\n", + "`corr_result` is an object of type `TestResult`. We can inspect the result object to see what the test has produced:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(type(corr_result))\n", + "print(\"Result ID: \", corr_result.result_id)\n", + "print(\"Params: \", corr_result.params)\n", + "print(\"Passed: \", corr_result.passed)\n", + "print(\"Tables: \", corr_result.tables)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's remove the highly correlated features and create a new VM `dataset` object.\n", + "\n", + "We'll begin by checking out the table in the result and extracting a list of features that failed the test:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Extract table from `corr_result.tables`\n", + "features_df = corr_result.tables[0].data\n", + "features_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Extract list of features that failed the test\n", + "high_correlation_features = features_df[features_df[\"Pass/Fail\"] == \"Fail\"][\"Columns\"].tolist()\n", + "high_correlation_features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, extract the feature names from the list of strings (example: `(Age, Exited)` > `Age`):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_correlation_features = [feature.split(\",\")[0].strip(\"()\") for feature in high_correlation_features]\n", + "high_correlation_features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, it's time to re-initialize the dataset with the highly correlated features removed.\n", + "\n", + "**Note the use of a different `input_id`.** This allows tracking the inputs used when running each individual test." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Remove the highly correlated features from the dataset\n", + "balanced_raw_no_age_df = balanced_raw_df.drop(columns=high_correlation_features)\n", + "\n", + "# Re-initialize the dataset object\n", + "vm_raw_dataset_preprocessed = vm.init_dataset(\n", + " dataset=balanced_raw_no_age_df,\n", + " input_id=\"raw_dataset_preprocessed\",\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Re-running the test with the reduced feature set should pass the test:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "corr_result = vm.tests.run_test(\n", + " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n", + " params={\"max_threshold\": 0.3},\n", + " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also plot the correlation matrix to visualize the new correlation between features:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "corr_result = vm.tests.run_test(\n", + " test_id=\"validmind.data_validation.PearsonCorrelationMatrix\",\n", + " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Documenting test results\n", + "\n", + "Now that we've done some analysis on two different datasets, we can use ValidMind to easily document why certain things were done to our raw data with testing to support it. As we learned above, every test result returned by the `run_test()` function has a `.log()` method that can be used to send the test results to the ValidMind Platform.\n", + "\n", + "When logging validation test results to the platform, you'll need to manually add those results to the desired section of the validation report. To demonstrate how to add test results to your validation report, we'll log our data quality tests and insert the results via the ValidMind Platform." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Configure and run comparison tests\n", + "\n", + "Below, we'll perform comparison tests between the original raw dataset (`raw_dataset`) and the final preprocessed (`raw_dataset_preprocessed`) dataset, again logging the results to the ValidMind Platform. \n", + "\n", + "We can specify all the tests we'd ike to run in a dictionary called `test_config`, and we'll pass in the following arguments for each test:\n", + "\n", + " - **`params`:** Individual test parameters.\n", + " - **`input_grid`:** Individual test inputs to compare. In this case, we'll input our two datasets for comparison.\n", + "\n", + "**Note here that the `input_grid` expects the `input_id` of the dataset as the value rather than the variable name we specified:**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Individual test config with inputs specified\n", + "test_config = {\n", + " \"validmind.data_validation.ClassImbalance\": {\n", + " \"input_grid\": {\"dataset\": [\"raw_dataset\", \"raw_dataset_preprocessed\"]},\n", + " \"params\": {\"min_percent_threshold\": 30}\n", + " },\n", + " \"validmind.data_validation.HighPearsonCorrelation\": {\n", + " \"input_grid\": {\"dataset\": [\"raw_dataset\", \"raw_dataset_preprocessed\"]},\n", + " \"params\": {\"max_threshold\": 0.3}\n", + " },\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then batch run and log our tests in `test_config`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for t in test_config:\n", + " print(t)\n", + " try:\n", + " # Check if test has input_grid\n", + " if 'input_grid' in test_config[t]:\n", + " # For tests with input_grid, pass the input_grid configuration\n", + " if 'params' in test_config[t]:\n", + " vm.tests.run_test(t, input_grid=test_config[t]['input_grid'], params=test_config[t]['params']).log()\n", + " else:\n", + " vm.tests.run_test(t, input_grid=test_config[t]['input_grid']).log()\n", + " else:\n", + " # Original logic for regular inputs\n", + " if 'params' in test_config[t]:\n", + " vm.tests.run_test(t, inputs=test_config[t]['inputs'], params=test_config[t]['params']).log()\n", + " else:\n", + " vm.tests.run_test(t, inputs=test_config[t]['inputs']).log()\n", + " except Exception as e:\n", + " print(f\"Error running test {t}: {str(e)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
Note the output returned indicating that a test-driven block doesn't currently exist in your model's documentation for some test IDs. \n", + "

\n", + "That's expected, as when we run validations tests the results logged need to be manually added to your report as part of your compliance assessment process within the ValidMind Platform.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Log tests with unique identifiers\n", + "\n", + "Next, we'll use the previously initialized `vm_balanced_raw_dataset` (that still has a highly correlated `Age` column) as input to run an individual test, then log the result to the ValidMind Platform.\n", + "\n", + "When running individual tests, **you can use a custom `result_id` to tag the individual result with a unique identifier:**\n", + "\n", + "- This `result_id` can be appended to `test_id` with a `:` separator.\n", + "- The `balanced_raw_dataset` result identifier will correspond to the `balanced_raw_dataset` input, the dataset that still has the `Age` column." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = vm.tests.run_test(\n", + " test_id=\"validmind.data_validation.HighPearsonCorrelation:balanced_raw_dataset\",\n", + " params={\"max_threshold\": 0.3},\n", + " inputs={\"dataset\": vm_balanced_raw_dataset},\n", + ")\n", + "result.log()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Add test results to reporting\n", + "\n", + "With some test results logged, let's head to the model we connected to at the beginning of this notebook and learn how to insert a test result into our validation report ([Need more help?](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#link-validator-evidence)).\n", + "\n", + "While the example below focuses on a specific test result, you can follow the same general procedure for your other results:\n", + "\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n", + "\n", + "2. In the left sidebar that appears for your model, click **Validation Report**.\n", + "\n", + "3. Locate the Data Preparation section and click on **2.2.1. Data Quality** to expand that section.\n", + "\n", + "4. Under the Class Imbalance Assessment section, locate Validator Evidence then click **Link Evidence to Report**:\n", + "\n", + " \"Screenshot\n", + "

\n", + "\n", + "5. Select the Class Imbalance test results we logged: **ValidMind Data Validation Class Imbalance** \n", + "\n", + " \"Screenshot\n", + "

\n", + "\n", + "6. Click **Update Linked Evidence** to add the test results to the validation report.\n", + "\n", + " Confirm that the results for the Class Imbalance test you inserted has been correctly inserted into section **2.2.1. Data Quality** of the report:\n", + "\n", + " \"Screenshot\n", + "

\n", + "\n", + "7. Note that these test results are flagged as **Requires Attention** — as they include comparative results from our initial raw dataset.\n", + "\n", + " Click **See evidence details** to review the LLM-generated description that summarizes the test results, that confirm that our final preprocessed dataset actually passes our test:\n", + "\n", + " \"Screenshot\n", + "

\n", + "\n", + "\n", + "
Here in this text editor, you can make qualitative edits to the draft that ValidMind generated to finalize the test results.\n", + "

\n", + "Learn more: Work with content blocks
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Split the preprocessed dataset\n", + "\n", + "With our raw dataset rebalanced with highly correlated features removed, let's now **spilt our dataset into train and test** in preparation for model evaluation testing.\n", + "\n", + "To start, let's grab the first few rows from the `balanced_raw_no_age_df` dataset we initialized earlier:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "balanced_raw_no_age_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Before training the model, we need to encode the categorical features in the dataset:\n", + "\n", + "- Use the `OneHotEncoder` class from the `sklearn.preprocessing` module to encode the categorical features.\n", + "- The categorical features in the dataset are `Geography` and `Gender`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "balanced_raw_no_age_df = pd.get_dummies(\n", + " balanced_raw_no_age_df, columns=[\"Geography\", \"Gender\"], drop_first=True\n", + ")\n", + "balanced_raw_no_age_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Splitting our dataset into training and testing is essential for proper validation testing, as this helps assess how well the model generalizes to unseen data:\n", + "\n", + "- We start by dividing our `balanced_raw_no_age_df` dataset into training and test subsets using `train_test_split`, with 80% of the data allocated to training (`train_df`) and 20% to testing (`test_df`).\n", + "- From each subset, we separate the features (all columns except \"Exited\") into `X_train` and `X_test`, and the target column (\"Exited\") into `y_train` and `y_test`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "train_df, test_df = train_test_split(balanced_raw_no_age_df, test_size=0.20)\n", + "\n", + "X_train = train_df.drop(\"Exited\", axis=1)\n", + "y_train = train_df[\"Exited\"]\n", + "X_test = test_df.drop(\"Exited\", axis=1)\n", + "y_test = test_df[\"Exited\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the split datasets\n", + "\n", + "Next, let's initialize the training and testing datasets so they are available for use:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vm_train_ds = vm.init_dataset(\n", + " input_id=\"train_dataset_final\",\n", + " dataset=train_df,\n", + " target_column=\"Exited\",\n", + ")\n", + "\n", + "vm_test_ds = vm.init_dataset(\n", + " input_id=\"test_dataset_final\",\n", + " dataset=test_df,\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## In summary\n", + "\n", + "In this second notebook, you learned how to:\n", + "\n", + "- [x] Import a sample dataset\n", + "- [x] Identify which tests you might want to run with ValidMind\n", + "- [x] Initialize ValidMind datasets\n", + "- [x] Run individual tests\n", + "- [x] Utilize the output from tests you’ve run\n", + "- [x] Log test results as evidence to the ValidMind Platform\n", + "- [x] Insert test results into your validation report" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Next steps\n", + "\n", + "\n", + "\n", + "### Develop potential challenger models\n", + "\n", + "Now that you're familiar with the basics of using the ValidMind Library, let's use it to develop a challenger model: **[3 — Developing a potential challenger model](3-developing_challenger_model.ipynb)**" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ValidMind Library", + "language": "python", + "name": "validmind" + }, + "language_info": { + "name": "python", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/site/notebooks/EXECUTED/model_validation/3-developing_challenger_model.ipynb b/site/notebooks/EXECUTED/model_validation/3-developing_challenger_model.ipynb new file mode 100644 index 0000000000..ba6958f286 --- /dev/null +++ b/site/notebooks/EXECUTED/model_validation/3-developing_challenger_model.ipynb @@ -0,0 +1,875 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# ValidMind for model validation 3 — Developing a potential challenger model\n", + "\n", + "Learn how to use ValidMind for your end-to-end model validation process with our series of four introductory notebooks. In this third notebook, develop a potential challenger model and then pass your model and its predictions to ValidMind.\n", + "\n", + "A *challenger model* is an alternate model that attempts to outperform the champion model, ensuring that the best performing fit-for-purpose model is always considered for deployment. Challenger models also help avoid over-reliance on a single model, and allow testing of new features, algorithms, or data sources without disrupting the production lifecycle.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for validators new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Validator Fundamentals
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [Prerequisites](#toc1_) \n", + "- [Setting up](#toc2_) \n", + " - [Initialize the ValidMind Library](#toc2_1_) \n", + " - [Import the sample dataset](#toc2_2_) \n", + " - [Preprocess the dataset](#toc2_2_1_) \n", + " - [Split the preprocessed dataset](#toc2_3_) \n", + "- [Import the champion model](#toc3_) \n", + "- [Training a potential challenger model](#toc4_) \n", + " - [Random forest classification model](#toc4_1_) \n", + "- [Initializing the model objects](#toc5_) \n", + " - [Initialize the model objects](#toc5_1_) \n", + " - [Assign predictions](#toc5_2_) \n", + "- [Running model evaluation tests](#toc6_) \n", + " - [Run model performance tests](#toc6_1_) \n", + " - [Evaluate performance of the champion model](#toc6_1_1_) \n", + " - [Log a model finding](#toc6_1_2_) \n", + " - [Evaluate performance of challenger model](#toc6_1_3_) \n", + " - [Run diagnostic tests](#toc6_2_) \n", + " - [Run feature importance tests](#toc6_3_) \n", + "- [In summary](#toc7_) \n", + "- [Next steps](#toc8_) \n", + " - [Finalize validation and reporting](#toc8_1_) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Prerequisites\n", + "\n", + "In order to develop potential challenger models with this notebook, you'll need to first have:\n", + "\n", + "- [x] Registered a model within the ValidMind Platform and granted yourself access to the model as a validator\n", + "- [x] Installed the ValidMind Library in your local environment, allowing you to access all its features\n", + "- [x] Learned how to import and initialize datasets for use with ValidMind\n", + "- [x] Understood the basics of how to run and log tests with ValidMind\n", + "- [x] Run data quality tests on the datasets used to train the champion model, and logged the results of those tests to ValidMind\n", + "- [x] Inserted your logged test results into your validation report\n", + "\n", + "
Need help with the above steps?\n", + "

\n", + "Refer to the first two notebooks in this series:\n", + "\n", + "- 1 — Set up the ValidMind Library for validation\n", + "- 2 — Start the model validation process\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Setting up\n", + "\n", + "This section should be quite familiar to you — as we performed the same actions in the previous notebook, **[2 — Start the model validation process](2-start_validation_process.ipynb)**." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind Library\n", + "\n", + "As usual, let's first connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n", + "\n", + "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", + "\n", + "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Make sure the ValidMind Library is installed\n", + "\n", + "%pip install -q validmind\n", + "\n", + "# Load your model identifier credentials from an `.env` file\n", + "\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " # api_host=\"...\",\n", + " # api_key=\"...\",\n", + " # api_secret=\"...\",\n", + " # model=\"...\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Import the sample dataset\n", + "\n", + "Next, we'll load in the sample [Bank Customer Churn Prediction](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction) dataset used to develop the champion model that we will independently preprocess:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load the sample dataset\n", + "from validmind.datasets.classification import customer_churn as demo_dataset\n", + "\n", + "print(\n", + " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n", + ")\n", + "\n", + "raw_df = demo_dataset.load_data()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Preprocess the dataset\n", + "\n", + "We’ll apply a simple rebalancing technique to the dataset before continuing:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "raw_copy_df = raw_df.sample(frac=1) # Create a copy of the raw dataset\n", + "\n", + "# Create a balanced dataset with the same number of exited and not exited customers\n", + "exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 1]\n", + "not_exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 0].sample(n=exited_df.shape[0])\n", + "\n", + "balanced_raw_df = pd.concat([exited_df, not_exited_df])\n", + "balanced_raw_df = balanced_raw_df.sample(frac=1, random_state=42)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let’s also quickly remove highly correlated features from the dataset using the output from a ValidMind test.\n", + "\n", + "As you know, before we can run tests you’ll need to initialize a ValidMind dataset object with the [`init_dataset` function](https://docs.validmind.ai/validmind/validmind.html#init_dataset):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Register new data and now 'balanced_raw_dataset' is the new dataset object of interest\n", + "vm_balanced_raw_dataset = vm.init_dataset(\n", + " dataset=balanced_raw_df,\n", + " input_id=\"balanced_raw_dataset\",\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With our balanced dataset initialized, we can then run our test and utilize the output to help us identify the features we want to remove:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Run HighPearsonCorrelation test with our balanced dataset as input and return a result object\n", + "corr_result = vm.tests.run_test(\n", + " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n", + " params={\"max_threshold\": 0.3},\n", + " inputs={\"dataset\": vm_balanced_raw_dataset},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# From result object, extract table from `corr_result.tables`\n", + "features_df = corr_result.tables[0].data\n", + "features_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Extract list of features that failed the test\n", + "high_correlation_features = features_df[features_df[\"Pass/Fail\"] == \"Fail\"][\"Columns\"].tolist()\n", + "high_correlation_features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Extract feature names from the list of strings\n", + "high_correlation_features = [feature.split(\",\")[0].strip(\"()\") for feature in high_correlation_features]\n", + "high_correlation_features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can then re-initialize the dataset with a different `input_id` and the highly correlated features removed and re-run the test for confirmation:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Remove the highly correlated features from the dataset\n", + "balanced_raw_no_age_df = balanced_raw_df.drop(columns=high_correlation_features)\n", + "\n", + "# Re-initialize the dataset object\n", + "vm_raw_dataset_preprocessed = vm.init_dataset(\n", + " dataset=balanced_raw_no_age_df,\n", + " input_id=\"raw_dataset_preprocessed\",\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Re-run the test with the reduced feature set\n", + "corr_result = vm.tests.run_test(\n", + " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n", + " params={\"max_threshold\": 0.3},\n", + " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Split the preprocessed dataset\n", + "\n", + "With our raw dataset rebalanced with highly correlated features removed, let's now **spilt our dataset into train and test** in preparation for model evaluation testing:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Encode categorical features in the dataset\n", + "balanced_raw_no_age_df = pd.get_dummies(\n", + " balanced_raw_no_age_df, columns=[\"Geography\", \"Gender\"], drop_first=True\n", + ")\n", + "balanced_raw_no_age_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "# Split the dataset into train and test\n", + "train_df, test_df = train_test_split(balanced_raw_no_age_df, test_size=0.20)\n", + "\n", + "X_train = train_df.drop(\"Exited\", axis=1)\n", + "y_train = train_df[\"Exited\"]\n", + "X_test = test_df.drop(\"Exited\", axis=1)\n", + "y_test = test_df[\"Exited\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the split datasets\n", + "vm_train_ds = vm.init_dataset(\n", + " input_id=\"train_dataset_final\",\n", + " dataset=train_df,\n", + " target_column=\"Exited\",\n", + ")\n", + "\n", + "vm_test_ds = vm.init_dataset(\n", + " input_id=\"test_dataset_final\",\n", + " dataset=test_df,\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Import the champion model\n", + "\n", + "With our raw dataset assessed and preprocessed, let's go ahead and import the champion model submitted by the model development team in the format of a `.pkl` file: **[lr_model_champion.pkl](lr_model_champion.pkl)**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import the champion model\n", + "import pickle as pkl\n", + "\n", + "with open(\"lr_model_champion.pkl\", \"rb\") as f:\n", + " log_reg = pkl.load(f)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Training a potential challenger model\n", + "\n", + "We're curious how an alternate model compares to our champion model, so let's train a challenger model as a basis for our testing.\n", + "\n", + "Our champion *logistic regression model* is a simpler, parametric model that assumes a linear relationship between the independent variables and the log-odds of the outcome. While logistic regression may not capture complex patterns as effectively, it offers a high degree of interpretability and is easier to explain to stakeholders. However, model risk is not calculated in isolation from a single factor, but rather in consideration with trade-offs in predictive performance, ease of interpretability, and overall alignment with business objectives." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Random forest classification model\n", + "\n", + "A *random forest classification model* is an ensemble machine learning algorithm that uses multiple decision trees to classify data. In ensemble learning, multiple models are combined to improve prediction accuracy and robustness.\n", + "\n", + "Random forest classification models generally have higher accuracy because they capture complex, non-linear relationships, but as a result they lack transparency in their predictions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import the Random Forest Classification model\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "\n", + "# Create the model instance with 50 decision trees\n", + "rf_model = RandomForestClassifier(\n", + " n_estimators=50,\n", + " random_state=42,\n", + ")\n", + "\n", + "# Train the model\n", + "rf_model.fit(X_train, y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Initializing the model objects" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the model objects\n", + "\n", + "In addition to the initialized datasets, you'll also need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data for each of our two models.\n", + "\n", + "You simply initialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the champion logistic regression model\n", + "vm_log_model = vm.init_model(\n", + " log_reg,\n", + " input_id=\"log_model_champion\",\n", + ")\n", + "\n", + "# Initialize the challenger random forest classification model\n", + "vm_rf_model = vm.init_model(\n", + " rf_model,\n", + " input_id=\"rf_model\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Assign predictions\n", + "\n", + "With our models registered, we'll move on to assigning both the predictive probabilities coming directly from each model's predictions, and the binary prediction after applying the cutoff threshold described in the Compute binary predictions step above.\n", + "\n", + "- The [`assign_predictions()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#assign_predictions) from the `Dataset` object can link existing predictions to any number of models.\n", + "- This method links the model's class prediction values and probabilities to our `vm_train_ds` and `vm_test_ds` datasets.\n", + "\n", + "If no prediction values are passed, the method will compute predictions automatically:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Champion — Logistic regression model\n", + "vm_train_ds.assign_predictions(model=vm_log_model)\n", + "vm_test_ds.assign_predictions(model=vm_log_model)\n", + "\n", + "# Challenger — Random forest classification model\n", + "vm_train_ds.assign_predictions(model=vm_rf_model)\n", + "vm_test_ds.assign_predictions(model=vm_rf_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Running model evaluation tests\n", + "\n", + "With everything ready for us, let's run the rest of our validation tests. We'll focus on comprehensive testing around model performance of both the champion and challenger models going forward as we've already verified the data quality of the datasets used to train the champion model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Run model performance tests\n", + "\n", + "Let's run some performance tests, beginning with independent testing of our champion logistic regression model, then moving on to our potential challenger model.\n", + "\n", + "Use [`vm.tests.list_tests()`](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to identify all the model performance tests for classification:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "vm.tests.list_tests(tags=[\"model_performance\"], task=\"classification\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll isolate the specific tests we want to run in `mpt`:\n", + "\n", + "- [`ClassifierPerformance`](https://docs.validmind.ai/tests/model_validation/sklearn/ClassifierPerformance.html)\n", + "- [`ConfusionMatrix`](https://docs.validmind.ai/tests/model_validation/sklearn/ConfusionMatrix.html)\n", + "- [`MinimumAccuracy`](https://docs.validmind.ai/tests/model_validation/sklearn/MinimumAccuracy.html)\n", + "- [`MinimumF1Score`](https://docs.validmind.ai/tests/model_validation/sklearn/MinimumF1Score.html)\n", + "- [`ROCCurve`](https://docs.validmind.ai/tests/model_validation/sklearn/ROCCurve.html)\n", + "\n", + "As we learned in the previous notebook [2 — Start the model validation process](2-start_validation_process.ipynb), you can use a custom `result_id` to tag the individual result with a unique identifier by appending this `result_id` to the `test_id` with a `:` separator. We'll append an identifier for our champion model here:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mpt = [\n", + " \"validmind.model_validation.sklearn.ClassifierPerformance:logreg_champion\",\n", + " \"validmind.model_validation.sklearn.ConfusionMatrix:logreg_champion\",\n", + " \"validmind.model_validation.sklearn.MinimumAccuracy:logreg_champion\",\n", + " \"validmind.model_validation.sklearn.MinimumF1Score:logreg_champion\",\n", + " \"validmind.model_validation.sklearn.ROCCurve:logreg_champion\"\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Evaluate performance of the champion model\n", + "\n", + "Now, let's run and log our batch of model performance tests using our testing dataset (`vm_test_ds`) for our champion model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for test in mpt:\n", + " vm.tests.run_test(\n", + " test,\n", + " inputs={\n", + " \"dataset\": vm_test_ds, \"model\" : vm_log_model,\n", + " },\n", + " ).log()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
Note the output returned indicating that a test-driven block doesn't currently exist in your model's documentation for some test IDs. \n", + "

\n", + "That's expected, as when we run validations tests the results logged need to be manually added to your report as part of your compliance assessment process within the ValidMind Platform.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Log a model finding\n", + "\n", + "As we can observe from the output above, our champion model doesn't pass the `MinimumAccuracy` based on the default thresholds of the out-of-the-box test, so let's log a model finding in the ValidMind Platform ([Need more help?](https://docs.validmind.ai/guide/model-validation/add-manage-model-findings.html)):\n", + "\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n", + "\n", + "2. In the left sidebar that appears for your model, click **Validation Report**.\n", + "\n", + "3. Locate the Data Preparation section and click on **2.2.2. Model Performance** to expand that section.\n", + "\n", + "4. Under the Model Performance Metrics section, locate Findings then click **Link Finding to Report**:\n", + "\n", + " \"Screenshot\n", + "

\n", + "\n", + "5. Click **+ Create New Finding** to add a finding.\n", + "\n", + "6. Enter in the details for your finding, for example:\n", + "\n", + " - **TITLE** — Champion Logistic Regression Model Fails Minimum Accuracy Threshold\n", + " - **RISK AREA** — Model Performance\n", + " - **DOCUMENTATION SECTION** — 3.2. Model Evaluation\n", + " - **DESCRIPTION** — The logistic regression champion model was subjected to a Minimum Accuracy test to determine whether its predictive accuracy meets the predefined performance threshold of 0.7. The model achieved an accuracy score of 0.6136, which falls below the required minimum. As a result, the test produced a Fail outcome.\n", + "\n", + "7. Click **Save**.\n", + "\n", + "8. Select the finding you just added to link to your validation report:\n", + "\n", + " \"Screenshot\n", + "

\n", + "\n", + "9. Click **Update Linked Findings** to insert your finding.\n", + "\n", + "10. Confirm that finding you inserted has been correctly inserted into section **2.2.2. Model Performance** of the report:\n", + "\n", + " \"Screenshot\n", + "

\n", + "\n", + "11. Click on the finding to expand the finding, where you can adjust details such as severity, owner, due date, status, etc. as well as include proposed remediation plans or supporting documentation as attachments." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Evaluate performance of challenger model\n", + "\n", + "We've now conducted similar tests as the model development team for our champion model, with the aim of verifying their test results.\n", + "\n", + "Next, let's see how our challenger models compare. We'll use the same batch of tests here as we did in `mpt`, but append a different `result_id` to indicate that these results should be associated with our challenger model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mpt_chall = [\n", + " \"validmind.model_validation.sklearn.ClassifierPerformance:champion_vs_challenger\",\n", + " \"validmind.model_validation.sklearn.ConfusionMatrix:champion_vs_challenger\",\n", + " \"validmind.model_validation.sklearn.MinimumAccuracy:champion_vs_challenger\",\n", + " \"validmind.model_validation.sklearn.MinimumF1Score:champion_vs_challenger\",\n", + " \"validmind.model_validation.sklearn.ROCCurve:champion_vs_challenger\"\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll run each test once for each model with the same `vm_test_ds` dataset to compare them:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for test in mpt_chall:\n", + " vm.tests.run_test(\n", + " test,\n", + " input_grid={\n", + " \"dataset\": [vm_test_ds], \"model\" : [vm_log_model,vm_rf_model]\n", + " }\n", + " ).log()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
Based on the performance metrics, our challenger random forest classification model passes the MinimumAccuracy where our champion did not.\n", + "

\n", + "In your validation report, support your recommendation in your finding's Proposed Remediation Plan to investigate the usage of our challenger model by inserting the performance tests we logged with this notebook into the appropriate section.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Run diagnostic tests\n", + "\n", + "Next we want to inspect the robustness and stability testing comparison between our champion and challenger model.\n", + "\n", + "Use `list_tests()` to identify all the model diagnosis tests for classification:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.list_tests(tags=[\"model_diagnosis\"], task=\"classification\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's see if models suffer from any *overfit* potentials and also where there are potential sub-segments of issues with the [`OverfitDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/OverfitDiagnosis.html). \n", + "\n", + "Overfitting occurs when a model learns the training data too well, capturing not only the true pattern but noise and random fluctuations resulting in excellent performance on the training dataset but poor generalization to new, unseen data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " test_id=\"validmind.model_validation.sklearn.OverfitDiagnosis:champion_vs_challenger\",\n", + " input_grid={\n", + " \"datasets\": [[vm_train_ds,vm_test_ds]],\n", + " \"model\" : [vm_log_model,vm_rf_model]\n", + " }\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's also conduct *robustness* and *stability* testing of the two models with the [`RobustnessDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/RobustnessDiagnosis.html).\n", + "\n", + "Robustness refers to a model's ability to maintain consistent performance, and stability refers to a model's ability to produce consistent outputs over time across different data subsets." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " test_id=\"validmind.model_validation.sklearn.RobustnessDiagnosis:Champion_vs_LogRegression\",\n", + " input_grid={\n", + " \"datasets\": [[vm_train_ds,vm_test_ds]],\n", + " \"model\" : [vm_log_model,vm_rf_model]\n", + " },\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Run feature importance tests\n", + "\n", + "We also want to verify the relative influence of different input features on our models' predictions, as well as inspect the differences between our champion and challenger model to see if a certain model offers more understandable or logical importance scores for features.\n", + "\n", + "Use `list_tests()` to identify all the feature importance tests for classification:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Store the feature importance tests\n", + "FI = vm.tests.list_tests(tags=[\"feature_importance\"], task=\"classification\",pretty=False)\n", + "FI" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Run and log our feature importance tests for both models for the testing dataset\n", + "for test in FI:\n", + " vm.tests.run_test(\n", + " \"\".join((test,':champion_vs_challenger')),\n", + " input_grid={\n", + " \"dataset\": [vm_test_ds], \"model\" : [vm_log_model,vm_rf_model]\n", + " },\n", + " ).log()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## In summary\n", + "\n", + "In this third notebook, you learned how to:\n", + "\n", + "- [x] Initialize ValidMind model objects\n", + "- [x] Assign predictions and probabilities to your ValidMind model objects\n", + "- [x] Use tests from ValidMind to evaluate the potential of models, including comparative tests between champion and challenger models\n", + "- [x] Log a model finding in the ValidMind Platform" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Next steps\n", + "\n", + "\n", + "\n", + "### Finalize validation and reporting\n", + "\n", + "Now that you're familiar with the basics of using the ValidMind Library to run and log validation tests, let's learn how to implement some custom tests and wrap up our validation: **[4 — Finalize validation and reporting](4-finalize_validation_reporting.ipynb)**" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ValidMind Library", + "language": "python", + "name": "validmind" + }, + "language_info": { + "name": "python", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/site/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.ipynb b/site/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.ipynb new file mode 100644 index 0000000000..886f9e9061 --- /dev/null +++ b/site/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.ipynb @@ -0,0 +1,1211 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# ValidMind for model validation 4 — Finalize testing and reporting\n", + "\n", + "Learn how to use ValidMind for your end-to-end model validation process with our series of four introductory notebooks. In this last notebook, finalize the compliance assessment process and have a complete validation report ready for review.\n", + "\n", + "This notebook will walk you through how to supplement ValidMind tests with your own custom tests and include them as additional evidence in your validation report. A custom test is any function that takes a set of inputs and parameters as arguments and returns one or more outputs:\n", + "\n", + "- The function can be as simple or as complex as you need it to be — it can use external libraries, make API calls, or do anything else that you can do in Python.\n", + "- The only requirement is that the function signature and return values can be \"understood\" and handled by the ValidMind Library. As such, custom tests offer added flexibility by extending the default tests provided by ValidMind, enabling you to document any type of model or use case.\n", + "\n", + "**For a more in-depth introduction to custom tests,** refer to our [Implement custom tests](../../code_samples/custom_tests/implement_custom_tests.ipynb) notebook.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for validators new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Validator Fundamentals
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [Prerequisites](#toc1_) \n", + "- [Setting up](#toc2_) \n", + " - [Initialize the ValidMind Library](#toc2_1_) \n", + " - [Import the sample dataset](#toc2_2_) \n", + " - [Split the preprocessed dataset](#toc2_3_) \n", + " - [Import the champion model](#toc2_4_) \n", + " - [Train potential challenger model](#toc2_5_) \n", + " - [Initialize the model objects](#toc2_6_) \n", + "- [Implementing custom tests](#toc3_) \n", + " - [Implement a custom inline test](#toc3_1_) \n", + " - [Create a confusion matrix plot](#toc3_1_1_) \n", + " - [Add parameters to custom tests](#toc3_1_2_) \n", + " - [Pass parameters to custom tests](#toc3_1_3_) \n", + " - [Use external test providers](#toc3_2_) \n", + " - [Create custom tests folder](#toc3_2_1_) \n", + " - [Save an inline test](#toc3_2_2_) \n", + " - [Register a local test provider](#toc3_2_3_) \n", + "- [Verify test runs](#toc4_) \n", + "- [In summary](#toc5_) \n", + "- [Next steps](#toc6_) \n", + " - [Work with your validation report](#toc6_1_) \n", + " - [Learn more](#toc6_2_) \n", + " - [More how-to guides and code samples](#toc6_2_1_) \n", + " - [Discover more learning resources](#toc6_2_2_) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Prerequisites\n", + "\n", + "In order to finalize validation and reporting, you'll need to first have:\n", + "\n", + "- [x] Registered a model within the ValidMind Platform and granted yourself access to the model as a validator\n", + "- [x] Installed the ValidMind Library in your local environment, allowing you to access all its features\n", + "- [x] Learned how to import and initialize datasets and models for use with ValidMind\n", + "- [x] Understood the basics of how to identify and run validation tests\n", + "- [x] Run validation tests for your champion and challenger models, and logged the results of those tests to the ValidMind Platform\n", + "- [x] Inserted your logged test results into your validation report\n", + "- [x] Added some preliminary findings to your validation report\n", + "\n", + "
Need help with the above steps?\n", + "

\n", + "Refer to the first three notebooks in this series:\n", + "\n", + "- 1 — Set up the ValidMind Library for validation\n", + "- 2 — Start the model validation process\n", + "- 2 — Developing a potential challenger model\n", + "\n", + "
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Setting up\n", + "\n", + "This section should be very familiar to you now — as we performed the same actions in the previous two notebooks in this series." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind Library\n", + "\n", + "As usual, let's first connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n", + "\n", + "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", + "\n", + "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Make sure the ValidMind Library is installed\n", + "\n", + "%pip install -q validmind\n", + "\n", + "# Load your model identifier credentials from an `.env` file\n", + "\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " # api_host=\"...\",\n", + " # api_key=\"...\",\n", + " # api_secret=\"...\",\n", + " # model=\"...\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Import the sample dataset\n", + "\n", + "Next, we'll load in the same sample [Bank Customer Churn Prediction](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction) dataset used to develop the champion model that we will independently preprocess:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load the sample dataset\n", + "from validmind.datasets.classification import customer_churn as demo_dataset\n", + "\n", + "print(\n", + " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n", + ")\n", + "\n", + "raw_df = demo_dataset.load_data()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the raw dataset for use in ValidMind tests\n", + "vm_raw_dataset = vm.init_dataset(\n", + " dataset=raw_df,\n", + " input_id=\"raw_dataset\",\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "raw_copy_df = raw_df.sample(frac=1) # Create a copy of the raw dataset\n", + "\n", + "# Create a balanced dataset with the same number of exited and not exited customers\n", + "exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 1]\n", + "not_exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 0].sample(n=exited_df.shape[0])\n", + "\n", + "balanced_raw_df = pd.concat([exited_df, not_exited_df])\n", + "balanced_raw_df = balanced_raw_df.sample(frac=1, random_state=42)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let’s also quickly remove highly correlated features from the dataset using the output from a ValidMind test:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Register new data and now 'balanced_raw_dataset' is the new dataset object of interest\n", + "vm_balanced_raw_dataset = vm.init_dataset(\n", + " dataset=balanced_raw_df,\n", + " input_id=\"balanced_raw_dataset\",\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Run HighPearsonCorrelation test with our balanced dataset as input and return a result object\n", + "corr_result = vm.tests.run_test(\n", + " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n", + " params={\"max_threshold\": 0.3},\n", + " inputs={\"dataset\": vm_balanced_raw_dataset},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# From result object, extract table from `corr_result.tables`\n", + "features_df = corr_result.tables[0].data\n", + "features_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Extract list of features that failed the test\n", + "high_correlation_features = features_df[features_df[\"Pass/Fail\"] == \"Fail\"][\"Columns\"].tolist()\n", + "high_correlation_features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Extract feature names from the list of strings\n", + "high_correlation_features = [feature.split(\",\")[0].strip(\"()\") for feature in high_correlation_features]\n", + "high_correlation_features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Remove the highly correlated features from the dataset\n", + "balanced_raw_no_age_df = balanced_raw_df.drop(columns=high_correlation_features)\n", + "\n", + "# Re-initialize the dataset object\n", + "vm_raw_dataset_preprocessed = vm.init_dataset(\n", + " dataset=balanced_raw_no_age_df,\n", + " input_id=\"raw_dataset_preprocessed\",\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Re-run the test with the reduced feature set\n", + "corr_result = vm.tests.run_test(\n", + " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n", + " params={\"max_threshold\": 0.3},\n", + " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Split the preprocessed dataset\n", + "\n", + "With our raw dataset rebalanced with highly correlated features removed, let's now **spilt our dataset into train and test** in preparation for model evaluation testing:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Encode categorical features in the dataset\n", + "balanced_raw_no_age_df = pd.get_dummies(\n", + " balanced_raw_no_age_df, columns=[\"Geography\", \"Gender\"], drop_first=True\n", + ")\n", + "balanced_raw_no_age_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "# Split the dataset into train and test\n", + "train_df, test_df = train_test_split(balanced_raw_no_age_df, test_size=0.20)\n", + "\n", + "X_train = train_df.drop(\"Exited\", axis=1)\n", + "y_train = train_df[\"Exited\"]\n", + "X_test = test_df.drop(\"Exited\", axis=1)\n", + "y_test = test_df[\"Exited\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the split datasets\n", + "vm_train_ds = vm.init_dataset(\n", + " input_id=\"train_dataset_final\",\n", + " dataset=train_df,\n", + " target_column=\"Exited\",\n", + ")\n", + "\n", + "vm_test_ds = vm.init_dataset(\n", + " input_id=\"test_dataset_final\",\n", + " dataset=test_df,\n", + " target_column=\"Exited\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Import the champion model\n", + "\n", + "With our raw dataset assessed and preprocessed, let's go ahead and import the champion model submitted by the model development team in the format of a `.pkl` file: **[lr_model_champion.pkl](lr_model_champion.pkl)**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import the champion model\n", + "import pickle as pkl\n", + "\n", + "with open(\"lr_model_champion.pkl\", \"rb\") as f:\n", + " log_reg = pkl.load(f)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Train potential challenger model\n", + "\n", + "We'll also train our random forest classification challenger model to see how it compares:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import the Random Forest Classification model\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "\n", + "# Create the model instance with 50 decision trees\n", + "rf_model = RandomForestClassifier(\n", + " n_estimators=50,\n", + " random_state=42,\n", + ")\n", + "\n", + "# Train the model\n", + "rf_model.fit(X_train, y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the model objects\n", + "\n", + "In addition to the initialized datasets, you'll also need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data for each of our two models:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the champion logistic regression model\n", + "vm_log_model = vm.init_model(\n", + " log_reg,\n", + " input_id=\"log_model_champion\",\n", + ")\n", + "\n", + "# Initialize the challenger random forest classification model\n", + "vm_rf_model = vm.init_model(\n", + " rf_model,\n", + " input_id=\"rf_model\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assign predictions to Champion — Logistic regression model\n", + "vm_train_ds.assign_predictions(model=vm_log_model)\n", + "vm_test_ds.assign_predictions(model=vm_log_model)\n", + "\n", + "# Assign predictions to Challenger — Random forest classification model\n", + "vm_train_ds.assign_predictions(model=vm_rf_model)\n", + "vm_test_ds.assign_predictions(model=vm_rf_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Implementing custom tests\n", + "\n", + "Thanks to the model documentation ([Learn more ...](https://docs.validmind.ai/developer/validmind-library.html#for-model-development)), we know that the model development team implemented a custom test to further evaluate the performance of the champion model.\n", + "\n", + "In a usual model validation situation, you would load a saved custom test provided by the model development team. In the following section, we'll have you implement the same custom test and make it available for reuse, to familiarize you with the processes.\n", + "\n", + "
Want to learn more about custom tests?\n", + "

\n", + "Refer to our in-depth introduction to custom tests: Implement custom tests
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Implement a custom inline test\n", + "\n", + "Let's implement the same custom *inline test* that calculates the confusion matrix for a binary classification model that the model development team used in their performance evaluations.\n", + "\n", + "- An inline test refers to a test written and executed within the same environment as the code being tested — in this case, right in this Jupyter Notebook — without requiring a separate test file or framework.\n", + "- You'll note that the custom test function is just a regular Python function that can include and require any Python library as you see fit." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Create a confusion matrix plot\n", + "\n", + "Let's first create a confusion matrix plot using the `confusion_matrix` function from the `sklearn.metrics` module:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "from sklearn import metrics\n", + "\n", + "# Get the predicted classes\n", + "y_pred = log_reg.predict(vm_test_ds.x)\n", + "\n", + "confusion_matrix = metrics.confusion_matrix(y_test, y_pred)\n", + "\n", + "cm_display = metrics.ConfusionMatrixDisplay(\n", + " confusion_matrix=confusion_matrix, display_labels=[False, True]\n", + ")\n", + "cm_display.plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, create a [`@vm.test` wrapper](https://docs.validmind.ai/validmind/validmind.html#test) that will allow you to create a reusable test. **Note the following changes in the code below:**\n", + "\n", + "- The function `confusion_matrix` takes two arguments `dataset` and `model`. This is a `VMDataset` and `VMModel` object respectively.\n", + " - `VMDataset` objects allow you to access the dataset's true (target) values by accessing the `.y` attribute.\n", + " - `VMDataset` objects allow you to access the predictions for a given model by accessing the `.y_pred()` method.\n", + "- The function docstring provides a description of what the test does. This will be displayed along with the result in this notebook as well as in the ValidMind Platform.\n", + "- The function body calculates the confusion matrix using the `sklearn.metrics.confusion_matrix` function as we just did above.\n", + "- The function then returns the `ConfusionMatrixDisplay.figure_` object — this is important as the ValidMind Library expects the output of the custom test to be a plot or a table.\n", + "- The `@vm.test` decorator is doing the work of creating a wrapper around the function that will allow it to be run by the ValidMind Library. It also registers the test so it can be found by the ID `my_custom_tests.ConfusionMatrix`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@vm.test(\"my_custom_tests.ConfusionMatrix\")\n", + "def confusion_matrix(dataset, model):\n", + " \"\"\"The confusion matrix is a table that is often used to describe the performance of a classification model on a set of data for which the true values are known.\n", + "\n", + " The confusion matrix is a 2x2 table that contains 4 values:\n", + "\n", + " - True Positive (TP): the number of correct positive predictions\n", + " - True Negative (TN): the number of correct negative predictions\n", + " - False Positive (FP): the number of incorrect positive predictions\n", + " - False Negative (FN): the number of incorrect negative predictions\n", + "\n", + " The confusion matrix can be used to assess the holistic performance of a classification model by showing the accuracy, precision, recall, and F1 score of the model on a single figure.\n", + " \"\"\"\n", + " y_true = dataset.y\n", + " y_pred = dataset.y_pred(model=model)\n", + "\n", + " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n", + "\n", + " cm_display = metrics.ConfusionMatrixDisplay(\n", + " confusion_matrix=confusion_matrix, display_labels=[False, True]\n", + " )\n", + " cm_display.plot()\n", + "\n", + " plt.close() # close the plot to avoid displaying it\n", + "\n", + " return cm_display.figure_ # return the figure object itself" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can now run the newly created custom test on both the training and test datasets for both models using the [`run_test()` function](https://docs.validmind.ai/validmind/validmind/tests.html#run_test):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Champion train and test\n", + "vm.tests.run_test(\n", + " test_id=\"my_custom_tests.ConfusionMatrix:champion\",\n", + " input_grid={\n", + " \"dataset\": [vm_train_ds,vm_test_ds],\n", + " \"model\" : [vm_log_model]\n", + " }\n", + ").log()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Challenger train and test\n", + "vm.tests.run_test(\n", + " test_id=\"my_custom_tests.ConfusionMatrix:challenger\",\n", + " input_grid={\n", + " \"dataset\": [vm_train_ds,vm_test_ds],\n", + " \"model\" : [vm_rf_model]\n", + " }\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
Note the output returned indicating that a test-driven block doesn't currently exist in your model's documentation for some test IDs. \n", + "

\n", + "That's expected, as when we run validations tests the results logged need to be manually added to your report as part of your compliance assessment process within the ValidMind Platform.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Add parameters to custom tests\n", + "\n", + "Custom tests can take parameters just like any other function. To demonstrate, let's modify the `confusion_matrix` function to take an additional parameter `normalize` that will allow you to normalize the confusion matrix:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@vm.test(\"my_custom_tests.ConfusionMatrix\")\n", + "def confusion_matrix(dataset, model, normalize=False):\n", + " \"\"\"The confusion matrix is a table that is often used to describe the performance of a classification model on a set of data for which the true values are known.\n", + "\n", + " The confusion matrix is a 2x2 table that contains 4 values:\n", + "\n", + " - True Positive (TP): the number of correct positive predictions\n", + " - True Negative (TN): the number of correct negative predictions\n", + " - False Positive (FP): the number of incorrect positive predictions\n", + " - False Negative (FN): the number of incorrect negative predictions\n", + "\n", + " The confusion matrix can be used to assess the holistic performance of a classification model by showing the accuracy, precision, recall, and F1 score of the model on a single figure.\n", + " \"\"\"\n", + " y_true = dataset.y\n", + " y_pred = dataset.y_pred(model=model)\n", + "\n", + " if normalize:\n", + " confusion_matrix = metrics.confusion_matrix(y_true, y_pred, normalize=\"all\")\n", + " else:\n", + " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n", + "\n", + " cm_display = metrics.ConfusionMatrixDisplay(\n", + " confusion_matrix=confusion_matrix, display_labels=[False, True]\n", + " )\n", + " cm_display.plot()\n", + "\n", + " plt.close() # close the plot to avoid displaying it\n", + "\n", + " return cm_display.figure_ # return the figure object itself" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Pass parameters to custom tests\n", + "\n", + "You can pass parameters to custom tests by providing a dictionary of parameters to the `run_test()` function.\n", + "\n", + "- The parameters will override any default parameters set in the custom test definition. Note that `dataset` and `model` are still passed as `inputs`.\n", + "- Since these are `VMDataset` or `VMModel` inputs, they have a special meaning.\n", + "\n", + "Re-running and logging the custom confusion matrix with `normalize=True` for both models and our testing dataset looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Champion with test dataset and normalize=True\n", + "vm.tests.run_test(\n", + " test_id=\"my_custom_tests.ConfusionMatrix:test_normalized_champion\",\n", + " input_grid={\n", + " \"dataset\": [vm_test_ds],\n", + " \"model\" : [vm_log_model]\n", + " },\n", + " params={\"normalize\": True}\n", + ").log()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Challenger with test dataset and normalize=True\n", + "vm.tests.run_test(\n", + " test_id=\"my_custom_tests.ConfusionMatrix:test_normalized_challenger\",\n", + " input_grid={\n", + " \"dataset\": [vm_test_ds],\n", + " \"model\" : [vm_rf_model]\n", + " },\n", + " params={\"normalize\": True}\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Use external test providers\n", + "\n", + "Sometimes you may want to reuse the same set of custom tests across multiple models and share them with others in your organization, like the model development team would have done with you in this example workflow featured in this series of notebooks. In this case, you can create an external custom *test provider* that will allow you to load custom tests from a local folder or a Git repository.\n", + "\n", + "In this section you will learn how to declare a local filesystem test provider that allows loading tests from a local folder following these high level steps:\n", + "\n", + "1. Create a folder of custom tests from existing inline tests (tests that exist in your active Jupyter Notebook)\n", + "2. Save an inline test to a file\n", + "3. Define and register a [`LocalTestProvider`](https://docs.validmind.ai/validmind/validmind/tests.html#LocalTestProvider) that points to that folder\n", + "4. Run test provider tests\n", + "5. Add the test results to your documentation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Create custom tests folder\n", + "\n", + "Let's start by creating a new folder that will contain reusable custom tests from your existing inline tests.\n", + "\n", + "The following code snippet will create a new `my_tests` directory in the current working directory if it doesn't exist:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tests_folder = \"my_tests\"\n", + "\n", + "import os\n", + "\n", + "# create tests folder\n", + "os.makedirs(tests_folder, exist_ok=True)\n", + "\n", + "# remove existing tests\n", + "for f in os.listdir(tests_folder):\n", + " # remove files and pycache\n", + " if f.endswith(\".py\") or f == \"__pycache__\":\n", + " os.system(f\"rm -rf {tests_folder}/{f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After running the command above, confirm that a new `my_tests` directory was created successfully. For example:\n", + "\n", + "```\n", + "~/notebooks/tutorials/model_validation/my_tests/\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Save an inline test\n", + "\n", + "The `@vm.test` decorator we used in **Implement a custom inline test** above to register one-off custom tests also includes a convenience method on the function object that allows you to simply call `.save()` to save the test to a Python file at a specified path.\n", + "\n", + "While `save()` will get you started by creating the file and saving the function code with the correct name, it won't automatically include any imports, or other functions or variables, outside of the functions that are needed for the test to run. To solve this, pass in an optional `imports` argument ensuring necessary imports are added to the file.\n", + "\n", + "The `confusion_matrix` test requires the following additional imports:\n", + "\n", + "```python\n", + "import matplotlib.pyplot as plt\n", + "from sklearn import metrics\n", + "```\n", + "\n", + "Let's pass these imports to the `save()` method to ensure they are included in the file with the following command:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "confusion_matrix.save(\n", + " # Save it to the custom tests folder we created\n", + " tests_folder,\n", + " imports=[\"import matplotlib.pyplot as plt\", \"from sklearn import metrics\"],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- [x] Confirm that the `save()` method saved the `confusion_matrix` function to a file named `ConfusionMatrix.py` in the `my_tests` folder.\n", + "- [x] Note that the new file provides some context on the origin of the test, which is useful for traceability:\n", + "\n", + " ```\n", + " # Saved from __main__.confusion_matrix\n", + " # Original Test ID: my_custom_tests.ConfusionMatrix\n", + " # New Test ID: .ConfusionMatrix\n", + " ```\n", + "\n", + "- [x] Additionally, the new test function has been stripped off its decorator, as it now resides in a file that will be loaded by the test provider:\n", + "\n", + " ```python\n", + " def ConfusionMatrix(dataset, model, normalize=False):\n", + " ```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Register a local test provider\n", + "\n", + "Now that your `my_tests` folder has a sample custom test, let's initialize a test provider that will tell the ValidMind Library where to find your custom tests:\n", + "\n", + "- ValidMind offers out-of-the-box test providers for local tests (tests in a folder) or a Github provider for tests in a Github repository.\n", + "- You can also create your own test provider by creating a class that has a [`load_test` method](https://docs.validmind.ai/validmind/validmind/tests.html#load_test) that takes a test ID and returns the test function matching that ID.\n", + "\n", + "
Want to learn more about test providers?\n", + "

\n", + "An extended introduction to test providers can be found in: Integrate external test providers
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Initialize a local test provider\n", + "\n", + "For most use cases, using a `LocalTestProvider` that allows you to load custom tests from a designated directory should be sufficient.\n", + "\n", + "**The most important attribute for a test provider is its `namespace`.** This is a string that will be used to prefix test IDs in model documentation. This allows you to have multiple test providers with tests that can even share the same ID, but are distinguished by their namespace.\n", + "\n", + "Let's go ahead and load the custom tests from our `my_tests` directory:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.tests import LocalTestProvider\n", + "\n", + "# initialize the test provider with the tests folder we created earlier\n", + "my_test_provider = LocalTestProvider(tests_folder)\n", + "\n", + "vm.tests.register_test_provider(\n", + " namespace=\"my_test_provider\",\n", + " test_provider=my_test_provider,\n", + ")\n", + "# `my_test_provider.load_test()` will be called for any test ID that starts with `my_test_provider`\n", + "# e.g. `my_test_provider.ConfusionMatrix` will look for a function named `ConfusionMatrix` in `my_tests/ConfusionMatrix.py` file" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Run test provider tests\n", + "\n", + "Now that we've set up the test provider, we can run any test that's located in the tests folder by using the `run_test()` method as with any other test:\n", + "\n", + "- For tests that reside in a test provider directory, the test ID will be the `namespace` specified when registering the provider, followed by the path to the test file relative to the tests folder.\n", + "- For example, the Confusion Matrix test we created earlier will have the test ID `my_test_provider.ConfusionMatrix`. You could organize the tests in subfolders, say `classification` and `regression`, and the test ID for the Confusion Matrix test would then be `my_test_provider.classification.ConfusionMatrix`.\n", + "\n", + "Let's go ahead and re-run the confusion matrix test with our testing dataset for our two models by using the test ID `my_test_provider.ConfusionMatrix`. This should load the test from the test provider and run it as before." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Champion with test dataset and test provider custom test\n", + "vm.tests.run_test(\n", + " test_id=\"my_test_provider.ConfusionMatrix:champion\",\n", + " input_grid={\n", + " \"dataset\": [vm_test_ds],\n", + " \"model\" : [vm_log_model]\n", + " }\n", + ").log()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Challenger with test dataset and test provider custom test\n", + "vm.tests.run_test(\n", + " test_id=\"my_test_provider.ConfusionMatrix:challenger\",\n", + " input_grid={\n", + " \"dataset\": [vm_test_ds],\n", + " \"model\" : [vm_rf_model]\n", + " }\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Verify test runs\n", + "\n", + "Our final task is to verify that all the tests provided by the model development team were run and reported accurately. Note the appended `result_ids` to delineate which dataset we ran the test with for the relevant tests.\n", + "\n", + "Here, we'll specify all the tests we'd like to independently rerun in a dictionary called `test_config`. **Note here that `inputs` and `input_grid` expect the `input_id` of the dataset or model as the value rather than the variable name we specified**:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_config = {\n", + " # Run with the raw dataset\n", + " 'validmind.data_validation.DatasetDescription:raw_data': {\n", + " 'inputs': {'dataset': 'raw_dataset'}\n", + " },\n", + " 'validmind.data_validation.DescriptiveStatistics:raw_data': {\n", + " 'inputs': {'dataset': 'raw_dataset'}\n", + " },\n", + " 'validmind.data_validation.MissingValues:raw_data': {\n", + " 'inputs': {'dataset': 'raw_dataset'},\n", + " 'params': {'min_threshold': 1}\n", + " },\n", + " 'validmind.data_validation.ClassImbalance:raw_data': {\n", + " 'inputs': {'dataset': 'raw_dataset'},\n", + " 'params': {'min_percent_threshold': 10}\n", + " },\n", + " 'validmind.data_validation.Duplicates:raw_data': {\n", + " 'inputs': {'dataset': 'raw_dataset'},\n", + " 'params': {'min_threshold': 1}\n", + " },\n", + " 'validmind.data_validation.HighCardinality:raw_data': {\n", + " 'inputs': {'dataset': 'raw_dataset'},\n", + " 'params': {\n", + " 'num_threshold': 100,\n", + " 'percent_threshold': 0.1,\n", + " 'threshold_type': 'percent'\n", + " }\n", + " },\n", + " 'validmind.data_validation.Skewness:raw_data': {\n", + " 'inputs': {'dataset': 'raw_dataset'},\n", + " 'params': {'max_threshold': 1}\n", + " },\n", + " 'validmind.data_validation.UniqueRows:raw_data': {\n", + " 'inputs': {'dataset': 'raw_dataset'},\n", + " 'params': {'min_percent_threshold': 1}\n", + " },\n", + " 'validmind.data_validation.TooManyZeroValues:raw_data': {\n", + " 'inputs': {'dataset': 'raw_dataset'},\n", + " 'params': {'max_percent_threshold': 0.03}\n", + " },\n", + " 'validmind.data_validation.IQROutliersTable:raw_data': {\n", + " 'inputs': {'dataset': 'raw_dataset'},\n", + " 'params': {'threshold': 5}\n", + " },\n", + " # Run with the preprocessed dataset\n", + " 'validmind.data_validation.DescriptiveStatistics:preprocessed_data': {\n", + " 'inputs': {'dataset': 'raw_dataset_preprocessed'}\n", + " },\n", + " 'validmind.data_validation.TabularDescriptionTables:preprocessed_data': {\n", + " 'inputs': {'dataset': 'raw_dataset_preprocessed'}\n", + " },\n", + " 'validmind.data_validation.MissingValues:preprocessed_data': {\n", + " 'inputs': {'dataset': 'raw_dataset_preprocessed'},\n", + " 'params': {'min_threshold': 1}\n", + " },\n", + " 'validmind.data_validation.TabularNumericalHistograms:preprocessed_data': {\n", + " 'inputs': {'dataset': 'raw_dataset_preprocessed'}\n", + " },\n", + " 'validmind.data_validation.TabularCategoricalBarPlots:preprocessed_data': {\n", + " 'inputs': {'dataset': 'raw_dataset_preprocessed'}\n", + " },\n", + " 'validmind.data_validation.TargetRateBarPlots:preprocessed_data': {\n", + " 'inputs': {'dataset': 'raw_dataset_preprocessed'},\n", + " 'params': {'default_column': 'loan_status'}\n", + " },\n", + " # Run with the training and test datasets\n", + " 'validmind.data_validation.DescriptiveStatistics:development_data': {\n", + " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']}\n", + " },\n", + " 'validmind.data_validation.TabularDescriptionTables:development_data': {\n", + " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']}\n", + " },\n", + " 'validmind.data_validation.ClassImbalance:development_data': {\n", + " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']},\n", + " 'params': {'min_percent_threshold': 10}\n", + " },\n", + " 'validmind.data_validation.UniqueRows:development_data': {\n", + " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']},\n", + " 'params': {'min_percent_threshold': 1}\n", + " },\n", + " 'validmind.data_validation.TabularNumericalHistograms:development_data': {\n", + " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']}\n", + " },\n", + " 'validmind.data_validation.MutualInformation:development_data': {\n", + " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']},\n", + " 'params': {'min_threshold': 0.01}\n", + " },\n", + " 'validmind.data_validation.PearsonCorrelationMatrix:development_data': {\n", + " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']}\n", + " },\n", + " 'validmind.data_validation.HighPearsonCorrelation:development_data': {\n", + " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final']},\n", + " 'params': {'max_threshold': 0.3, 'top_n_correlations': 10}\n", + " },\n", + " 'validmind.model_validation.ModelMetadata': {\n", + " 'input_grid': {'model': ['log_model_champion', 'rf_model']}\n", + " },\n", + " 'validmind.model_validation.sklearn.ModelParameters': {\n", + " 'input_grid': {'model': ['log_model_champion', 'rf_model']}\n", + " },\n", + " 'validmind.model_validation.sklearn.ROCCurve': {\n", + " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final'], 'model': ['log_model_champion']}\n", + " },\n", + " 'validmind.model_validation.sklearn.MinimumROCAUCScore': {\n", + " 'input_grid': {'dataset': ['train_dataset_final', 'test_dataset_final'], 'model': ['log_model_champion']},\n", + " 'params': {'min_threshold': 0.5}\n", + " }\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then batch run and log our tests in `test_config`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for t in test_config:\n", + " print(t)\n", + " try:\n", + " # Check if test has input_grid\n", + " if 'input_grid' in test_config[t]:\n", + " # For tests with input_grid, pass the input_grid configuration\n", + " if 'params' in test_config[t]:\n", + " vm.tests.run_test(t, input_grid=test_config[t]['input_grid'], params=test_config[t]['params']).log()\n", + " else:\n", + " vm.tests.run_test(t, input_grid=test_config[t]['input_grid']).log()\n", + " else:\n", + " # Original logic for regular inputs\n", + " if 'params' in test_config[t]:\n", + " vm.tests.run_test(t, inputs=test_config[t]['inputs'], params=test_config[t]['params']).log()\n", + " else:\n", + " vm.tests.run_test(t, inputs=test_config[t]['inputs']).log()\n", + " except Exception as e:\n", + " print(f\"Error running test {t}: {str(e)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## In summary\n", + "\n", + "In this final notebook, you learned how to:\n", + "\n", + "- [x] Implement a custom inline test\n", + "- [x] Run and log your custom inline tests\n", + "- [x] Use external custom test providers\n", + "- [x] Run and log tests from your custom test providers\n", + "- [x] Re-run tests provided by your model development team to verify that they were run and reported accurately\n", + "\n", + "With our ValidMind for model validation series of notebooks, you learned how to validate a model end-to-end with the ValidMind Library by running through some common scenarios in a typical model validation setting:\n", + "\n", + "- Verifying the data quality steps performed by the model development team\n", + "- Independently replicating the champion model's results and conducting additional tests to assess performance, stability, and robustness\n", + "- Setting up test inputs and a challenger model for comparative analysis\n", + "- Running validation tests, analyzing results, and logging findings to ValidMind" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Next steps" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Work with your validation report\n", + "\n", + "Now that you've logged all your test results and verified the work done by the model development team, head to the ValidMind Platform to wrap up your validation report. Continue to work on your validation report by:\n", + "\n", + "- **Inserting additional test results:** Click **Link Evidence to Report** under any section of 2. Validation in your validation report. (Learn more: [Link evidence to reports](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#link-evidence-to-reports))\n", + "\n", + "- **Making qualitative edits to your test descriptions:** Expand any linked evidence under Validator Evidence and click **See evidence details** to review and edit the ValidMind-generated test descriptions for quality and accuracy.\n", + "\n", + "- **Adding more findings:** Click **Link Finding to Report** in any validation report section, then click **+ Create New Finding**. (Learn more: [Add and manage model findings](https://docs.validmind.ai/guide/model-validation/add-manage-model-findings.html))\n", + "\n", + "- **Adding risk assessment notes:** Click under **Risk Assessment Notes** in any validation report section to access the text editor and content editing toolbar, including an option to generate a draft with AI. Edit your ValidMind-generated test descriptions (Learn more: [Work with content blocks](https://docs.validmind.ai/guide/model-documentation/work-with-content-blocks.html#content-editing-toolbar))\n", + "\n", + "- **Assessing compliance:** Under the Guideline for any validation report section, click **ASSESSMENT** and select the compliance status from the drop-down menu. (Learn more: [Provide compliance assessments](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#provide-compliance-assessments))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Learn more\n", + "\n", + "Now that you're familiar with the basics, you can explore the following notebooks to get a deeper understanding on how the ValidMind Library assists you in streamlining model validation:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### More how-to guides and code samples\n", + "\n", + "- [Explore available tests in detail](../../how_to/explore_tests.ipynb)\n", + "- [In-depth guide on running dataset based tests](../../how_to/run_tests/1_run_dataset_based_tests.ipynb)\n", + "- [In-depth guide for running comparison tests](../../how_to/run_tests/2_run_comparison_tests.ipynb)\n", + "- [In-depth guide for implementing custom tests](../../code_samples/custom_tests/implement_custom_tests.ipynb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Discover more learning resources\n", + "\n", + "All notebook samples can be found in the following directories of the ValidMind Library GitHub repository:\n", + "\n", + "- [Code samples](https://github.com/validmind/validmind-library/tree/main/notebooks/code_samples)\n", + "- [How-to guides](https://github.com/validmind/validmind-library/tree/main/notebooks/how_to)\n", + "\n", + "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ValidMind Library", + "language": "python", + "name": "validmind" + }, + "language_info": { + "name": "python", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/site/notebooks/EXECUTED/model_validation/class-imbalance-results-detail.png b/site/notebooks/EXECUTED/model_validation/class-imbalance-results-detail.png new file mode 100644 index 0000000000..00251ecaac Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/class-imbalance-results-detail.png differ diff --git a/site/notebooks/EXECUTED/model_validation/compliance-summary.png b/site/notebooks/EXECUTED/model_validation/compliance-summary.png new file mode 100644 index 0000000000..a8abd2dd39 Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/compliance-summary.png differ diff --git a/site/notebooks/EXECUTED/model_validation/inserted-class-imbalance-results.png b/site/notebooks/EXECUTED/model_validation/inserted-class-imbalance-results.png new file mode 100644 index 0000000000..2efea1a09b Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/inserted-class-imbalance-results.png differ diff --git a/site/notebooks/EXECUTED/model_validation/inserted-finding.png b/site/notebooks/EXECUTED/model_validation/inserted-finding.png new file mode 100644 index 0000000000..bb78436532 Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/inserted-finding.png differ diff --git a/site/notebooks/EXECUTED/model_validation/inserted-minimum-f1-scores.png b/site/notebooks/EXECUTED/model_validation/inserted-minimum-f1-scores.png new file mode 100644 index 0000000000..52ae43c719 Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/inserted-minimum-f1-scores.png differ diff --git a/site/notebooks/EXECUTED/model_validation/link-finding.png b/site/notebooks/EXECUTED/model_validation/link-finding.png new file mode 100644 index 0000000000..f1855aab23 Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/link-finding.png differ diff --git a/site/notebooks/EXECUTED/model_validation/link-validator-evidence.png b/site/notebooks/EXECUTED/model_validation/link-validator-evidence.png new file mode 100644 index 0000000000..660a19cfab Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/link-validator-evidence.png differ diff --git a/site/notebooks/EXECUTED/model_validation/lr_model_champion.pkl b/site/notebooks/EXECUTED/model_validation/lr_model_champion.pkl new file mode 100644 index 0000000000..9b81662b24 Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/lr_model_champion.pkl differ diff --git a/site/notebooks/EXECUTED/model_validation/select-finding.png b/site/notebooks/EXECUTED/model_validation/select-finding.png new file mode 100644 index 0000000000..ba35661d58 Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/select-finding.png differ diff --git a/site/notebooks/EXECUTED/model_validation/selecting-class-imbalance-results.png b/site/notebooks/EXECUTED/model_validation/selecting-class-imbalance-results.png new file mode 100644 index 0000000000..cf86874228 Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/selecting-class-imbalance-results.png differ diff --git a/site/notebooks/EXECUTED/model_validation/selecting-minimum-f1-scores.png b/site/notebooks/EXECUTED/model_validation/selecting-minimum-f1-scores.png new file mode 100644 index 0000000000..60ae6b9605 Binary files /dev/null and b/site/notebooks/EXECUTED/model_validation/selecting-minimum-f1-scores.png differ diff --git a/site/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb b/site/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb index 80393d5f31..963179129f 100644 --- a/site/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb +++ b/site/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb @@ -37,7 +37,9 @@ " - [Custom Test: External API Call](#toc9_2_) \n", " - [Custom Test: Passing Parameters](#toc9_3_) \n", " - [Custom Test: Multiple Tables and Plots in a Single Test](#toc9_4_) \n", - " - [Custom Test: Images](#toc9_5_) \n", + " - [Custom Test: Images](#toc9_5_)\n", + " - [Custom Test: Description](#toc9_6_)\n", + "\n", "- [Conclusion](#toc10_) \n", "- [Next steps](#toc11_) \n", " - [Work with your model documentation](#toc11_1_) \n", @@ -867,6 +869,66 @@ "![screenshot showing image from file](../../images/pearson-correlation-matrix-test-output.png)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Custom Test: Description\n", + "\n", + "If you want to write a custom test description for your custom test instead of it is interpreted through llm, you can do so by returning string in your test." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "@vm.test(\"my_custom_tests.MyCustomTest\")\n", + "def my_custom_test(dataset, model):\n", + " \"\"\"\n", + " This is a custom computed test that computes confusion matrix for a binary classification model and return a string as a test description.\n", + " \"\"\"\n", + " y_true = dataset.y\n", + " y_pred = dataset.y_pred(model)\n", + "\n", + " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n", + "\n", + " cm_display = metrics.ConfusionMatrixDisplay(\n", + " confusion_matrix=confusion_matrix, display_labels=[False, True]\n", + " )\n", + " cm_display.plot()\n", + "\n", + " plt.close() # close the plot to avoid displaying it\n", + "\n", + " return cm_display.figure_, \"Test Description - Confusion Matrix\", pd.DataFrame({\"Value\": [1, 2, 3]}) # return the figure object itself\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can see here test result description has been customized here. The same result description will be displayed in the UI." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = run_test(\n", + " \"my_custom_tests.MyCustomTest\",\n", + " inputs={\"model\": \"model\", \"dataset\": \"test_dataset\"},\n", + ")\n", + "result.log()" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/site/notebooks/how_to/add_context_to_llm_descriptions.ipynb b/site/notebooks/how_to/add_context_to_llm_descriptions.ipynb index 45736d2c7a..e8d06f0ebf 100644 --- a/site/notebooks/how_to/add_context_to_llm_descriptions.ipynb +++ b/site/notebooks/how_to/add_context_to_llm_descriptions.ipynb @@ -41,7 +41,8 @@ " - [IQR Outliers Table](#toc6_3_7_) \n", " - [Descriptive Statistics](#toc6_3_8_) \n", " - [Pearson Correlation Matrix](#toc6_3_9_) \n", - "\n", + " - [Add test-specific context using the docstring](#toc6_4_)\n", + "- [Best practices for adding custom context](#toc7_)\n", ":::\n", "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## About ValidMind\n", - "\n", - "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models. You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n", - "\n", - "\n", - "\n", - "### Before you begin\n", - "\n", - "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language.\n", - "\n", - "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n", - "\n", - "\n", - "\n", - "### New to ValidMind?\n", - "\n", - "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.\n", - "\n", - "
For access to all features available in this notebook, create a free ValidMind account.\n", + "
Or, take our Developer Fundamentals course which walks you through the basics of ValidMind paired with this notebook series.\n", "

\n", - "Signing up is FREE — Register with ValidMind
\n", - "\n", - "\n", - "\n", - "### Key concepts\n", - "\n", - "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n", - "\n", - "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", - "\n", - "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n", - "\n", - "**Custom tests**: Custom tests are functions that you define to evaluate your model or dataset. These functions can be registered via the ValidMind Library to be used with the ValidMind Platform.\n", - "\n", - "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n", - "\n", - "- **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n", - "- **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n", - "- **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom test.\n", - "- **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom test. See this [example](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html) for more information.\n", - "\n", - "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a test, customize its behavior, or provide additional context.\n", - "\n", - "**Outputs**: Custom tests can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n", - "\n", - "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n", - "\n", - "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## 1. Initializing the ValidMind Library\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Install the ValidMind Library\n", - "\n", - "Please note the following recommended Python versions to use:\n", - "\n", - "- Python 3.8 <= x <= 3.11\n", - "\n", - "To install the library:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "%pip install -q validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the ValidMind Library\n", - "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", - "\n", - "\n", - "\n", - "### Get your code snippet\n", - "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Model Inventory** and click **+ Register Model**.\n", - "\n", - "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", - "\n", - " For example, to register a model for use with this notebook, select:\n", - "\n", - " - Documentation template: `Binary classification`\n", - " - Use case: `Marketing/Sales - Attrition/Churn Management`\n", - "\n", - " You can fill in other options according to your preference.\n", - "\n", - "4. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", - "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Load your model identifier credentials from an `.env` file\n", - "\n", - "%load_ext dotenv\n", - "%dotenv .env\n", - "\n", - "# Or replace with your code snippet\n", - "\n", - "import validmind as vm\n", - "\n", - "vm.init(\n", - " # api_host=\"...\",\n", - " # api_key=\"...\",\n", - " # api_secret=\"...\",\n", - " # model=\"...\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Verify & preview the documentation template\n", - "\n", - "Let's verify that you have connected to ValidMind and that the appropriate template is selected. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", - "\n", - "You will upload documentation and test results for this template later on. For now, take a look at the structure that the template provides with the `vm.preview_template()` function from the ValidMind library and note the empty sections:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.preview_template()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Before learning how to run tests, let's explore the list of all available tests in the ValidMind Library. You can see that the documentation template for this model has references to some of the test IDs listed below.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.tests.list_tests()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## 2. Start the model development process with raw data, run out-of-the box tests, and add evidence to model documentation\n", - "\n", - "In this section you learn how to explore the individual tests available in ValidMind and how to run them and change parameters as necessary. You will use a public dataset from Kaggle that models a bank customer churn prediction use case. The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n", - "\n", - "You can find more information about this dataset [here](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction).\n", - "\n", - "The ValidMind Library provides a wrapper to automatically load the dataset as a Pandas DataFrame object.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "from validmind.datasets.classification import customer_churn as demo_dataset\n", - "\n", - "print(\n", - " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n", - ")\n", - "\n", - "raw_df = demo_dataset.load_data()\n", - "raw_df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's do some data quality assessments by running a few individual tests related to data assessment. You will use the `vm.tests.list_tests()` function introduced above in combination with `vm.tests.list_tags()` and `vm.tests.list_tasks()` to find which prebuilt tests are relevant for data quality assessment.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Get the list of available tags\n", - "sorted(vm.tests.list_tags())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Get the list of available task types\n", - "sorted(vm.tests.list_tasks())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can pass `tags` and `tasks` as parameters to the `vm.tests.list_tests()` function to filter the tests based on the tags and task types. For example, to find tests related to tabular data quality for classification models, you can call `list_tests()` like this:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.tests.list_tests(task=\"classification\", tags=[\"tabular_data\", \"data_quality\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the ValidMind datasets\n", - "\n", - "Now, assume we have identified some tests we want to run with regards to the data we are intending to use. The next step is to connect your data with a ValidMind `Dataset` object. This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind. You only need to do it one time per dataset.\n", - "\n", - "You can initialize a ValidMind dataset object using the [`init_dataset`](https://docs.validmind.ai/validmind/validmind.html#init_dataset) function from the ValidMind (`vm`) module.\n", - "\n", - "This function takes a number of arguments:\n", - "\n", - "- `dataset` — the raw dataset that you want to provide as input to tests\n", - "- `input_id` - a unique identifier that allows tracking what inputs are used when running each individual test\n", - "- `target_column` — a required argument if tests require access to true values. This is the name of the target column in the dataset\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# vm_raw_dataset is now a VMDataset object that you can pass to any ValidMind test\n", - "vm_raw_dataset = vm.init_dataset(\n", - " dataset=raw_df,\n", - " input_id=\"raw_dataset\",\n", - " target_column=\"Exited\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Run some tabular data tests\n", - "\n", - "Individual tests can be easily run by calling the `run_test` function provided by the `validmind.tests` module. The function takes the following arguments:\n", - "\n", - "- `test_id`: The ID of the test to run. To find a particular test and get its ID, refer to the [explore_tests](../how_to/explore_tests.ipynb) notebook. Look above for example after running 'vm.test_suites.describe_suite' as column 'Test ID' will contain the id.\n", - "- `params`: A dictionary of parameters for the test. These will override any `default_params` set in the test definition. Refer to the [explore_tests](../how_to/explore_tests.ipynb) notebook to find the default parameters for a test. See below for examples.\n", - "\n", - "The inputs expected by a test can also be found in the test definition. Let's take `validmind.data_validation.DescriptiveStatistics` as an example. Note that the output of the `describe_test()` function below shows that this test expects a `dataset` as input:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.tests.describe_test(\"validmind.data_validation.DescriptiveStatistics\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, let's run a few tests to assess the quality of the dataset.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.DescriptiveStatistics\",\n", - " inputs={\"dataset\": vm_raw_dataset},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result2 = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.ClassImbalance\",\n", - " inputs={\"dataset\": vm_raw_dataset},\n", - " params={\"min_percent_threshold\": 30},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can see that the class imbalance test did not pass according to the value of `min_percent_threshold` we have set. Here is how you can re-run the test on some processed data to address this data quality issue. In this case we apply a very simple rebalancing technique to the dataset.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "raw_copy_df = raw_df.sample(frac=1) # Create a copy of the raw dataset\n", - "\n", - "# Create a balanced dataset with the same number of exited and not exited customers\n", - "exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 1]\n", - "not_exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 0].sample(n=exited_df.shape[0])\n", - "\n", - "balanced_raw_df = pd.concat([exited_df, not_exited_df])\n", - "balanced_raw_df = balanced_raw_df.sample(frac=1, random_state=42)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "With this new raw dataset, you can re-run the individual test to see if it passes the class imbalance test requirement. Remember to register new VM Dataset object since that is the type of input required by `run_test()`:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Register new data and now 'balanced_raw_dataset' is the new dataset object of interest\n", - "vm_balanced_raw_dataset = vm.init_dataset(\n", - " dataset=balanced_raw_df,\n", - " input_id=\"balanced_raw_dataset\",\n", - " target_column=\"Exited\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.ClassImbalance\",\n", - " inputs={\"dataset\": vm_balanced_raw_dataset},\n", - " params={\"min_percent_threshold\": 30},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Utilize test output\n", - "\n", - "Here is an example for how you can utilize the output from a ValidMind test for futher use, for example, if you want to remove highly correlated features. The example below shows how you can get the list of features with the highest correlation coefficients and use them to reduce the final list of features for modeling.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "corr_result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n", - " params={\"max_threshold\": 0.3},\n", - " inputs={\"dataset\": vm_balanced_raw_dataset},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's assume we want to remove highly correlated features from the dataset. `corr_result` is an object of type `TestResult`. We can inspect the result object to see what the test has produced." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "print(type(corr_result))\n", - "print(\"Result ID: \", corr_result.result_id)\n", - "print(\"Params: \", corr_result.params)\n", - "print(\"Passed: \", corr_result.passed)\n", - "print(\"Tables: \", corr_result.tables)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's check out the table in the result and extract a list of features that failed the test:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "features_df = corr_result.tables[0].data\n", - "features_df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Remove the highly correlated features and create a new VM dataset object. Note the use of different `input_id`s. This allows tracking the inputs used when running each individual test.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "high_correlation_features = features_df[features_df[\"Pass/Fail\"] == \"Fail\"][\"Columns\"].tolist()\n", - "high_correlation_features" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Extract the feature names from the list of strings (e.g. '(Age, Exited)' -> 'Age')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_correlation_features = [feature.split(\",\")[0].strip(\"()\") for feature in high_correlation_features]\n", - "high_correlation_features" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Remove the highly correlated features from the dataset\n", - "balanced_raw_no_age_df = balanced_raw_df.drop(columns=high_correlation_features)\n", - "\n", - "# Re-initialize the dataset object\n", - "vm_raw_dataset_preprocessed = vm.init_dataset(\n", - " dataset=balanced_raw_no_age_df,\n", - " input_id=\"raw_dataset_preprocessed\",\n", - " target_column=\"Exited\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Re-running the test with the reduced feature set should pass the test. You can also plot the correlation matrix to visualize the new correlation between features:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "corr_result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n", - " params={\"max_threshold\": 0.3},\n", - " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "corr_result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.PearsonCorrelationMatrix\",\n", - " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Documenting the results based on two datasets\n", - "\n", - "We have now done some analysis on two different datasets and we should able to document why certain things were done to the raw data with testing to support it. Every test result returned by the `run_test()` function has a `.log()` method that can be used to log the test results to ValidMind. When logging individual results to ValidMind you need to manually add those results in a specific section of the model documentation.\n", - "\n", - "When using `run_documentation_tests()`, it's possible to automatically populate a section with the results of all tests that were registered in the documentation template.\n", - "\n", - "To show how to add individual results to any documentation section, we're going to populate the entire `data_preparation` section of the documentation using the clean `vm_raw_dataset_preprocessed` dataset as input, and then we're going to document an additional result for the highly correlated dataset `vm_balanced_raw_dataset`. The following two steps will accomplish this:\n", - "\n", - "1. Run `run_documentation_tests()` using `vm_raw_dataset_preprocessed` as input. This populates the entire data preparation section for every test that is already part of the documentation template.\n", - "2. Log the individual result of the high correlation test that used `vm_balanced_raw_dataset` (that had a highly correlated `Age` column) as input\n", - "\n", - "After adding the result of step #2 to the documentation you will be able to explain the changes made to the raw data by editing the default description of the test result within the ValidMind Platform.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Run `run_documentation_tests()` using `vm_raw_dataset_preprocessed` as input\n", - "\n", - "`run_documentation_tests()` allows you to run multiple tests at once and log the results to the documentation. The function takes the following arguments:\n", - "\n", - "- `inputs`: any inputs to be passed to the tests\n", - "- `config`: a dictionary `:` that allows configuring each test individually. Each test config has the following form:\n", - " - `params`: individual test parameters\n", - " - `inputs`: individual test inputs. When passed, this overrides any inputs passed from the `run_documentation_tests()` function\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "test_config = {\n", - " \"validmind.data_validation.ClassImbalance\": {\n", - " \"params\": {\"min_percent_threshold\": 30},\n", - " },\n", - " \"validmind.data_validation.HighPearsonCorrelation\": {\n", - " \"params\": {\"max_threshold\": 0.3},\n", - " },\n", - "}\n", - "\n", - "tests_suite = vm.run_documentation_tests(\n", - " inputs={\n", - " \"dataset\": vm_raw_dataset_preprocessed,\n", - " },\n", - " config=test_config,\n", - " section=[\"data_preparation\"],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Log the individual result of the high correlation test that used `vm_balanced_raw_dataset` (that had a highly correlated `Age` column) as input\n", - "\n", - "Here you can use a custom `result_id` to tag the individual result with a unique identifier. This `result_id` can be appended to `test_id` with a `:` separator. The `balanced_raw_dataset` result identifier will correspond to the `balanced_raw_dataset` input, the dataset that still has the `Age` column.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.HighPearsonCorrelation:balanced_raw_dataset\",\n", - " params={\"max_threshold\": 0.3},\n", - " inputs={\"dataset\": vm_balanced_raw_dataset},\n", - ")\n", - "result.log()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Add individual test results to model documentation\n", - "\n", - "You can now visit the documentation page for the model you connected to at the beginning of this notebook and add a new content block in the relevant section.\n", - "\n", - "To do this, go to the documentation page of your model and navigate to the `Data Preparation` -> `Correlations and Interactions` section. Then hover after the \"Pearson Correlation Matrix\" content block to reveal the `+` button as shown in the screenshot below.\n", - "\n", - "![screenshot showing insert button for test-driven blocks](../images/insert-test-driven-block-correlations.png)\n", - "\n", - "Click on the `+` button and select `Test-Driven Block`. This will open a dialog where you can select `Threshold Test` as the type of the test-driven content block, and then select `High Pearson Correlation Vm Raw Dataset Test`. This will show a preview of the result and it should match the results shown above.\n", - "\n", - "![screenshot showing the selected test result in the dialog](../images/selecting-high-pearson-correlation-test.png)\n", - "\n", - "Finally, click on the `Insert block` button to add the test result to the documentation. You'll now see two individual results for the high correlation test in the `Correlations and Interactions` section of the documentation. To finalize the documentation, you can edit the test result's description block to explain the changes made to the raw data and the reasons behind them as we can see in the screenshot below.\n", - "\n", - "![screenshot showing the high pearson correlation block](../images/high-pearson-correlation-block.png)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Model Testing\n", - "\n", - "We have focused so far on the data assessment and pre-processing that usually occurs prior to any models being built. Now we are going to assume we have built a model and we want to incorporate some model results in our documentation.\n", - "\n", - "Let's train a simple logistic regression model on the dataset and evaluate its performance. You will use the `LogisticRegression` class from the `sklearn.linear_model` and use ValidMind tests to evaluate the model's performance.\n", - "\n", - "Before training the model, we need to encode the categorical features in the dataset. You will use the `OneHotEncoder` class from the `sklearn.preprocessing` module to encode the categorical features. The categorical features in the dataset are `Geography` and `Gender`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "balanced_raw_no_age_df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "balanced_raw_no_age_df = pd.get_dummies(\n", - " balanced_raw_no_age_df, columns=[\"Geography\", \"Gender\"], drop_first=True\n", - ")\n", - "balanced_raw_no_age_df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "# Split the input and target variables\n", - "X = balanced_raw_no_age_df.drop(\"Exited\", axis=1)\n", - "y = balanced_raw_no_age_df[\"Exited\"]\n", - "X_train, X_test, y_train, y_test = train_test_split(\n", - " X,\n", - " y,\n", - " test_size=0.2,\n", - " random_state=42,\n", - ")\n", - "\n", - "# Logistic Regression grid params\n", - "log_reg_params = {\n", - " \"penalty\": [\"l1\", \"l2\"],\n", - " \"C\": [0.001, 0.01, 0.1, 1, 10, 100, 1000],\n", - " \"solver\": [\"liblinear\"],\n", - "}\n", - "\n", - "# Grid search for Logistic Regression\n", - "from sklearn.model_selection import GridSearchCV\n", - "\n", - "grid_log_reg = GridSearchCV(LogisticRegression(), log_reg_params)\n", - "grid_log_reg.fit(X_train, y_train)\n", - "\n", - "# Logistic Regression best estimator\n", - "log_reg = grid_log_reg.best_estimator_" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize model evaluation objects and assigning predictions\n", - "\n", - "The last step for evaluating the model's performance is to initialize the ValidMind `Dataset` and `Model` objects and assign model predictions to each dataset. You will use the `init_dataset`, `init_model` and `assign_predictions` functions to initialize these objects.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "train_df = X_train\n", - "train_df[\"Exited\"] = y_train\n", - "test_df = X_test\n", - "test_df[\"Exited\"] = y_test\n", - "\n", - "vm_train_ds = vm.init_dataset(\n", - " input_id=\"train_dataset_final\",\n", - " dataset=train_df,\n", - " target_column=\"Exited\",\n", - ")\n", - "\n", - "vm_test_ds = vm.init_dataset(\n", - " input_id=\"test_dataset_final\",\n", - " dataset=test_df,\n", - " target_column=\"Exited\",\n", - ")\n", - "\n", - "# Register the model\n", - "vm_model = vm.init_model(log_reg, input_id=\"log_reg_model_v1\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once the model has been registered you can assign model predictions to the training and test datasets. The `assign_predictions()` method from the `Dataset` object can link existing predictions to any number of models. If no prediction values are passed, the method will compute predictions automatically:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm_train_ds.assign_predictions(model=vm_model)\n", - "vm_test_ds.assign_predictions(model=vm_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Run the model evaluation tests\n", - "\n", - "In this part, we focus on running the tests within the model development section of the model documentation. Only tests associated with this section will be executed, and the corresponding results will be updated in the model documentation. In the example below, you will focus on only running tests for the `model development` section of the document.\n", - "\n", - "Note the additional config that is passed to `run_documentation_tests()`. This allows you to override inputs or params in certain tests. In our case, we want to explicitly use the `vm_train_ds` for the `validmind.model_validation.sklearn.ClassifierPerformance:in_sample` test, since it's supposed to run on the training dataset and not the test dataset.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "test_config = {\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance:in_sample\": {\n", - " \"inputs\": {\n", - " \"dataset\": vm_train_ds,\n", - " \"model\": vm_model,\n", - " },\n", - " }\n", - "}\n", - "results = vm.run_documentation_tests(\n", - " section=[\"model_development\"],\n", - " inputs={\n", - " \"dataset\": vm_test_ds, # Any test that requires a single dataset will use vm_test_ds\n", - " \"model\": vm_model,\n", - " \"datasets\": (\n", - " vm_train_ds,\n", - " vm_test_ds,\n", - " ), # Any test that requires multiple datasets will use vm_train_ds and vm_test_ds\n", - " },\n", - " config=test_config,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## 3. Implementing custom tests\n", - "\n", - "This section assumes that model developers already have a repository of custom made tests that they consider critical to include in the documentation. Here we provide details on how to easily integrate custom tests with ValidMind.\n", - "\n", - "For a more in-depth introduction to custom tests, refer to this [notebook](../code_samples/custom_tests/implement_custom_tests.ipynb).\n", - "\n", - "A custom test is any function that takes a set of inputs and parameters as arguments and returns one or more outputs. The function can be as simple or as complex as you need it to be. It can use external libraries, make API calls, or do anything else that you can do in Python. The only requirement is that the function signature and return values can be \"understood\" and handled by the ValidMind Library. As such, custom tests offer added flexibility by extending the default tests provided by ValidMind, enabling you to document any type of model or use case.\n", - "\n", - "In the following example, you will learn how to implement a custom `inline` test that calculates the confusion matrix for a binary classification model. You will see that the custom test function is just a regular Python function that can include and require any Python library as you see fit.\n", - "\n", - "**NOTE**: in the context of Jupyter notebooks, we will use the word `inline` to refer to functions (or code) defined in the same notebook where they are used (this one) and not in a separate file, as we will see later with test providers.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Create a confusion matrix plot\n", - "\n", - "To understand how to create a custom test from anything, let's first create a confusion matrix plot using the `confusion_matrix` function from the `sklearn.metrics` module.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "from sklearn import metrics\n", - "\n", - "# Get the predicted classes\n", - "y_pred = log_reg.predict(vm_test_ds.x)\n", - "\n", - "confusion_matrix = metrics.confusion_matrix(y_test, y_pred)\n", - "\n", - "cm_display = metrics.ConfusionMatrixDisplay(\n", - " confusion_matrix=confusion_matrix, display_labels=[False, True]\n", - ")\n", - "cm_display.plot()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will now create a @vm.test wrapper that will allow you to create a reusable test. Note the following changes in the code below:\n", - "\n", - "- The function `confusion_matrix` takes two arguments `dataset` and `model`. This is a `VMDataset` and `VMModel` object respectively.\n", - " - `VMDataset` objects allow you to access the dataset's true (target) values by accessing the `.y` attribute.\n", - " - `VMDataset` objects allow you to access the predictions for a given model by accessing the `.y_pred()` method.\n", - "- The function docstring provides a description of what the test does. This will be displayed along with the result in this notebook as well as in the ValidMind Platform.\n", - "- The function body calculates the confusion matrix using the `sklearn.metrics.confusion_matrix` function as we just did above.\n", - "- The function then returns the `ConfusionMatrixDisplay.figure_` object - this is important as the ValidMind Library expects the output of the custom test to be a plot or a table.\n", - "- The `@vm.test` decorator is doing the work of creating a wrapper around the function that will allow it to be run by the ValidMind Library. It also registers the test so it can be found by the ID `my_custom_tests.ConfusionMatrix` (see the section below on how test IDs work in ValidMind and why this format is important)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "@vm.test(\"my_custom_tests.ConfusionMatrix\")\n", - "def confusion_matrix(dataset, model):\n", - " \"\"\"The confusion matrix is a table that is often used to describe the performance of a classification model on a set of data for which the true values are known.\n", - "\n", - " The confusion matrix is a 2x2 table that contains 4 values:\n", - "\n", - " - True Positive (TP): the number of correct positive predictions\n", - " - True Negative (TN): the number of correct negative predictions\n", - " - False Positive (FP): the number of incorrect positive predictions\n", - " - False Negative (FN): the number of incorrect negative predictions\n", - "\n", - " The confusion matrix can be used to assess the holistic performance of a classification model by showing the accuracy, precision, recall, and F1 score of the model on a single figure.\n", - " \"\"\"\n", - " y_true = dataset.y\n", - " y_pred = dataset.y_pred(model=model)\n", - "\n", - " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n", - "\n", - " cm_display = metrics.ConfusionMatrixDisplay(\n", - " confusion_matrix=confusion_matrix, display_labels=[False, True]\n", - " )\n", - " cm_display.plot()\n", - "\n", - " plt.close() # close the plot to avoid displaying it\n", - "\n", - " return cm_display.figure_ # return the figure object itself" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can now run the newly created custom test on both the training and test datasets using the `run_test()` function:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Training dataset\n", - "result = vm.tests.run_test(\n", - " \"my_custom_tests.ConfusionMatrix:training_dataset\",\n", - " inputs={\"model\": vm_model, \"dataset\": vm_train_ds},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Test dataset\n", - "result = vm.tests.run_test(\n", - " \"my_custom_tests.ConfusionMatrix:test_dataset\",\n", - " inputs={\"model\": vm_model, \"dataset\": vm_test_ds},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Add parameters to custom tests\n", - "\n", - "Custom tests can take parameters just like any other function. Let's modify the `confusion_matrix` function to take an additional parameter `normalize` that will allow you to normalize the confusion matrix.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "@vm.test(\"my_custom_tests.ConfusionMatrix\")\n", - "def confusion_matrix(dataset, model, normalize=False):\n", - " \"\"\"The confusion matrix is a table that is often used to describe the performance of a classification model on a set of data for which the true values are known.\n", - "\n", - " The confusion matrix is a 2x2 table that contains 4 values:\n", - "\n", - " - True Positive (TP): the number of correct positive predictions\n", - " - True Negative (TN): the number of correct negative predictions\n", - " - False Positive (FP): the number of incorrect positive predictions\n", - " - False Negative (FN): the number of incorrect negative predictions\n", - "\n", - " The confusion matrix can be used to assess the holistic performance of a classification model by showing the accuracy, precision, recall, and F1 score of the model on a single figure.\n", - " \"\"\"\n", - " y_true = dataset.y\n", - " y_pred = dataset.y_pred(model=model)\n", - "\n", - " if normalize:\n", - " confusion_matrix = metrics.confusion_matrix(y_true, y_pred, normalize=\"all\")\n", - " else:\n", - " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n", - "\n", - " cm_display = metrics.ConfusionMatrixDisplay(\n", - " confusion_matrix=confusion_matrix, display_labels=[False, True]\n", - " )\n", - " cm_display.plot()\n", - "\n", - " plt.close() # close the plot to avoid displaying it\n", - "\n", - " return cm_display.figure_ # return the figure object itself" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Pass parameters to custom tests\n", - "\n", - "You can pass parameters to custom tests by providing a dictionary of parameters to the `run_test()` function. The parameters will override any default parameters set in the custom test definition. Note that `dataset` and `model` are still passed as `inputs`. Since these are `VMDataset` or `VMModel` inputs, they have a special meaning. When declaring a `dataset`, `model`, `datasets` or `models` argument in a custom test function, the ValidMind Library will expect these get passed as `inputs` to `run_test()` (or `run_documentation_tests()` instead).\n", - "\n", - "Re-running the confusion matrix with `normalize=True` looks like this:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Test dataset with normalize=True\n", - "result = vm.tests.run_test(\n", - " \"my_custom_tests.ConfusionMatrix:test_dataset_normalized\",\n", - " inputs={\"model\": vm_model, \"dataset\": vm_test_ds},\n", - " params={\"normalize\": True},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Log the confusion matrix results\n", - "\n", - "As you saw in the pearson correlation example, you can log any result to the ValidMind Platform with the `.log()` method of the result object. This will allow you to add the result to the documentation.\n", - "\n", - "You can now do the same for the confusion matrix results.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result.log()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Using external test providers\n", - "\n", - "Creating inline custom tests with a function is a great way to customize your model documentation. However, sometimes you may want to reuse the same set of tests across multiple models and share them with developers in your organization. In this case, you can create a custom test provider that will allow you to load custom tests from a local folder or a git repository.\n", - "\n", - "In this section you will learn how to declare a local filesystem test provider that allows loading tests from a local folder following these high level steps:\n", - "\n", - "1. Create a folder of custom tests from existing, inline tests (tests that exists in your active Jupyter notebook)\n", - "2. Save an inline test to a file\n", - "3. Define and register a `LocalTestProvider` that points to that folder\n", - "4. Run test provider tests\n", - "5. Add the test results to your documentation\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Create a folder of custom tests from existing inline tests\n", - "\n", - "Here you will create a new folder that will contain reusable, custom tests. The following code snippet will create a new `my_tests` directory in the current working directory if it doesn't exist.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "tests_folder = \"my_tests\"\n", - "\n", - "import os\n", - "\n", - "# create tests folder\n", - "os.makedirs(tests_folder, exist_ok=True)\n", - "\n", - "# remove existing tests\n", - "for f in os.listdir(tests_folder):\n", - " # remove files and pycache\n", - " if f.endswith(\".py\") or f == \"__pycache__\":\n", - " os.system(f\"rm -rf {tests_folder}/{f}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After running the command above, you should see a new directory next to this notebook file:\n", - "\n", - "![screenshot showing my_tests directory](../images/my_tests_directory.png)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Save an inline test to a file\n", - "\n", - "The `@vm.test` decorator that was used above to register these as one-off custom tests also adds a convenience method to the function object that allows you to simply call `.save()` to save it to a file. This will save the function to a Python file to a path you specify. In this case, you can pass the variable `tests_folder` to save it to the custom tests folder we created.\n", - "\n", - "Normally, this will get you started by creating the file and saving the function code with the correct name. But it won't automatically add any import or other functions/variables outside of the function that are needed for the test to run. The `save()` method allows you to pass an optional `imports` argument that will ensure the necessary imports are added to the file.\n", - "\n", - "For the `confusion_matrix` test, note the imports that are required for the function to run properly:\n", - "\n", - "```python\n", - "import matplotlib.pyplot as plt\n", - "from sklearn import metrics\n", - "```\n", - "\n", - "You can pass these imports to the `save()` method to ensure they are included in the file with the following command:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "confusion_matrix.save(\n", - " tests_folder,\n", - " imports=[\"import matplotlib.pyplot as plt\", \"from sklearn import metrics\"],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### What happened?\n", - "\n", - "The `save()` method saved the `confusion_matrix` function to a file named `ConfusionMatrix.py` in the `my_tests` folder. Note that the new file provides some context on the origin of the test, which is useful for traceability.\n", - "\n", - "```\n", - "# Saved from __main__.confusion_matrix\n", - "# Original Test ID: my_custom_tests.ConfusionMatrix\n", - "# New Test ID: .ConfusionMatrix\n", - "```\n", - "\n", - "Additionally, the new test function has been stripped off its decorator, as it now resides in a file that will be loaded by the test provider:\n", - "\n", - "```python\n", - "def ConfusionMatrix(dataset, model, normalize=False):\n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Define and register a `LocalTestProvider` that points to that folder\n", - "\n", - "With the `my_tests` folder now having a sample custom test, you can now initialize a test provider that will tell the ValidMind Library where to find these tests. ValidMind offers out-of-the-box test providers for local tests (i.e. tests in a folder) or a Github provider for tests in a Github repository. You can also create your own test provider by creating a class that has a `load_test` method that takes a test ID and returns the test function matching that ID.\n", - "\n", - "The most important attribute for a test provider is its `namespace`. This is a string that will be used to prefix test IDs in model documentation. This allows you to have multiple test providers with tests that can even share the same ID, but are distinguished by their namespace.\n", - "\n", - "An extended introduction to test providers can be found in [this](../code_samples/custom_tests/integrate_external_test_providers.ipynb) notebook.\n", - "\n", - "\n", - "\n", - "### Initializing a local test provider\n", - "\n", - "For most use-cases, the local test provider should be sufficient. This test provider allows you load custom tests from a designated directory. Let's go ahead and see how we can do this with our custom tests.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "from validmind.tests import LocalTestProvider\n", - "\n", - "# initialize the test provider with the tests folder we created earlier\n", - "my_test_provider = LocalTestProvider(tests_folder)\n", - "\n", - "vm.tests.register_test_provider(\n", - " namespace=\"my_test_provider\",\n", - " test_provider=my_test_provider,\n", - ")\n", - "# `my_test_provider.load_test()` will be called for any test ID that starts with `my_test_provider`\n", - "# e.g. `my_test_provider.ConfusionMatrix` will look for a function named `ConfusionMatrix` in `my_tests/ConfusionMatrix.py` file" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Run test provider tests\n", - "\n", - "Now that you have set up the test provider, you can run any test that's located in the tests folder by using the `run_test()` method as with any other test. For tests that reside in a test provider directory, the test ID will be the `namespace` specified when registering the provider, followed by the path to the test file relative to the tests folder. For example, the Confusion Matrix test we created earlier will have the test ID `my_test_provider.ConfusionMatrix`. You could organize the tests in subfolders, say `classification` and `regression`, and the test ID for the Confusion Matrix test would then be `my_test_provider.classification.ConfusionMatrix`.\n", - "\n", - "Let's go ahead and re-run the confusion matrix test by using the test ID `my_test_provider.ConfusionMatrix`. This should load the test from the test provider and run it as before.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result = vm.tests.run_test(\n", - " \"my_test_provider.ConfusionMatrix\",\n", - " inputs={\"model\": vm_model, \"dataset\": vm_test_ds},\n", - " params={\"normalize\": True},\n", - ")\n", - "\n", - "result.log()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Add the test results to your documentation\n", - "\n", - "You have already seen how to add individual results to the model documentation using the ValidMind Platform. Let's repeat the process and add the confusion matrix to the `Model Development` -> `Model Evaluation` section of the documentation. The \"add test driven block\" dialog should now show the new test result coming from the test provider:\n", - "\n", - "![screenshot showing confusion matrix result](../images/insert-test-driven-block-custom-confusion-matrix.png)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## 4. Finalize testing and documentation\n", - "\n", - "In this section we cover how to finalize the testing and documentation of your model by focusing on:\n", - "\n", - "1. Using `run_documentation_tests()` to ensure custom test results are included in your documentation\n", - "2. Viewing and updating the configuration for the entire model documentation template\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Use `run_documentation_tests()` to ensure custom test results are included in your documentation\n", - "\n", - "After adding test driven blocks to your model documentation, changes should persist and become available every time you call `vm.preview_template()`. However, you need to reload the connection to the ValidMind Platform if you have added test driven blocks when the connection was already established.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.reload()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, run `preview_template()` and verify that the new confusion matrix test you added is included in the proper section.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.preview_template()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Since the test ID is now registered in the document you can now run tests for an entire section and all additional custom tests should be loaded without issues. Let's run all tests in the `model_evaluation` section of the documentation. Note that we have been running the sample custom confusion matrix with `normalize=True` to demonstrate the ability to provide custom parameters.\n", - "\n", - "In the `Run the model evaluation tests` section above you learned how to assign inputs to individual tests with `run_documentation_tests()`. Assigning parametesr is similar, you only need to provide assign a `params` dictionary to a given test ID, `my_test_provider.ConfusionMatrix` in this case.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "test_config = {\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance:in_sample\": {\n", - " \"inputs\": {\n", - " \"dataset\": vm_train_ds,\n", - " \"model\": vm_model,\n", - " },\n", - " },\n", - " \"my_test_provider.ConfusionMatrix\": {\n", - " \"params\": {\"normalize\": True},\n", - " },\n", - "}\n", - "results = vm.run_documentation_tests(\n", - " section=[\"model_evaluation\"],\n", - " inputs={\n", - " \"dataset\": vm_test_ds, # Any test that requires a single dataset will use vm_test_ds\n", - " \"model\": vm_model,\n", - " \"datasets\": (\n", - " vm_train_ds,\n", - " vm_test_ds,\n", - " ), # Any test that requires multiple datasets will use vm_train_ds and vm_test_ds\n", - " },\n", - " config=test_config,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Viewing and updating the configuration for the entire model documentation template\n", - "\n", - "The ValidMind Library provides a utility function called `vm.get_test_suite().get_default_config()` that allows you to render the default configuration for the entire documentation template. This configuration will contain all the test IDs and their default parameters. You can then modify this configuration as needed and pass it to `run_documentation_tests()` to run all tests in the documentation template if needed. You also have the option to continue running tests for one section at a time, `get_default_config()` still provides a useful reference for providing default parametes to every test.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import json\n", - "\n", - "model_test_suite = vm.get_test_suite()\n", - "config = model_test_suite.get_default_config()\n", - "print(\"Suite Config: \\n\", json.dumps(config, indent=2))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Update the config\n", - "\n", - "Note that the default config does not assign any inputs to a test, this is expected. You can assign inputs to individual tests as needed, depending on the datasets and models you want to pass to individual tests. The `config` dictionary, as a mapping of test IDs to test configurations, allows you to do this.\n", - "\n", - "For this particular documentation template (binary classification), the ValidMind Library provides a sample configuration that can be used to populate the entire model documentation using the following inputs as placeholders:\n", - "\n", - "- A `raw_dataset` raw dataset\n", - "- A `train_dataset` training dataset\n", - "- A `test_dataset` test dataset\n", - "- A trained `model` instance\n", - "\n", - "As part of updating the `config` you will need to ensure the correct `input_id`s are used in the final config passed to `run_documentation_tests()`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "from validmind.datasets.classification import customer_churn\n", - "from validmind.utils import preview_test_config\n", - "\n", - "test_config = customer_churn.get_demo_test_config()\n", - "preview_test_config(test_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Using this sample configuration, let's finish populating model documentation by running all tests for the `model_development` section of the documentation. Recall that the training and test datasets in our exercise have the following `input_id` values:\n", - "\n", - "- `train_dataset_final` for the training dataset\n", - "- `test_dataset_final` for the test dataset\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "config = {\n", - " \"validmind.model_validation.ModelMetadata\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\"},\n", - " },\n", - " \"validmind.data_validation.DatasetSplit\": {\n", - " \"inputs\": {\"datasets\": [\"train_dataset_final\", \"test_dataset_final\"]},\n", - " },\n", - " \"validmind.model_validation.sklearn.PopulationStabilityIndex\": {\n", - " \"inputs\": {\n", - " \"model\": \"log_reg_model_v1\",\n", - " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n", - " },\n", - " \"params\": {\"num_bins\": 10, \"mode\": \"fixed\"},\n", - " },\n", - " \"validmind.model_validation.sklearn.ConfusionMatrix\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " },\n", - " \"my_test_provider.ConfusionMatrix\": {\n", - " \"inputs\": {\"dataset\": \"test_dataset_final\", \"model\": \"log_reg_model_v1\"},\n", - " },\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance:in_sample\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"train_dataset_final\"}\n", - " },\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance:out_of_sample\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"}\n", - " },\n", - " \"validmind.model_validation.sklearn.PrecisionRecallCurve\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " },\n", - " \"validmind.model_validation.sklearn.ROCCurve\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " },\n", - " \"validmind.model_validation.sklearn.TrainingTestDegradation\": {\n", - " \"inputs\": {\n", - " \"model\": \"log_reg_model_v1\",\n", - " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n", - " },\n", - " \"params\": {\n", - " \"metrics\": [\"accuracy\", \"precision\", \"recall\", \"f1\"],\n", - " \"max_threshold\": 0.1,\n", - " },\n", - " },\n", - " \"validmind.model_validation.sklearn.MinimumAccuracy\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " \"params\": {\"min_threshold\": 0.7},\n", - " },\n", - " \"validmind.model_validation.sklearn.MinimumF1Score\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " \"params\": {\"min_threshold\": 0.5},\n", - " },\n", - " \"validmind.model_validation.sklearn.MinimumROCAUCScore\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " \"params\": {\"min_threshold\": 0.5},\n", - " },\n", - " \"validmind.model_validation.sklearn.PermutationFeatureImportance\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " },\n", - " \"validmind.model_validation.sklearn.SHAPGlobalImportance\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " \"params\": {\"kernel_explainer_samples\": 10},\n", - " },\n", - " \"validmind.model_validation.sklearn.WeakspotsDiagnosis\": {\n", - " \"inputs\": {\n", - " \"model\": \"log_reg_model_v1\",\n", - " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n", - " },\n", - " \"params\": {\n", - " \"thresholds\": {\"accuracy\": 0.75, \"precision\": 0.5, \"recall\": 0.5, \"f1\": 0.7}\n", - " },\n", - " },\n", - " \"validmind.model_validation.sklearn.OverfitDiagnosis\": {\n", - " \"inputs\": {\n", - " \"model\": \"log_reg_model_v1\",\n", - " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n", - " },\n", - " \"params\": {\"cut_off_percentage\": 4},\n", - " },\n", - " \"validmind.model_validation.sklearn.RobustnessDiagnosis\": {\n", - " \"inputs\": {\n", - " \"model\": \"log_reg_model_v1\",\n", - " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n", - " },\n", - " \"params\": {\n", - " \"scaling_factor_std_dev_list\": [0.0, 0.1, 0.2, 0.3, 0.4, 0.5],\n", - " \"accuracy_decay_threshold\": 4,\n", - " },\n", - " },\n", - "}\n", - "\n", - "\n", - "full_suite = vm.run_documentation_tests(\n", - " section=\"model_development\",\n", - " config=config,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Where to go from here\n", - "\n", - "In this notebook you have learned the end-to-end process to document a model with the ValidMind Library, running through some very common scenarios in a typical model development setting:\n", - "\n", - "- Running out-of-the-box tests\n", - "- Documenting your model by adding evidence to model documentation\n", - "- Extending the capabilities of the ValidMind Library by implementing custom tests\n", - "- Ensuring that the documentation is complete by running all tests in the documentation template\n", - "\n", - "As a next step, you can explore the following notebooks to get a deeper understanding on how the ValidMind Library allows you generate model documentation for any use case:\n", - "\n", - "\n", - "\n", - "### Use cases\n", - "\n", - "- [Application scorecard demo](../code_samples/credit_risk/application_scorecard_demo.ipynb)\n", - "- [Linear regression documentation demo](../code_samples/regression/quickstart_regression_full_suite.ipynb)\n", - "- [LLM model documentation demo](../code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb)\n", - "\n", - "\n", - "\n", - "### More how-to guides and code samples\n", - "\n", - "- [Explore available tests in detail](../how_to/explore_tests.ipynb)\n", - "- [In-depth guide for implementing custom tests](../code_samples/custom_tests/implement_custom_tests.ipynb)\n", - "- [In-depth guide to external test providers](../code_samples/custom_tests/integrate_external_test_providers.ipynb)\n", - "- [Configuring dataset features](../how_to/configure_dataset_features.ipynb)\n", - "- [Introduction to unit and composite metrics](../how_to/run_unit_metrics.ipynb)\n", - "\n", - "\n", - "\n", - "### Discover more learning resources\n", - "\n", - "All notebook samples can be found in the following directories of the ValidMind Library GitHub repository:\n", - "\n", - "- [Code samples](https://github.com/validmind/validmind-library/tree/main/notebooks/code_samples)\n", - "- [How-to guides](https://github.com/validmind/validmind-library/tree/main/notebooks/how_to)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Upgrade ValidMind\n", - "\n", - "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n", - "\n", - "Retrieve the information for the currently installed version of ValidMind:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip show validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n", - "\n", - "```bash\n", - "%pip install --upgrade validmind\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You may need to restart your kernel after running the upgrade package for changes to be applied." + "Training is FREE — Register now!
" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "ValidMind Library", "language": "python", - "name": "python3" + "name": "validmind" }, "language_info": { "codemirror_mode": { @@ -1798,7 +35,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/site/notebooks/tutorials/intro_for_model_developers_EXECUTED.ipynb b/site/notebooks/tutorials/intro_for_model_developers_EXECUTED.ipynb deleted file mode 100644 index 95e1b19f28..0000000000 --- a/site/notebooks/tutorials/intro_for_model_developers_EXECUTED.ipynb +++ /dev/null @@ -1,1806 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# ValidMind Introduction for Model Developers\n", - "\n", - "As a model developer, learn how the end-to-end documentation process works based on common scenarios you encounter in model development settings.\n", - "\n", - "As a prerequisite, a model documentation template must be available on the ValidMind Platform. You can [view the available templates](https://docs.validmind.com/guide/model-documentation/swap-documentation-templates.html#view-current-templates) to see what templates been set up for your organization.\n", - "\n", - "This notebook uses a binary classification model as an example, but the same principles shown here apply to other model types.\n", - "\n", - "\n", - "\n", - "## Overview of the notebook\n", - "\n", - "**1. Initializing the ValidMind Library**\n", - "\n", - "The ValidMind Library provides a rich collection of documentation tools and test suites, from documenting descriptions of datasets to validation and testing of models using a variety of open-source testing frameworks.\n", - "\n", - "**2. Start the model development process with raw data, run out-of-the box tests, and add evidence to model documentation**\n", - "\n", - "Learn how to access ValidMind's test repository of individual tests that you will use as building blocks to ensure a model is being built appropriately. The goal is to show how to run tests, investigate results, and add tests results or evidence to the documentation.\n", - "\n", - "For a full list of out-of-the-box tests, see [Test descriptions](https://docs.validmind.ai/developer/model-testing/test-descriptions.html) or try the interactive [Test sandbox](https://docs.validmind.ai/developer/model-testing/test-sandbox.html).\n", - "\n", - "**3. Implementing custom tests**\n", - "\n", - "Usually, model developers have their own custom tests and it is important to include this within the model documentation. We will show you how to include custom tests and then how they can be added to documentation as additional evidence.\n", - "\n", - "**4. Finalize testing and documentation**\n", - "\n", - "Learn how you can ensure that model documentation includes custom tests and how to make test configuration changes that apply to all tests in the model documentation template. At the end of this section you should have a fully documented model ready for review.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Contents\n", - "\n", - "- [Overview of the notebook](#toc1_)\n", - "- [About ValidMind](#toc2_)\n", - " - [Before you begin](#toc2_1_)\n", - " - [New to ValidMind?](#toc2_2_)\n", - " - [Key concepts](#toc2_3_)\n", - "- [1. Initializing the ValidMind Library](#toc4_)\n", - " - [Install the ValidMind Library](#toc4_1_)\n", - " - [Initialize the ValidMind Library](#toc4_2_)\n", - " - [Get your code snippet](#toc4_2_1_)\n", - " - [Verify & preview the documentation template](#toc4_3_)\n", - "- [2. Start the model development process with raw data, run out-of-the box tests, and add evidence to model documentation](#toc5_)\n", - " - [Initialize the ValidMind datasets](#toc5_1_)\n", - " - [Run some tabular data tests](#toc5_2_)\n", - " - [Utilize test output](#toc5_3_)\n", - " - [Documenting the results based on two datasets](#toc5_4_)\n", - " - [Run `run_documentation_tests()` using `vm_raw_dataset_preprocessed` as input](#toc5_4_1_)\n", - " - [Log the individual result of the high correlation test that used `vm_balanced_raw_dataset` (that had a highly correlated `Age` column) as input](#toc5_4_2_)\n", - " - [Add individual test results to model documentation](#toc5_5_)\n", - " - [Model Testing](#toc5_6_)\n", - " - [Initialize model evaluation objects and assigning predictions](#toc5_7_)\n", - " - [Run the model evaluation tests](#toc5_8_)\n", - "- [3. Implementing custom tests](#toc6_)\n", - " - [Create a confusion matrix plot](#toc6_1_)\n", - " - [Add parameters to custom tests](#toc6_2_)\n", - " - [Pass parameters to custom tests](#toc6_3_)\n", - " - [Log the confusion matrix results](#toc6_4_)\n", - " - [Using external test providers](#toc6_5_)\n", - " - [Create a folder of custom tests from existing inline tests](#toc6_5_1_)\n", - " - [Save an inline test to a file](#toc6_5_2_)\n", - " - [Define and register a `LocalTestProvider` that points to that folder](#toc6_5_3_)\n", - " - [Initializing a local test provider](#toc6_6_)\n", - " - [Run test provider tests](#toc6_6_1_)\n", - " - [Add the test results to your documentation](#toc6_6_2_)\n", - "- [4. Finalize testing and documentation](#toc7_)\n", - " - [Use `run_documentation_tests()` to ensure custom test results are included in your documentation](#toc7_1_)\n", - " - [Viewing and updating the configuration for the entire model documentation template](#toc7_2_)\n", - " - [Update the config](#toc7_2_1_)\n", - "- [Where to go from here](#toc8_)\n", - " - [Use cases](#toc8_1_)\n", - " - [More how-to guides and code samples](#toc8_2_)\n", - " - [Discover more learning resources](#toc8_3_)\n", - "- [Upgrade ValidMind](#toc9_)\n", - "\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## About ValidMind\n", - "\n", - "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models. You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n", - "\n", - "\n", - "\n", - "### Before you begin\n", - "\n", - "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language.\n", - "\n", - "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n", - "\n", - "\n", - "\n", - "### New to ValidMind?\n", - "\n", - "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models, find code samples, or read our developer reference.\n", - "\n", - "
For access to all features available in this notebook, create a free ValidMind account.\n", - "

\n", - "Signing up is FREE — Register with ValidMind
\n", - "\n", - "\n", - "\n", - "### Key concepts\n", - "\n", - "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n", - "\n", - "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", - "\n", - "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n", - "\n", - "**Custom tests**: Custom tests are functions that you define to evaluate your model or dataset. These functions can be registered via the ValidMind Library to be used with the ValidMind Platform.\n", - "\n", - "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n", - "\n", - "- **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n", - "- **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n", - "- **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom test.\n", - "- **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom test. See this [example](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html) for more information.\n", - "\n", - "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a test, customize its behavior, or provide additional context.\n", - "\n", - "**Outputs**: Custom tests can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n", - "\n", - "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n", - "\n", - "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## 1. Initializing the ValidMind Library\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Install the ValidMind Library\n", - "\n", - "Please note the following recommended Python versions to use:\n", - "\n", - "- Python 3.8 <= x <= 3.11\n", - "\n", - "To install the library:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "%pip install -q validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the ValidMind Library\n", - "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", - "\n", - "\n", - "\n", - "### Get your code snippet\n", - "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Model Inventory** and click **+ Register Model**.\n", - "\n", - "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", - "\n", - " For example, to register a model for use with this notebook, select:\n", - "\n", - " - Documentation template: `Binary classification`\n", - " - Use case: `Marketing/Sales - Attrition/Churn Management`\n", - "\n", - " You can fill in other options according to your preference.\n", - "\n", - "4. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", - "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Load your model identifier credentials from an `.env` file\n", - "\n", - "%load_ext dotenv\n", - "%dotenv .env\n", - "\n", - "# Or replace with your code snippet\n", - "\n", - "import validmind as vm\n", - "\n", - "vm.init(\n", - " # api_host=\"...\",\n", - " # api_key=\"...\",\n", - " # api_secret=\"...\",\n", - " # model=\"...\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Verify & preview the documentation template\n", - "\n", - "Let's verify that you have connected to ValidMind and that the appropriate template is selected. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", - "\n", - "You will upload documentation and test results for this template later on. For now, take a look at the structure that the template provides with the `vm.preview_template()` function from the ValidMind library and note the empty sections:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.preview_template()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Before learning how to run tests, let's explore the list of all available tests in the ValidMind Library. You can see that the documentation template for this model has references to some of the test IDs listed below.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.tests.list_tests()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## 2. Start the model development process with raw data, run out-of-the box tests, and add evidence to model documentation\n", - "\n", - "In this section you learn how to explore the individual tests available in ValidMind and how to run them and change parameters as necessary. You will use a public dataset from Kaggle that models a bank customer churn prediction use case. The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n", - "\n", - "You can find more information about this dataset [here](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction).\n", - "\n", - "The ValidMind Library provides a wrapper to automatically load the dataset as a Pandas DataFrame object.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "from validmind.datasets.classification import customer_churn as demo_dataset\n", - "\n", - "print(\n", - " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{demo_dataset.target_column}' \\n\\t• Class labels: {demo_dataset.class_labels}\"\n", - ")\n", - "\n", - "raw_df = demo_dataset.load_data()\n", - "raw_df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's do some data quality assessments by running a few individual tests related to data assessment. You will use the `vm.tests.list_tests()` function introduced above in combination with `vm.tests.list_tags()` and `vm.tests.list_tasks()` to find which prebuilt tests are relevant for data quality assessment.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Get the list of available tags\n", - "sorted(vm.tests.list_tags())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Get the list of available task types\n", - "sorted(vm.tests.list_tasks())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can pass `tags` and `tasks` as parameters to the `vm.tests.list_tests()` function to filter the tests based on the tags and task types. For example, to find tests related to tabular data quality for classification models, you can call `list_tests()` like this:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.tests.list_tests(task=\"classification\", tags=[\"tabular_data\", \"data_quality\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the ValidMind datasets\n", - "\n", - "Now, assume we have identified some tests we want to run with regards to the data we are intending to use. The next step is to connect your data with a ValidMind `Dataset` object. This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind. You only need to do it one time per dataset.\n", - "\n", - "You can initialize a ValidMind dataset object using the [`init_dataset`](https://docs.validmind.ai/validmind/validmind.html#init_dataset) function from the ValidMind (`vm`) module.\n", - "\n", - "This function takes a number of arguments:\n", - "\n", - "- `dataset` — the raw dataset that you want to provide as input to tests\n", - "- `input_id` - a unique identifier that allows tracking what inputs are used when running each individual test\n", - "- `target_column` — a required argument if tests require access to true values. This is the name of the target column in the dataset\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# vm_raw_dataset is now a VMDataset object that you can pass to any ValidMind test\n", - "vm_raw_dataset = vm.init_dataset(\n", - " dataset=raw_df,\n", - " input_id=\"raw_dataset\",\n", - " target_column=\"Exited\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Run some tabular data tests\n", - "\n", - "Individual tests can be easily run by calling the `run_test` function provided by the `validmind.tests` module. The function takes the following arguments:\n", - "\n", - "- `test_id`: The ID of the test to run. To find a particular test and get its ID, refer to the [explore_tests](../how_to/explore_tests.ipynb) notebook. Look above for example after running 'vm.test_suites.describe_suite' as column 'Test ID' will contain the id.\n", - "- `params`: A dictionary of parameters for the test. These will override any `default_params` set in the test definition. Refer to the [explore_tests](../how_to/explore_tests.ipynb) notebook to find the default parameters for a test. See below for examples.\n", - "\n", - "The inputs expected by a test can also be found in the test definition. Let's take `validmind.data_validation.DescriptiveStatistics` as an example. Note that the output of the `describe_test()` function below shows that this test expects a `dataset` as input:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.tests.describe_test(\"validmind.data_validation.DescriptiveStatistics\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, let's run a few tests to assess the quality of the dataset.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.DescriptiveStatistics\",\n", - " inputs={\"dataset\": vm_raw_dataset},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result2 = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.ClassImbalance\",\n", - " inputs={\"dataset\": vm_raw_dataset},\n", - " params={\"min_percent_threshold\": 30},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can see that the class imbalance test did not pass according to the value of `min_percent_threshold` we have set. Here is how you can re-run the test on some processed data to address this data quality issue. In this case we apply a very simple rebalancing technique to the dataset.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "raw_copy_df = raw_df.sample(frac=1) # Create a copy of the raw dataset\n", - "\n", - "# Create a balanced dataset with the same number of exited and not exited customers\n", - "exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 1]\n", - "not_exited_df = raw_copy_df.loc[raw_copy_df[\"Exited\"] == 0].sample(n=exited_df.shape[0])\n", - "\n", - "balanced_raw_df = pd.concat([exited_df, not_exited_df])\n", - "balanced_raw_df = balanced_raw_df.sample(frac=1, random_state=42)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "With this new raw dataset, you can re-run the individual test to see if it passes the class imbalance test requirement. Remember to register new VM Dataset object since that is the type of input required by `run_test()`:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Register new data and now 'balanced_raw_dataset' is the new dataset object of interest\n", - "vm_balanced_raw_dataset = vm.init_dataset(\n", - " dataset=balanced_raw_df,\n", - " input_id=\"balanced_raw_dataset\",\n", - " target_column=\"Exited\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.ClassImbalance\",\n", - " inputs={\"dataset\": vm_balanced_raw_dataset},\n", - " params={\"min_percent_threshold\": 30},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Utilize test output\n", - "\n", - "Here is an example for how you can utilize the output from a ValidMind test for futher use, for example, if you want to remove highly correlated features. The example below shows how you can get the list of features with the highest correlation coefficients and use them to reduce the final list of features for modeling.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "corr_result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n", - " params={\"max_threshold\": 0.3},\n", - " inputs={\"dataset\": vm_balanced_raw_dataset},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's assume we want to remove highly correlated features from the dataset. `corr_result` is an object of type `TestResult`. We can inspect the result object to see what the test has produced." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "print(type(corr_result))\n", - "print(\"Result ID: \", corr_result.result_id)\n", - "print(\"Params: \", corr_result.params)\n", - "print(\"Passed: \", corr_result.passed)\n", - "print(\"Tables: \", corr_result.tables)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's check out the table in the result and extract a list of features that failed the test:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "features_df = corr_result.tables[0].data\n", - "features_df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Remove the highly correlated features and create a new VM dataset object. Note the use of different `input_id`s. This allows tracking the inputs used when running each individual test.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "high_correlation_features = features_df[features_df[\"Pass/Fail\"] == \"Fail\"][\"Columns\"].tolist()\n", - "high_correlation_features" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Extract the feature names from the list of strings (e.g. '(Age, Exited)' -> 'Age')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_correlation_features = [feature.split(\",\")[0].strip(\"()\") for feature in high_correlation_features]\n", - "high_correlation_features" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Remove the highly correlated features from the dataset\n", - "balanced_raw_no_age_df = balanced_raw_df.drop(columns=high_correlation_features)\n", - "\n", - "# Re-initialize the dataset object\n", - "vm_raw_dataset_preprocessed = vm.init_dataset(\n", - " dataset=balanced_raw_no_age_df,\n", - " input_id=\"raw_dataset_preprocessed\",\n", - " target_column=\"Exited\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Re-running the test with the reduced feature set should pass the test. You can also plot the correlation matrix to visualize the new correlation between features:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "corr_result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.HighPearsonCorrelation\",\n", - " params={\"max_threshold\": 0.3},\n", - " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "corr_result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.PearsonCorrelationMatrix\",\n", - " inputs={\"dataset\": vm_raw_dataset_preprocessed},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Documenting the results based on two datasets\n", - "\n", - "We have now done some analysis on two different datasets and we should able to document why certain things were done to the raw data with testing to support it. Every test result returned by the `run_test()` function has a `.log()` method that can be used to log the test results to ValidMind. When logging individual results to ValidMind you need to manually add those results in a specific section of the model documentation.\n", - "\n", - "When using `run_documentation_tests()`, it's possible to automatically populate a section with the results of all tests that were registered in the documentation template.\n", - "\n", - "To show how to add individual results to any documentation section, we're going to populate the entire `data_preparation` section of the documentation using the clean `vm_raw_dataset_preprocessed` dataset as input, and then we're going to document an additional result for the highly correlated dataset `vm_balanced_raw_dataset`. The following two steps will accomplish this:\n", - "\n", - "1. Run `run_documentation_tests()` using `vm_raw_dataset_preprocessed` as input. This populates the entire data preparation section for every test that is already part of the documentation template.\n", - "2. Log the individual result of the high correlation test that used `vm_balanced_raw_dataset` (that had a highly correlated `Age` column) as input\n", - "\n", - "After adding the result of step #2 to the documentation you will be able to explain the changes made to the raw data by editing the default description of the test result within the ValidMind Platform.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Run `run_documentation_tests()` using `vm_raw_dataset_preprocessed` as input\n", - "\n", - "`run_documentation_tests()` allows you to run multiple tests at once and log the results to the documentation. The function takes the following arguments:\n", - "\n", - "- `inputs`: any inputs to be passed to the tests\n", - "- `config`: a dictionary `:` that allows configuring each test individually. Each test config has the following form:\n", - " - `params`: individual test parameters\n", - " - `inputs`: individual test inputs. When passed, this overrides any inputs passed from the `run_documentation_tests()` function\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "test_config = {\n", - " \"validmind.data_validation.ClassImbalance\": {\n", - " \"params\": {\"min_percent_threshold\": 30},\n", - " },\n", - " \"validmind.data_validation.HighPearsonCorrelation\": {\n", - " \"params\": {\"max_threshold\": 0.3},\n", - " },\n", - "}\n", - "\n", - "tests_suite = vm.run_documentation_tests(\n", - " inputs={\n", - " \"dataset\": vm_raw_dataset_preprocessed,\n", - " },\n", - " config=test_config,\n", - " section=[\"data_preparation\"],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Log the individual result of the high correlation test that used `vm_balanced_raw_dataset` (that had a highly correlated `Age` column) as input\n", - "\n", - "Here you can use a custom `result_id` to tag the individual result with a unique identifier. This `result_id` can be appended to `test_id` with a `:` separator. The `balanced_raw_dataset` result identifier will correspond to the `balanced_raw_dataset` input, the dataset that still has the `Age` column.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result = vm.tests.run_test(\n", - " test_id=\"validmind.data_validation.HighPearsonCorrelation:balanced_raw_dataset\",\n", - " params={\"max_threshold\": 0.3},\n", - " inputs={\"dataset\": vm_balanced_raw_dataset},\n", - ")\n", - "result.log()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Add individual test results to model documentation\n", - "\n", - "You can now visit the documentation page for the model you connected to at the beginning of this notebook and add a new content block in the relevant section.\n", - "\n", - "To do this, go to the documentation page of your model and navigate to the `Data Preparation` -> `Correlations and Interactions` section. Then hover after the \"Pearson Correlation Matrix\" content block to reveal the `+` button as shown in the screenshot below.\n", - "\n", - "![screenshot showing insert button for test-driven blocks](../images/insert-test-driven-block-correlations.png)\n", - "\n", - "Click on the `+` button and select `Test-Driven Block`. This will open a dialog where you can select `Threshold Test` as the type of the test-driven content block, and then select `High Pearson Correlation Vm Raw Dataset Test`. This will show a preview of the result and it should match the results shown above.\n", - "\n", - "![screenshot showing the selected test result in the dialog](../images/selecting-high-pearson-correlation-test.png)\n", - "\n", - "Finally, click on the `Insert block` button to add the test result to the documentation. You'll now see two individual results for the high correlation test in the `Correlations and Interactions` section of the documentation. To finalize the documentation, you can edit the test result's description block to explain the changes made to the raw data and the reasons behind them as we can see in the screenshot below.\n", - "\n", - "![screenshot showing the high pearson correlation block](../images/high-pearson-correlation-block.png)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Model Testing\n", - "\n", - "We have focused so far on the data assessment and pre-processing that usually occurs prior to any models being built. Now we are going to assume we have built a model and we want to incorporate some model results in our documentation.\n", - "\n", - "Let's train a simple logistic regression model on the dataset and evaluate its performance. You will use the `LogisticRegression` class from the `sklearn.linear_model` and use ValidMind tests to evaluate the model's performance.\n", - "\n", - "Before training the model, we need to encode the categorical features in the dataset. You will use the `OneHotEncoder` class from the `sklearn.preprocessing` module to encode the categorical features. The categorical features in the dataset are `Geography` and `Gender`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "balanced_raw_no_age_df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "balanced_raw_no_age_df = pd.get_dummies(\n", - " balanced_raw_no_age_df, columns=[\"Geography\", \"Gender\"], drop_first=True\n", - ")\n", - "balanced_raw_no_age_df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "# Split the input and target variables\n", - "X = balanced_raw_no_age_df.drop(\"Exited\", axis=1)\n", - "y = balanced_raw_no_age_df[\"Exited\"]\n", - "X_train, X_test, y_train, y_test = train_test_split(\n", - " X,\n", - " y,\n", - " test_size=0.2,\n", - " random_state=42,\n", - ")\n", - "\n", - "# Logistic Regression grid params\n", - "log_reg_params = {\n", - " \"penalty\": [\"l1\", \"l2\"],\n", - " \"C\": [0.001, 0.01, 0.1, 1, 10, 100, 1000],\n", - " \"solver\": [\"liblinear\"],\n", - "}\n", - "\n", - "# Grid search for Logistic Regression\n", - "from sklearn.model_selection import GridSearchCV\n", - "\n", - "grid_log_reg = GridSearchCV(LogisticRegression(), log_reg_params)\n", - "grid_log_reg.fit(X_train, y_train)\n", - "\n", - "# Logistic Regression best estimator\n", - "log_reg = grid_log_reg.best_estimator_" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize model evaluation objects and assigning predictions\n", - "\n", - "The last step for evaluating the model's performance is to initialize the ValidMind `Dataset` and `Model` objects and assign model predictions to each dataset. You will use the `init_dataset`, `init_model` and `assign_predictions` functions to initialize these objects.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "train_df = X_train\n", - "train_df[\"Exited\"] = y_train\n", - "test_df = X_test\n", - "test_df[\"Exited\"] = y_test\n", - "\n", - "vm_train_ds = vm.init_dataset(\n", - " input_id=\"train_dataset_final\",\n", - " dataset=train_df,\n", - " target_column=\"Exited\",\n", - ")\n", - "\n", - "vm_test_ds = vm.init_dataset(\n", - " input_id=\"test_dataset_final\",\n", - " dataset=test_df,\n", - " target_column=\"Exited\",\n", - ")\n", - "\n", - "# Register the model\n", - "vm_model = vm.init_model(log_reg, input_id=\"log_reg_model_v1\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once the model has been registered you can assign model predictions to the training and test datasets. The `assign_predictions()` method from the `Dataset` object can link existing predictions to any number of models. If no prediction values are passed, the method will compute predictions automatically:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm_train_ds.assign_predictions(model=vm_model)\n", - "vm_test_ds.assign_predictions(model=vm_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Run the model evaluation tests\n", - "\n", - "In this part, we focus on running the tests within the model development section of the model documentation. Only tests associated with this section will be executed, and the corresponding results will be updated in the model documentation. In the example below, you will focus on only running tests for the `model development` section of the document.\n", - "\n", - "Note the additional config that is passed to `run_documentation_tests()`. This allows you to override inputs or params in certain tests. In our case, we want to explicitly use the `vm_train_ds` for the `validmind.model_validation.sklearn.ClassifierPerformance:in_sample` test, since it's supposed to run on the training dataset and not the test dataset.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "test_config = {\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance:in_sample\": {\n", - " \"inputs\": {\n", - " \"dataset\": vm_train_ds,\n", - " \"model\": vm_model,\n", - " },\n", - " }\n", - "}\n", - "results = vm.run_documentation_tests(\n", - " section=[\"model_development\"],\n", - " inputs={\n", - " \"dataset\": vm_test_ds, # Any test that requires a single dataset will use vm_test_ds\n", - " \"model\": vm_model,\n", - " \"datasets\": (\n", - " vm_train_ds,\n", - " vm_test_ds,\n", - " ), # Any test that requires multiple datasets will use vm_train_ds and vm_test_ds\n", - " },\n", - " config=test_config,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## 3. Implementing custom tests\n", - "\n", - "This section assumes that model developers already have a repository of custom made tests that they consider critical to include in the documentation. Here we provide details on how to easily integrate custom tests with ValidMind.\n", - "\n", - "For a more in-depth introduction to custom tests, refer to this [notebook](../code_samples/custom_tests/implement_custom_tests.ipynb).\n", - "\n", - "A custom test is any function that takes a set of inputs and parameters as arguments and returns one or more outputs. The function can be as simple or as complex as you need it to be. It can use external libraries, make API calls, or do anything else that you can do in Python. The only requirement is that the function signature and return values can be \"understood\" and handled by the ValidMind Library. As such, custom tests offer added flexibility by extending the default tests provided by ValidMind, enabling you to document any type of model or use case.\n", - "\n", - "In the following example, you will learn how to implement a custom `inline` test that calculates the confusion matrix for a binary classification model. You will see that the custom test function is just a regular Python function that can include and require any Python library as you see fit.\n", - "\n", - "**NOTE**: in the context of Jupyter notebooks, we will use the word `inline` to refer to functions (or code) defined in the same notebook where they are used (this one) and not in a separate file, as we will see later with test providers.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Create a confusion matrix plot\n", - "\n", - "To understand how to create a custom test from anything, let's first create a confusion matrix plot using the `confusion_matrix` function from the `sklearn.metrics` module.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "from sklearn import metrics\n", - "\n", - "# Get the predicted classes\n", - "y_pred = log_reg.predict(vm_test_ds.x)\n", - "\n", - "confusion_matrix = metrics.confusion_matrix(y_test, y_pred)\n", - "\n", - "cm_display = metrics.ConfusionMatrixDisplay(\n", - " confusion_matrix=confusion_matrix, display_labels=[False, True]\n", - ")\n", - "cm_display.plot()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will now create a @vm.test wrapper that will allow you to create a reusable test. Note the following changes in the code below:\n", - "\n", - "- The function `confusion_matrix` takes two arguments `dataset` and `model`. This is a `VMDataset` and `VMModel` object respectively.\n", - " - `VMDataset` objects allow you to access the dataset's true (target) values by accessing the `.y` attribute.\n", - " - `VMDataset` objects allow you to access the predictions for a given model by accessing the `.y_pred()` method.\n", - "- The function docstring provides a description of what the test does. This will be displayed along with the result in this notebook as well as in the ValidMind Platform.\n", - "- The function body calculates the confusion matrix using the `sklearn.metrics.confusion_matrix` function as we just did above.\n", - "- The function then returns the `ConfusionMatrixDisplay.figure_` object - this is important as the ValidMind Library expects the output of the custom test to be a plot or a table.\n", - "- The `@vm.test` decorator is doing the work of creating a wrapper around the function that will allow it to be run by the ValidMind Library. It also registers the test so it can be found by the ID `my_custom_tests.ConfusionMatrix` (see the section below on how test IDs work in ValidMind and why this format is important)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "@vm.test(\"my_custom_tests.ConfusionMatrix\")\n", - "def confusion_matrix(dataset, model):\n", - " \"\"\"The confusion matrix is a table that is often used to describe the performance of a classification model on a set of data for which the true values are known.\n", - "\n", - " The confusion matrix is a 2x2 table that contains 4 values:\n", - "\n", - " - True Positive (TP): the number of correct positive predictions\n", - " - True Negative (TN): the number of correct negative predictions\n", - " - False Positive (FP): the number of incorrect positive predictions\n", - " - False Negative (FN): the number of incorrect negative predictions\n", - "\n", - " The confusion matrix can be used to assess the holistic performance of a classification model by showing the accuracy, precision, recall, and F1 score of the model on a single figure.\n", - " \"\"\"\n", - " y_true = dataset.y\n", - " y_pred = dataset.y_pred(model=model)\n", - "\n", - " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n", - "\n", - " cm_display = metrics.ConfusionMatrixDisplay(\n", - " confusion_matrix=confusion_matrix, display_labels=[False, True]\n", - " )\n", - " cm_display.plot()\n", - "\n", - " plt.close() # close the plot to avoid displaying it\n", - "\n", - " return cm_display.figure_ # return the figure object itself" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can now run the newly created custom test on both the training and test datasets using the `run_test()` function:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Training dataset\n", - "result = vm.tests.run_test(\n", - " \"my_custom_tests.ConfusionMatrix:training_dataset\",\n", - " inputs={\"model\": vm_model, \"dataset\": vm_train_ds},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Test dataset\n", - "result = vm.tests.run_test(\n", - " \"my_custom_tests.ConfusionMatrix:test_dataset\",\n", - " inputs={\"model\": vm_model, \"dataset\": vm_test_ds},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Add parameters to custom tests\n", - "\n", - "Custom tests can take parameters just like any other function. Let's modify the `confusion_matrix` function to take an additional parameter `normalize` that will allow you to normalize the confusion matrix.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "@vm.test(\"my_custom_tests.ConfusionMatrix\")\n", - "def confusion_matrix(dataset, model, normalize=False):\n", - " \"\"\"The confusion matrix is a table that is often used to describe the performance of a classification model on a set of data for which the true values are known.\n", - "\n", - " The confusion matrix is a 2x2 table that contains 4 values:\n", - "\n", - " - True Positive (TP): the number of correct positive predictions\n", - " - True Negative (TN): the number of correct negative predictions\n", - " - False Positive (FP): the number of incorrect positive predictions\n", - " - False Negative (FN): the number of incorrect negative predictions\n", - "\n", - " The confusion matrix can be used to assess the holistic performance of a classification model by showing the accuracy, precision, recall, and F1 score of the model on a single figure.\n", - " \"\"\"\n", - " y_true = dataset.y\n", - " y_pred = dataset.y_pred(model=model)\n", - "\n", - " if normalize:\n", - " confusion_matrix = metrics.confusion_matrix(y_true, y_pred, normalize=\"all\")\n", - " else:\n", - " confusion_matrix = metrics.confusion_matrix(y_true, y_pred)\n", - "\n", - " cm_display = metrics.ConfusionMatrixDisplay(\n", - " confusion_matrix=confusion_matrix, display_labels=[False, True]\n", - " )\n", - " cm_display.plot()\n", - "\n", - " plt.close() # close the plot to avoid displaying it\n", - "\n", - " return cm_display.figure_ # return the figure object itself" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Pass parameters to custom tests\n", - "\n", - "You can pass parameters to custom tests by providing a dictionary of parameters to the `run_test()` function. The parameters will override any default parameters set in the custom test definition. Note that `dataset` and `model` are still passed as `inputs`. Since these are `VMDataset` or `VMModel` inputs, they have a special meaning. When declaring a `dataset`, `model`, `datasets` or `models` argument in a custom test function, the ValidMind Library will expect these get passed as `inputs` to `run_test()` (or `run_documentation_tests()` instead).\n", - "\n", - "Re-running the confusion matrix with `normalize=True` looks like this:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# Test dataset with normalize=True\n", - "result = vm.tests.run_test(\n", - " \"my_custom_tests.ConfusionMatrix:test_dataset_normalized\",\n", - " inputs={\"model\": vm_model, \"dataset\": vm_test_ds},\n", - " params={\"normalize\": True},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Log the confusion matrix results\n", - "\n", - "As you saw in the pearson correlation example, you can log any result to the ValidMind Platform with the `.log()` method of the result object. This will allow you to add the result to the documentation.\n", - "\n", - "You can now do the same for the confusion matrix results.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result.log()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Using external test providers\n", - "\n", - "Creating inline custom tests with a function is a great way to customize your model documentation. However, sometimes you may want to reuse the same set of tests across multiple models and share them with developers in your organization. In this case, you can create a custom test provider that will allow you to load custom tests from a local folder or a git repository.\n", - "\n", - "In this section you will learn how to declare a local filesystem test provider that allows loading tests from a local folder following these high level steps:\n", - "\n", - "1. Create a folder of custom tests from existing, inline tests (tests that exists in your active Jupyter notebook)\n", - "2. Save an inline test to a file\n", - "3. Define and register a `LocalTestProvider` that points to that folder\n", - "4. Run test provider tests\n", - "5. Add the test results to your documentation\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Create a folder of custom tests from existing inline tests\n", - "\n", - "Here you will create a new folder that will contain reusable, custom tests. The following code snippet will create a new `my_tests` directory in the current working directory if it doesn't exist.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "tests_folder = \"my_tests\"\n", - "\n", - "import os\n", - "\n", - "# create tests folder\n", - "os.makedirs(tests_folder, exist_ok=True)\n", - "\n", - "# remove existing tests\n", - "for f in os.listdir(tests_folder):\n", - " # remove files and pycache\n", - " if f.endswith(\".py\") or f == \"__pycache__\":\n", - " os.system(f\"rm -rf {tests_folder}/{f}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After running the command above, you should see a new directory next to this notebook file:\n", - "\n", - "![screenshot showing my_tests directory](../images/my_tests_directory.png)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Save an inline test to a file\n", - "\n", - "The `@vm.test` decorator that was used above to register these as one-off custom tests also adds a convenience method to the function object that allows you to simply call `.save()` to save it to a file. This will save the function to a Python file to a path you specify. In this case, you can pass the variable `tests_folder` to save it to the custom tests folder we created.\n", - "\n", - "Normally, this will get you started by creating the file and saving the function code with the correct name. But it won't automatically add any import or other functions/variables outside of the function that are needed for the test to run. The `save()` method allows you to pass an optional `imports` argument that will ensure the necessary imports are added to the file.\n", - "\n", - "For the `confusion_matrix` test, note the imports that are required for the function to run properly:\n", - "\n", - "```python\n", - "import matplotlib.pyplot as plt\n", - "from sklearn import metrics\n", - "```\n", - "\n", - "You can pass these imports to the `save()` method to ensure they are included in the file with the following command:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "confusion_matrix.save(\n", - " tests_folder,\n", - " imports=[\"import matplotlib.pyplot as plt\", \"from sklearn import metrics\"],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### What happened?\n", - "\n", - "The `save()` method saved the `confusion_matrix` function to a file named `ConfusionMatrix.py` in the `my_tests` folder. Note that the new file provides some context on the origin of the test, which is useful for traceability.\n", - "\n", - "```\n", - "# Saved from __main__.confusion_matrix\n", - "# Original Test ID: my_custom_tests.ConfusionMatrix\n", - "# New Test ID: .ConfusionMatrix\n", - "```\n", - "\n", - "Additionally, the new test function has been stripped off its decorator, as it now resides in a file that will be loaded by the test provider:\n", - "\n", - "```python\n", - "def ConfusionMatrix(dataset, model, normalize=False):\n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Define and register a `LocalTestProvider` that points to that folder\n", - "\n", - "With the `my_tests` folder now having a sample custom test, you can now initialize a test provider that will tell the ValidMind Library where to find these tests. ValidMind offers out-of-the-box test providers for local tests (i.e. tests in a folder) or a Github provider for tests in a Github repository. You can also create your own test provider by creating a class that has a `load_test` method that takes a test ID and returns the test function matching that ID.\n", - "\n", - "The most important attribute for a test provider is its `namespace`. This is a string that will be used to prefix test IDs in model documentation. This allows you to have multiple test providers with tests that can even share the same ID, but are distinguished by their namespace.\n", - "\n", - "An extended introduction to test providers can be found in [this](../code_samples/custom_tests/integrate_external_test_providers.ipynb) notebook.\n", - "\n", - "\n", - "\n", - "### Initializing a local test provider\n", - "\n", - "For most use-cases, the local test provider should be sufficient. This test provider allows you load custom tests from a designated directory. Let's go ahead and see how we can do this with our custom tests.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "from validmind.tests import LocalTestProvider\n", - "\n", - "# initialize the test provider with the tests folder we created earlier\n", - "my_test_provider = LocalTestProvider(tests_folder)\n", - "\n", - "vm.tests.register_test_provider(\n", - " namespace=\"my_test_provider\",\n", - " test_provider=my_test_provider,\n", - ")\n", - "# `my_test_provider.load_test()` will be called for any test ID that starts with `my_test_provider`\n", - "# e.g. `my_test_provider.ConfusionMatrix` will look for a function named `ConfusionMatrix` in `my_tests/ConfusionMatrix.py` file" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Run test provider tests\n", - "\n", - "Now that you have set up the test provider, you can run any test that's located in the tests folder by using the `run_test()` method as with any other test. For tests that reside in a test provider directory, the test ID will be the `namespace` specified when registering the provider, followed by the path to the test file relative to the tests folder. For example, the Confusion Matrix test we created earlier will have the test ID `my_test_provider.ConfusionMatrix`. You could organize the tests in subfolders, say `classification` and `regression`, and the test ID for the Confusion Matrix test would then be `my_test_provider.classification.ConfusionMatrix`.\n", - "\n", - "Let's go ahead and re-run the confusion matrix test by using the test ID `my_test_provider.ConfusionMatrix`. This should load the test from the test provider and run it as before.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "result = vm.tests.run_test(\n", - " \"my_test_provider.ConfusionMatrix\",\n", - " inputs={\"model\": vm_model, \"dataset\": vm_test_ds},\n", - " params={\"normalize\": True},\n", - ")\n", - "\n", - "result.log()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Add the test results to your documentation\n", - "\n", - "You have already seen how to add individual results to the model documentation using the ValidMind Platform. Let's repeat the process and add the confusion matrix to the `Model Development` -> `Model Evaluation` section of the documentation. The \"add test driven block\" dialog should now show the new test result coming from the test provider:\n", - "\n", - "![screenshot showing confusion matrix result](../images/insert-test-driven-block-custom-confusion-matrix.png)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## 4. Finalize testing and documentation\n", - "\n", - "In this section we cover how to finalize the testing and documentation of your model by focusing on:\n", - "\n", - "1. Using `run_documentation_tests()` to ensure custom test results are included in your documentation\n", - "2. Viewing and updating the configuration for the entire model documentation template\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Use `run_documentation_tests()` to ensure custom test results are included in your documentation\n", - "\n", - "After adding test driven blocks to your model documentation, changes should persist and become available every time you call `vm.preview_template()`. However, you need to reload the connection to the ValidMind Platform if you have added test driven blocks when the connection was already established.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.reload()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, run `preview_template()` and verify that the new confusion matrix test you added is included in the proper section.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "vm.preview_template()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Since the test ID is now registered in the document you can now run tests for an entire section and all additional custom tests should be loaded without issues. Let's run all tests in the `model_evaluation` section of the documentation. Note that we have been running the sample custom confusion matrix with `normalize=True` to demonstrate the ability to provide custom parameters.\n", - "\n", - "In the `Run the model evaluation tests` section above you learned how to assign inputs to individual tests with `run_documentation_tests()`. Assigning parametesr is similar, you only need to provide assign a `params` dictionary to a given test ID, `my_test_provider.ConfusionMatrix` in this case.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "test_config = {\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance:in_sample\": {\n", - " \"inputs\": {\n", - " \"dataset\": vm_train_ds,\n", - " \"model\": vm_model,\n", - " },\n", - " },\n", - " \"my_test_provider.ConfusionMatrix\": {\n", - " \"params\": {\"normalize\": True},\n", - " },\n", - "}\n", - "results = vm.run_documentation_tests(\n", - " section=[\"model_evaluation\"],\n", - " inputs={\n", - " \"dataset\": vm_test_ds, # Any test that requires a single dataset will use vm_test_ds\n", - " \"model\": vm_model,\n", - " \"datasets\": (\n", - " vm_train_ds,\n", - " vm_test_ds,\n", - " ), # Any test that requires multiple datasets will use vm_train_ds and vm_test_ds\n", - " },\n", - " config=test_config,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Viewing and updating the configuration for the entire model documentation template\n", - "\n", - "The ValidMind Library provides a utility function called `vm.get_test_suite().get_default_config()` that allows you to render the default configuration for the entire documentation template. This configuration will contain all the test IDs and their default parameters. You can then modify this configuration as needed and pass it to `run_documentation_tests()` to run all tests in the documentation template if needed. You also have the option to continue running tests for one section at a time, `get_default_config()` still provides a useful reference for providing default parametes to every test.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import json\n", - "\n", - "model_test_suite = vm.get_test_suite()\n", - "config = model_test_suite.get_default_config()\n", - "print(\"Suite Config: \\n\", json.dumps(config, indent=2))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Update the config\n", - "\n", - "Note that the default config does not assign any inputs to a test, this is expected. You can assign inputs to individual tests as needed, depending on the datasets and models you want to pass to individual tests. The `config` dictionary, as a mapping of test IDs to test configurations, allows you to do this.\n", - "\n", - "For this particular documentation template (binary classification), the ValidMind Library provides a sample configuration that can be used to populate the entire model documentation using the following inputs as placeholders:\n", - "\n", - "- A `raw_dataset` raw dataset\n", - "- A `train_dataset` training dataset\n", - "- A `test_dataset` test dataset\n", - "- A trained `model` instance\n", - "\n", - "As part of updating the `config` you will need to ensure the correct `input_id`s are used in the final config passed to `run_documentation_tests()`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "from validmind.datasets.classification import customer_churn\n", - "from validmind.utils import preview_test_config\n", - "\n", - "test_config = customer_churn.get_demo_test_config()\n", - "preview_test_config(test_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Using this sample configuration, let's finish populating model documentation by running all tests for the `model_development` section of the documentation. Recall that the training and test datasets in our exercise have the following `input_id` values:\n", - "\n", - "- `train_dataset_final` for the training dataset\n", - "- `test_dataset_final` for the test dataset\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "config = {\n", - " \"validmind.model_validation.ModelMetadata\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\"},\n", - " },\n", - " \"validmind.data_validation.DatasetSplit\": {\n", - " \"inputs\": {\"datasets\": [\"train_dataset_final\", \"test_dataset_final\"]},\n", - " },\n", - " \"validmind.model_validation.sklearn.PopulationStabilityIndex\": {\n", - " \"inputs\": {\n", - " \"model\": \"log_reg_model_v1\",\n", - " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n", - " },\n", - " \"params\": {\"num_bins\": 10, \"mode\": \"fixed\"},\n", - " },\n", - " \"validmind.model_validation.sklearn.ConfusionMatrix\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " },\n", - " \"my_test_provider.ConfusionMatrix\": {\n", - " \"inputs\": {\"dataset\": \"test_dataset_final\", \"model\": \"log_reg_model_v1\"},\n", - " },\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance:in_sample\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"train_dataset_final\"}\n", - " },\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance:out_of_sample\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"}\n", - " },\n", - " \"validmind.model_validation.sklearn.PrecisionRecallCurve\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " },\n", - " \"validmind.model_validation.sklearn.ROCCurve\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " },\n", - " \"validmind.model_validation.sklearn.TrainingTestDegradation\": {\n", - " \"inputs\": {\n", - " \"model\": \"log_reg_model_v1\",\n", - " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n", - " },\n", - " \"params\": {\n", - " \"metrics\": [\"accuracy\", \"precision\", \"recall\", \"f1\"],\n", - " \"max_threshold\": 0.1,\n", - " },\n", - " },\n", - " \"validmind.model_validation.sklearn.MinimumAccuracy\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " \"params\": {\"min_threshold\": 0.7},\n", - " },\n", - " \"validmind.model_validation.sklearn.MinimumF1Score\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " \"params\": {\"min_threshold\": 0.5},\n", - " },\n", - " \"validmind.model_validation.sklearn.MinimumROCAUCScore\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " \"params\": {\"min_threshold\": 0.5},\n", - " },\n", - " \"validmind.model_validation.sklearn.PermutationFeatureImportance\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " },\n", - " \"validmind.model_validation.sklearn.SHAPGlobalImportance\": {\n", - " \"inputs\": {\"model\": \"log_reg_model_v1\", \"dataset\": \"test_dataset_final\"},\n", - " \"params\": {\"kernel_explainer_samples\": 10},\n", - " },\n", - " \"validmind.model_validation.sklearn.WeakspotsDiagnosis\": {\n", - " \"inputs\": {\n", - " \"model\": \"log_reg_model_v1\",\n", - " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n", - " },\n", - " \"params\": {\n", - " \"thresholds\": {\"accuracy\": 0.75, \"precision\": 0.5, \"recall\": 0.5, \"f1\": 0.7}\n", - " },\n", - " },\n", - " \"validmind.model_validation.sklearn.OverfitDiagnosis\": {\n", - " \"inputs\": {\n", - " \"model\": \"log_reg_model_v1\",\n", - " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n", - " },\n", - " \"params\": {\"cut_off_percentage\": 4},\n", - " },\n", - " \"validmind.model_validation.sklearn.RobustnessDiagnosis\": {\n", - " \"inputs\": {\n", - " \"model\": \"log_reg_model_v1\",\n", - " \"datasets\": [\"train_dataset_final\", \"test_dataset_final\"],\n", - " },\n", - " \"params\": {\n", - " \"scaling_factor_std_dev_list\": [0.0, 0.1, 0.2, 0.3, 0.4, 0.5],\n", - " \"accuracy_decay_threshold\": 4,\n", - " },\n", - " },\n", - "}\n", - "\n", - "\n", - "full_suite = vm.run_documentation_tests(\n", - " section=\"model_development\",\n", - " config=config,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Where to go from here\n", - "\n", - "In this notebook you have learned the end-to-end process to document a model with the ValidMind Library, running through some very common scenarios in a typical model development setting:\n", - "\n", - "- Running out-of-the-box tests\n", - "- Documenting your model by adding evidence to model documentation\n", - "- Extending the capabilities of the ValidMind Library by implementing custom tests\n", - "- Ensuring that the documentation is complete by running all tests in the documentation template\n", - "\n", - "As a next step, you can explore the following notebooks to get a deeper understanding on how the ValidMind Library allows you generate model documentation for any use case:\n", - "\n", - "\n", - "\n", - "### Use cases\n", - "\n", - "- [Application scorecard demo](../code_samples/credit_risk/application_scorecard_demo.ipynb)\n", - "- [Linear regression documentation demo](../code_samples/regression/quickstart_regression_full_suite.ipynb)\n", - "- [LLM model documentation demo](../code_samples/nlp_and_llm/foundation_models_integration_demo.ipynb)\n", - "\n", - "\n", - "\n", - "### More how-to guides and code samples\n", - "\n", - "- [Explore available tests in detail](../how_to/explore_tests.ipynb)\n", - "- [In-depth guide for implementing custom tests](../code_samples/custom_tests/implement_custom_tests.ipynb)\n", - "- [In-depth guide to external test providers](../code_samples/custom_tests/integrate_external_test_providers.ipynb)\n", - "- [Configuring dataset features](../how_to/configure_dataset_features.ipynb)\n", - "- [Introduction to unit and composite metrics](../how_to/run_unit_metrics.ipynb)\n", - "\n", - "\n", - "\n", - "### Discover more learning resources\n", - "\n", - "All notebook samples can be found in the following directories of the ValidMind Library GitHub repository:\n", - "\n", - "- [Code samples](https://github.com/validmind/validmind-library/tree/main/notebooks/code_samples)\n", - "- [How-to guides](https://github.com/validmind/validmind-library/tree/main/notebooks/how_to)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Upgrade ValidMind\n", - "\n", - "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n", - "\n", - "Retrieve the information for the currently installed version of ValidMind:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip show validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n", - "\n", - "```bash\n", - "%pip install --upgrade validmind\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You may need to restart your kernel after running the upgrade package for changes to be applied." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/site/notebooks/tutorials/model_development/1-set_up_validmind.ipynb b/site/notebooks/tutorials/model_development/1-set_up_validmind.ipynb index 46a002a83d..b79b0b1b27 100644 --- a/site/notebooks/tutorials/model_development/1-set_up_validmind.ipynb +++ b/site/notebooks/tutorials/model_development/1-set_up_validmind.ipynb @@ -9,7 +9,11 @@ "\n", "Learn how to use ValidMind for your end-to-end model documentation process based on common model development scenarios with our series of four introductory notebooks. This first notebook walks you through the initial setup of the ValidMind Library.\n", "\n", - "These notebooks use a binary classification model as an example, but the same principles shown here apply to other model types." + "These notebooks use a binary classification model as an example, but the same principles shown here apply to other model types.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for developers new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Developer Fundamentals
" ] }, { diff --git a/site/notebooks/tutorials/model_development/2-start_development_process.ipynb b/site/notebooks/tutorials/model_development/2-start_development_process.ipynb index e886dc7ff4..939b907661 100644 --- a/site/notebooks/tutorials/model_development/2-start_development_process.ipynb +++ b/site/notebooks/tutorials/model_development/2-start_development_process.ipynb @@ -10,7 +10,11 @@ "\n", "You'll become familiar with the individual tests available in ValidMind, as well as how to run them and change parameters as necessary. Using ValidMind's repository of individual tests as building blocks helps you ensure that a model is being built appropriately. \n", "\n", - "**For a full list of out-of-the-box tests,** refer to our [Test descriptions](https://docs.validmind.ai/developer/model-testing/test-descriptions.html) or try the interactive [Test sandbox](https://docs.validmind.ai/developer/model-testing/test-sandbox.html)." + "**For a full list of out-of-the-box tests,** refer to our [Test descriptions](https://docs.validmind.ai/developer/model-testing/test-descriptions.html) or try the interactive [Test sandbox](https://docs.validmind.ai/developer/model-testing/test-sandbox.html).\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for developers new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Developer Fundamentals
" ] }, { @@ -74,6 +78,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "\n", + "\n", "## Setting up" ] }, @@ -686,8 +692,9 @@ " \"Screenshot\n", "

\n", "\n", - "5. Click **+** and then select **Test-Driven Block**:\n", + "5. Click **+** and then select **Test-Driven Block** under FROM LIBRARY:\n", "\n", + " - Click on **VM Library** under TEST-DRIVEN in the left sidebar.\n", " - In the search bar, type in `HighPearsonCorrelation`.\n", " - Select `HighPearsonCorrelation:balanced_raw_dataset` as the test.\n", "\n", diff --git a/site/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb b/site/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb index 038f45c38a..ab7a4c8805 100644 --- a/site/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb +++ b/site/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb @@ -13,7 +13,11 @@ "- The function can be as simple or as complex as you need it to be — it can use external libraries, make API calls, or do anything else that you can do in Python.\n", "- The only requirement is that the function signature and return values can be \"understood\" and handled by the ValidMind Library. As such, custom tests offer added flexibility by extending the default tests provided by ValidMind, enabling you to document any type of model or use case.\n", "\n", - "**For a more in-depth introduction to custom tests,** refer to our [Implement custom tests](../../code_samples/custom_tests/implement_custom_tests.ipynb) notebook." + "**For a more in-depth introduction to custom tests,** refer to our [Implement custom tests](../../code_samples/custom_tests/implement_custom_tests.ipynb) notebook.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for developers new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Developer Fundamentals
" ] }, { @@ -913,10 +917,10 @@ " \"Screenshot\n", "

\n", "\n", - "5. Click **+** and then select **Test-Driven Block**:\n", + "5. Click **+** and then select **Test-Driven Block** under FROM LIBRARY:\n", "\n", - " - In the search bar, type in `ConfusionMatrix`.\n", - " - Select the custom `ConfusionMatrix` tests you logged above:\n", + " - Click on **Custom** under TEST-DRIVEN in the left sidebar.\n", + " - Select the two custom `ConfusionMatrix` tests you logged above:\n", "\n", " \"Screenshot\n", "

\n", diff --git a/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb b/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb index 13a4f1e148..e38978aec3 100644 --- a/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb +++ b/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb @@ -8,7 +8,11 @@ "\n", "Learn how to use ValidMind for your end-to-end model documentation process with our introductory notebook series. In this last notebook, finalize the testing and documentation of your model and have a fully documented sample model ready for review.\n", "\n", - "We'll first use [`run_documentation_tests()`](https://docs.validmind.ai/validmind/validmind.html#run_documentation_tests) previously covered in **[2 — Start the model development process](2-start_development_process.ipynb)** to ensure that your custom test results generated in **[3 — Integrate custom tests](3-integrate_custom_tests.ipynb)** are included in your documentation. Then, we'll view and update the configuration for the entire model documentation template to suit your needs.\n" + "We'll first use [`run_documentation_tests()`](https://docs.validmind.ai/validmind/validmind.html#run_documentation_tests) previously covered in **[2 — Start the model development process](2-start_development_process.ipynb)** to ensure that your custom test results generated in **[3 — Integrate custom tests](3-integrate_custom_tests.ipynb)** are included in your documentation. Then, we'll view and update the configuration for the entire model documentation template to suit your needs.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for developers new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Developer Fundamentals
" ] }, { diff --git a/site/notebooks/tutorials/model_development/add-content-block.gif b/site/notebooks/tutorials/model_development/add-content-block.gif index daab9d44d8..60d9e5825d 100644 Binary files a/site/notebooks/tutorials/model_development/add-content-block.gif and b/site/notebooks/tutorials/model_development/add-content-block.gif differ diff --git a/site/notebooks/tutorials/model_development/selecting-confusion-matrix-test.png b/site/notebooks/tutorials/model_development/selecting-confusion-matrix-test.png index de27323a76..ce32255cc4 100644 Binary files a/site/notebooks/tutorials/model_development/selecting-confusion-matrix-test.png and b/site/notebooks/tutorials/model_development/selecting-confusion-matrix-test.png differ diff --git a/site/notebooks/tutorials/model_development/selecting-high-pearson-correlation-test.png b/site/notebooks/tutorials/model_development/selecting-high-pearson-correlation-test.png index 2c4b87c6bb..7f9868d1e1 100644 Binary files a/site/notebooks/tutorials/model_development/selecting-high-pearson-correlation-test.png and b/site/notebooks/tutorials/model_development/selecting-high-pearson-correlation-test.png differ diff --git a/site/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb b/site/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb index 2f85b39d77..71f08b5af6 100644 --- a/site/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb +++ b/site/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb @@ -9,7 +9,11 @@ "\n", "Learn how to use ValidMind for your end-to-end model validation process based on common scenarios with our series of four introductory notebooks. In this first notebook, set up the ValidMind Library in preparation for validating a champion model.\n", "\n", - "These notebooks use a binary classification model as an example, but the same principles shown here apply to other model types." + "These notebooks use a binary classification model as an example, but the same principles shown here apply to other model types.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for validators new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Validator Fundamentals
" ] }, { @@ -190,13 +194,19 @@ "\n", "In order to log tests as a validator instead of as a developer, on the model details page that appears after you've successfully registered your sample model:\n", "\n", - "1. Remove yourself as a developer: \n", + "1. Remove yourself as a model owner: \n", + "\n", + " - Click on the **OWNERS** tile.\n", + " - Click the **x** next to your name to remove yourself from that model's role.\n", + " - Click **Save** to apply your changes to that role.\n", + "\n", + "2. Remove yourself as a developer: \n", "\n", " - Click on the **DEVELOPERS** tile.\n", " - Click the **x** next to your name to remove yourself from that model's role.\n", " - Click **Save** to apply your changes to that role.\n", "\n", - "2. Add yourself as a validator: \n", + "3. Add yourself as a validator: \n", "\n", " - Click on the **VALIDATORS** tile.\n", " - Select your name from the drop-down menu.\n", diff --git a/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb b/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb index 5493f1f9c4..51bb9dc97b 100644 --- a/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb +++ b/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb @@ -15,7 +15,11 @@ "- Ensuring that data used for training and testing the model is of appropriate data quality\n", "- Ensuring that the raw data has been preprocessed appropriately and that the resulting final datasets reflects this\n", "\n", - "**For a full list of out-of-the-box tests,** refer to our [Test descriptions](https://docs.validmind.ai/developer/model-testing/test-descriptions.html) or try the interactive [Test sandbox](https://docs.validmind.ai/developer/model-testing/test-sandbox.html)." + "**For a full list of out-of-the-box tests,** refer to our [Test descriptions](https://docs.validmind.ai/developer/model-testing/test-descriptions.html) or try the interactive [Test sandbox](https://docs.validmind.ai/developer/model-testing/test-sandbox.html).\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for validators new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Validator Fundamentals
" ] }, { @@ -36,7 +40,7 @@ " - [Remove highly correlated features](#toc4_4_) \n", "- [Documenting test results](#toc5_) \n", " - [Configure and run comparison tests](#toc5_1_) \n", - " - [Log tests with a unique identifiers](#toc5_2_) \n", + " - [Log tests with unique identifiers](#toc5_2_) \n", " - [Add test results to reporting](#toc5_3_) \n", "- [Split the preprocessed dataset](#toc6_) \n", " - [Initialize the split datasets](#toc6_1_) \n", @@ -650,7 +654,7 @@ "source": [ "\n", "\n", - "### Log tests with a unique identifiers\n", + "### Log tests with unique identifiers\n", "\n", "Next, we'll use the previously initialized `vm_balanced_raw_dataset` (that still has a highly correlated `Age` column) as input to run an individual test, then log the result to the ValidMind Platform.\n", "\n", diff --git a/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb b/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb index b0d2260128..ba6958f286 100644 --- a/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb +++ b/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb @@ -8,7 +8,11 @@ "\n", "Learn how to use ValidMind for your end-to-end model validation process with our series of four introductory notebooks. In this third notebook, develop a potential challenger model and then pass your model and its predictions to ValidMind.\n", "\n", - "A *challenger model* is an alternate model that attempt to outperform the champion model, ensuring that the best performing fit-for-purpose model is always considered for deployment. Challenger models also help avoid over-reliance on a single model, and allow testing of new features, algorithms, or data sources without disrupting the production lifecycle." + "A *challenger model* is an alternate model that attempts to outperform the champion model, ensuring that the best performing fit-for-purpose model is always considered for deployment. Challenger models also help avoid over-reliance on a single model, and allow testing of new features, algorithms, or data sources without disrupting the production lifecycle.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for validators new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Validator Fundamentals
" ] }, { @@ -29,7 +33,7 @@ "- [Initializing the model objects](#toc5_) \n", " - [Initialize the model objects](#toc5_1_) \n", " - [Assign predictions](#toc5_2_) \n", - "- [Running model validation tests](#toc6_) \n", + "- [Running model evaluation tests](#toc6_) \n", " - [Run model performance tests](#toc6_1_) \n", " - [Evaluate performance of the champion model](#toc6_1_1_) \n", " - [Log a model finding](#toc6_1_2_) \n", @@ -514,7 +518,7 @@ "source": [ "\n", "\n", - "## Running model validation tests\n", + "## Running model evaluation tests\n", "\n", "With everything ready for us, let's run the rest of our validation tests. We'll focus on comprehensive testing around model performance of both the champion and challenger models going forward as we've already verified the data quality of the datasets used to train the champion model." ] diff --git a/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb b/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb index 1e5561c514..886f9e9061 100644 --- a/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb +++ b/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb @@ -13,7 +13,11 @@ "- The function can be as simple or as complex as you need it to be — it can use external libraries, make API calls, or do anything else that you can do in Python.\n", "- The only requirement is that the function signature and return values can be \"understood\" and handled by the ValidMind Library. As such, custom tests offer added flexibility by extending the default tests provided by ValidMind, enabling you to document any type of model or use case.\n", "\n", - "**For a more in-depth introduction to custom tests,** refer to our [Implement custom tests](../../code_samples/custom_tests/implement_custom_tests.ipynb) notebook." + "**For a more in-depth introduction to custom tests,** refer to our [Implement custom tests](../../code_samples/custom_tests/implement_custom_tests.ipynb) notebook.\n", + "\n", + "
Learn by doing\n", + "

\n", + "Our course tailor-made for validators new to ValidMind combines this series of notebooks with more a more in-depth introduction to the ValidMind Platform — Validator Fundamentals
" ] }, { diff --git a/site/notebooks/tutorials/model_validation/link-finding.png b/site/notebooks/tutorials/model_validation/link-finding.png index dd1c8a2662..f1855aab23 100644 Binary files a/site/notebooks/tutorials/model_validation/link-finding.png and b/site/notebooks/tutorials/model_validation/link-finding.png differ diff --git a/site/notebooks/tutorials/model_validation/link-validator-evidence.png b/site/notebooks/tutorials/model_validation/link-validator-evidence.png index 7403cad676..660a19cfab 100644 Binary files a/site/notebooks/tutorials/model_validation/link-validator-evidence.png and b/site/notebooks/tutorials/model_validation/link-validator-evidence.png differ diff --git a/site/notebooks/tutorials/model_validation/link-validator-evidence_OLD.png b/site/notebooks/tutorials/model_validation/link-validator-evidence_OLD.png deleted file mode 100644 index 58823e8f21..0000000000 Binary files a/site/notebooks/tutorials/model_validation/link-validator-evidence_OLD.png and /dev/null differ diff --git a/site/releases/2024/2024-may-22/release-notes.qmd b/site/releases/2024/2024-may-22/release-notes.qmd index bf4cb3ab6a..7375c7034d 100644 --- a/site/releases/2024/2024-may-22/release-notes.qmd +++ b/site/releases/2024/2024-may-22/release-notes.qmd @@ -43,7 +43,7 @@ You can use this notebook to learn how the end-to-end documentation process work ::: {.w-40-ns} -[Try notebook on JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/intro_for_model_developers.ipynb){.button .button-green} +[Try notebook on JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/101-set_up_validmind.ipynb){.button .button-green} ::: diff --git a/site/training/developer-fundamentals/finalizing-model-documentation.qmd b/site/training/developer-fundamentals/finalizing-model-documentation.qmd index 6254b82dd3..58149929bd 100644 --- a/site/training/developer-fundamentals/finalizing-model-documentation.qmd +++ b/site/training/developer-fundamentals/finalizing-model-documentation.qmd @@ -45,6 +45,7 @@ This final module is part of a four-part series: 3. [Submit documentation for approval](#submit-documentation-for-approval) 4. [Collaborate with others](#collaborate-with-others) 5. [Track model activity](#track-model-activity) +6. [View analytics](#view-analytics) ::: @@ -156,12 +157,12 @@ From the {{< var validmind.platform >}}: ## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} -:::: {.absolute bottom=0 left=50 right=50 .w-95 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} **Review model documentation** -::: {.f5} +::: {.f6} 1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Documentation**. +2. On the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**. 3. Click into any section of the documentation to review the test results logged via the {{< var validmind.developer >}}.
For example: **2.3 Correlations and Interactions** / **3.2 Model Evaluation** @@ -208,11 +209,11 @@ You can also use the content editing toolbar to revise the description of test r For example: -1. Locate the Data Preparation section and click on **2.3 Correlations and Interactions** to expand that section. +1. Within your model documentation, locate the Data Preparation section and click on **2.3 Correlations and Interactions** to expand that section. 2. Click on the text to edit the description for our individually inserted `HighPearsonCorrelation:balanced_raw_dataset` test: - ![Screenshot showing description added to the new content block](/notebooks/images/high-pearson-correlation-block.png){fig-alt="Screenshot showing description added to the new content block" .screenshot} + ![Editor for a test result description](/notebooks/images/high-pearson-correlation-block.png){fig-alt="Screenshot showing the editor for a test result description" .screenshot} ::: @@ -221,13 +222,11 @@ For example: ## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} -:::: {.absolute bottom=0 left=50 right=50 .w-95 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} **Add & edit content blocks** -::: {.f5} -1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Documentation**. -3. Click into any section of the documentation to add or edit a content block. +::: {.f6} +From your model's **{{< fa book-open >}} Documentation**, click into any section of the documentation to add and edit a content block. ::: @@ -279,10 +278,10 @@ To transition through the approval workflow, all required workflow steps must be ## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} -:::: {.absolute bottom=0 left=50 right=50 .w-95 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} **Submit for approval** -::: {.f5} +::: {.f6} 1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page. 2. Locate the **[model status]{.smallcaps}** section. 3. Open up the status transition panel, enter your **[notes]{.smallcaps}** and any other additional inventory fields, then click **Submit**. @@ -326,13 +325,11 @@ Comment threads ## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} -:::: {.absolute bottom=0 left=50 right=50 .w-95 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} **Comment on model documentation** -::: {.f5} -1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page. -2. In the left sidebar that appears for your model, click **Documentation**. -3. **In the content block you added earlier:** Post a comment, reply to it, and then resolve the thread. +::: {.f6} +**In the content block you added earlier** within your model's **{{< fa book-open >}} Documentation**: Post a comment, reply to it, and then resolve the thread. ::: @@ -369,12 +366,12 @@ Track changes & updates ## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} -:::: {.absolute bottom=0 left=50 right=50 .w-95 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} **Review model activity** -::: {.f5} +::: {.f6} 1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page. -2. In the left sidebar that appears for your model, click **Model Activity**. +2. In the left sidebar that appears for your model, click **{{< fa wifi >}} Model Activity**. 3. Filter the following activity: **Comments** | **Status Updates** | **Model Updates** | **Test Results** ::: @@ -383,6 +380,24 @@ When you're done, click [{{< fa chevron-right >}}]() to continue. :::: +# View analytics {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {background-iframe="https://app.prod.validmind.ai/analytics" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Welcome to {{< fa square-poll-vertical >}} Analytics** + +::: {.f6} +Here, you can find executive summaries, track information on models, findings, and more. For example: + +1. Click **Models** to review reports on models. +3. Click into any widget to review the models reported by that widget. +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + # In summary {background-color="#083E44" background-image="/training/assets/home-hero.svg"} @@ -408,10 +423,12 @@ In this final module, you learned how to: - [x] Submit your model documentation for approval - [x] Collaborate with other stakeholders on your model - [x] Track updates on your model +- [x] Navigate the analytics dashboard + ::: :::: -::: {.f2} +::: {.f3}
There is more that {{< var vm.product >}} can do to help you create model documentation, from using your own template to code samples you can adapt for your own use case: ::: diff --git a/site/training/developer-fundamentals/implementing-custom-tests.qmd b/site/training/developer-fundamentals/implementing-custom-tests.qmd index bdee729554..e575d8d37d 100644 --- a/site/training/developer-fundamentals/implementing-custom-tests.qmd +++ b/site/training/developer-fundamentals/implementing-custom-tests.qmd @@ -87,13 +87,13 @@ To continue, you need to have been [onboarded](developer-fundamentals-register.q :::: {.columns} ::: {.column width="60%"} ::: {.tc} -[Using {{< var vm.product >}} for model development](using-validmind-for-model-development.html){.button target="_blank"} +[Using {{< var vm.product >}} for Model Development](using-validmind-for-model-development.html){.button target="_blank"} ::: ::: ::: {.column width="40%"} ::: {.tc} -[Learning to run tests](learning-to-run-tests.html){.button target="_blank"} +[Learning to Run Tests](learning-to-run-tests.html){.button target="_blank"} ::: ::: @@ -112,7 +112,7 @@ Already logged in and refreshed this module? Click [{{< fa chevron-right >}}]() ::: {.w-50-ns .tc} -[Log in to JupyterHub](https://jupyterhub.validmind.ai/){.button target="_blank"} +[Log in to JupyterHub]({{< var url.jupyterhub >}}/){.button target="_blank"} ::: @@ -147,7 +147,7 @@ Be sure to return to this page afterwards. Jupyter Notebook series ::: {.f3} -When you run these notebooks, they will generate a draft of model documentation and upload it to {{< var vm.product >}}, complete with test supporting test results. +When you run these notebooks, they will generate a draft of model documentation and upload it to {{< var vm.product >}}, complete with supporting test results. ::: {.f5 .nt2 .pl2 .mb4}
@@ -183,10 +183,10 @@ Let's continue our journey with **Section 1** on the next page. {{< fa hand-poin ## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html" background-interactive="yes" data-preload="yes"} -:::: {.absolute bottom=15 left=0 right=50 .w-100 .f4 .tc .pl4 .pr4 .overlay} +:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay} **3 — Integrate custom tests** -::: {.f5} +::: {.f6} This is the third notebook in our introductory series, which will walk you through how to implement different types of custom tests with {{< var vm.product >}}. ::: @@ -243,13 +243,13 @@ Make sure you're logged in and have refreshed the page in a Chromium-based web b ## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#initialize-the-validmind-library" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Connect to your model** -::: {.f5} +::: {.f6} With your code snippet copied to your clipboard: -1. Open **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} +1. Open **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} 2. Run all the cells under the **Setting up** section. ::: @@ -271,7 +271,7 @@ Custom inline tests ::: ::: {.column width="70%" .bl .pl4 .f3} -Let's implement a custom *inline test* that calculates the confusion matrix for a binary classification model. +Let's implement a custom *inline test* that calculates the confusion matrix for a binary classification model: - An inline test refers to a test written and executed within the same environment as the code being tested — in the following example, right in our Jupyter Notebook — without requiring a separate test file or framework. - You'll note that the custom test function is just a regular Python function that can include and require any Python library as you see fit. @@ -281,13 +281,13 @@ Let's implement a custom *inline test* that calculates the confusion matrix for ## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#create-a-confusion-matrix-plot" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Create a custom test** -::: {.f5} +::: {.f6} The [`@vm.test` wrapper](/validmind/validmind.qmd#test){target="_blank"} allows you to create a reusable test: -1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} +1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} 2. Run all the cells in the following section under Implementing a custom inline test: **Create a confusion matrix plot** ::: @@ -298,14 +298,14 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#add-parameters-to-custom-tests" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Adjust your custom test** -::: {.f5} +::: {.f6} Custom tests can take parameters just like any other function: -1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} -2. Run all the cells in the following sections under Implementing a custom inline test:
**Add parameters to custom tests** / **Pass parameters to custom tests** +1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} +2. Run all the cells in the following sections under Implementing a custom inline test: **Add parameters to custom tests** / **Pass parameters to custom tests** ::: @@ -315,13 +315,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#log-the-confusion-matrix-results" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Log your custom test** -::: {.f5} +::: {.f6} Use the [`.log()` method](/validmind/validmind/vm_models.qmd#log){target="_blank"} to send the results of your custom test to the {{< var validmind.platform >}}: -1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} +1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} 2. Run the cell in the following section under Implementing a custom inline test: **Log the confusion matrix results** ::: @@ -356,13 +356,13 @@ Creating inline custom tests with a function is a great way to customize your mo ## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#create-custom-tests-folder" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Create custom tests folder** -::: {.f5} +::: {.f6} Create a new folder that will contain reusable custom tests from your existing inline tests: -1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} +1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} 2. Run the cell under the following Using external test providers section: **Create custom tests folder** ::: @@ -373,13 +373,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#save-an-inline-test" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Save inline test** -::: {.f5} +::: {.f6} The `@vm.test` decorator also includes a convenience method that allows you to save the test to a Python file at a specified path: -1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} +1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} 2. Run all the cells under the Using external test providers section: **Save an inline test** ::: @@ -390,13 +390,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#register-a-local-test-provider" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Register local test provider** -::: {.f5} +::: {.f6} Next, let's initialize a test provider that will tell the {{< var validmind.developer >}} where to find your saved custom tests: -1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} +1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"} 2. Run all the cells under the Using external test providers section: **Register a local test provider** ::: @@ -435,14 +435,14 @@ With the custom test results logged, let's head to the model we connected to at 3. Locate the Data Preparation section and click on **3.2 Model Evaluation** to expand that section. -4. Hover under the Pearson Correlation Matrix content block until a horizontal dashed line with a **+** button appears, indicating that you can insert a new block. +4. Hover under the Pearson Correlation Matrix content block until a horizontal dashed line with a **{{< fa plus >}}** button appears, indicating that you can insert a new block. -5. Click **+** and then select **Test-Driven Block** under [from library]{.smallcaps}: +5. Click **{{< fa plus >}}** and then select **Test-Driven Block** under [from library]{.smallcaps}: - Click on **Custom** under [test-driven]{.smallcaps} in the left sidebar. - Select the two custom `ConfusionMatrix` tests you logged above: - ![The ConfusionMatrix tests selected](/notebooks/EXECUTED/model_development/selecting-confusion-matrix-test.png){fig-alt="Screenshot showing the ConfusionMatrix tests selected" .screenshot} + ![The ConfusionMatrix tests selected](/notebooks/tutorials/model_development/selecting-confusion-matrix-test.png){fig-alt="Screenshot showing the ConfusionMatrix tests selected" .screenshot} 6. Finally, click **Insert 2 Test Results to Document** to add the test results to the documentation. @@ -472,10 +472,10 @@ When you're done, click [{{< fa chevron-right >}}]() to continue. ## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html" background-interactive="yes" data-preload="yes"} -:::: {.absolute bottom=15 left=0 right=50 .w-100 .f4 .tc .pl4 .pr4 .overlay} +:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay} **4 — Finalize testing and documentation** -::: {.f5} +::: {.f6} This is the final notebook in our introductory series, which will walk you through wrapping custom test results into your documentation, as well as how to update the configuration for the entire model documentation template to suit your needs. ::: @@ -524,13 +524,13 @@ When you're done, click [{{< fa chevron-right >}}]() to continue. ## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html#initialize-the-validmind-library" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Connect to your model** -::: {.f5} +::: {.f6} With your code snippet copied to your clipboard: -1. Open **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"} +1. Open **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"} 2. Run all the cells under the **Setting up** section. ::: @@ -543,13 +543,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html#reconnect-to-validmind" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Reconnect to {{< var vm.product >}}** -::: {.f5} +::: {.f6} After you insert test-driven blocks, changes should persist and become available every time you call the [`preview_template()` function](/validmind/validmind.qmd#preview_template){target="_blank"}: -1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"} +1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"} 2. Run all the cells under the **Reconnect to {{< var vm.product >}}** section. ::: @@ -560,13 +560,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html#include-custom-test-results" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Log custom test results** -::: {.f5} +::: {.f6} Now that your custom test IDs are part of your template, you can now run tests for an entire section and all additional custom tests will be loaded: -1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"} +1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"} 2. Run the cell under the **Include custom test results** section. ::: @@ -606,11 +606,11 @@ The utility function `vm.get_test_suite().get_default_config()` will return the ## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html#documentation-template-configuration" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Retrieve default configuration** -::: {.f5} -1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"} +::: {.f6} +1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"} 2. Run the first cell under the **Documentation template configuration** section. ::: @@ -621,13 +621,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html#update-the-config" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Update template configuration** -::: {.f5} +::: {.f6} The default configuration does not assign any inputs to a test, but you can assign inputs to individual tests as needed: -1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"} +1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"} 2. Run the all the cells under the following Documentation template configuration section: **Update the config** ::: @@ -663,5 +663,5 @@ In this third module, you learned how to: Continue your model development journey with:

-[Finalizing model documentation](finalizing-model-documentation.html){.button target="_blank"} +[Finalizing Model Documentation](finalizing-model-documentation.html){.button target="_blank"} ::: \ No newline at end of file diff --git a/site/training/developer-fundamentals/learning-to-run-tests.qmd b/site/training/developer-fundamentals/learning-to-run-tests.qmd index d34cc49fdf..dfc280afcc 100644 --- a/site/training/developer-fundamentals/learning-to-run-tests.qmd +++ b/site/training/developer-fundamentals/learning-to-run-tests.qmd @@ -63,7 +63,7 @@ To continue, you need to have been [onboarded](developer-fundamentals-register.q ::: {.tc} -[Using {{< var vm.product >}} for model development](using-validmind-for-model-development.html){.button target="_blank"} +[Using {{< var vm.product >}} for Model Development](using-validmind-for-model-development.html){.button target="_blank"} ::: :::: {.tc .mt5 .f2 .embed} @@ -79,7 +79,7 @@ Already logged in and refreshed this module? Click [{{< fa chevron-right >}}]() ::: {.w-50-ns .tc} -[Log in to JupyterHub](https://jupyterhub.validmind.ai/){.button target="_blank"} +[Log in to JupyterHub]({{< var url.jupyterhub >}}/){.button target="_blank"} ::: @@ -116,7 +116,7 @@ Be sure to return to this page afterwards. Jupyter Notebook series ::: {.f3} -When you run these notebooks, they will generate a draft of model documentation and upload it to {{< var vm.product >}}, complete with test supporting test results. +When you run these notebooks, they will generate a draft of model documentation and upload it to {{< var vm.product >}}, complete with supporting test results. ::: {.f5 .nt2 .pl2 .mb4}
@@ -148,10 +148,10 @@ Let's continue our journey with **2 — Start the model development process** on ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html" background-interactive="yes" data-preload="yes"} -:::: {.absolute bottom=15 left=0 right=50 .w-100 .f4 .tc .pl4 .pr4 .overlay} +:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay} **2 — Start the model development process** -::: {.f5} +::: {.f6} During this course, we'll run through these notebooks together, and at the end of your learning journey you'll have a fully documented sample model ready for review. ::: @@ -160,7 +160,6 @@ For now, **scroll through this notebook** to explore. When you are done, click [ :::: - # Explore ValidMind tests {background-color="#083E44" background-image="/training/assets/home-hero.svg"} @@ -168,10 +167,10 @@ For now, **scroll through this notebook** to explore. When you are done, click [ ## {background-iframe="/developer/model-testing/test-descriptions.html" background-interactive="true" data-preload="yes"} ::: footer -:::: {.absolute bottom=0 left=50 right=50 .w-95 .f3 .tc .pl4 .overlay} +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f4 .tc .pl4 .overlay} **{{< var vm.product >}} test repository** -::: {.f4} +::: {.f5} {{< var vm.product >}} provides a wealth out-of-the-box of tests to help you ensure that your model is being built appropriately. In this module, you'll become familiar with the individual tests available in {{< var vm.product >}}, as well as how to run them and change parameters as necessary. @@ -231,14 +230,14 @@ Make sure you're logged in and have refreshed the page in a Chromium-based web b ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#initialize-the-validmind-library" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Connect to your model** -::: {.f5} +::: {.f6} With your code snippet copied to your clipboard: -1. Open **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} -2. Run the following cells in the Setting up section:
**Initialize the {{< var validmind.developer >}}** / **Import sample dataset**. +1. Open **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +2. Run the following cells in the Setting up section: **Initialize the {{< var validmind.developer >}}** / **Import sample dataset**. ::: @@ -246,16 +245,15 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to :::: - ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#identify-qualitative-tests" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Identify qualitative tests** -::: {.f5} +::: {.f6} Next, we'll use the [`list_tests()` function](/notebooks/EXECUTED/model_development/1-set_up_validmind.ipynb#explore-available-tests){target="_blank"} to pinpoint tests we want to run: -1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} 2. Run all the cells under the Setting up section: **Identify qualitative tests** ::: @@ -263,16 +261,15 @@ Next, we'll use the [`list_tests()` function](/notebooks/EXECUTED/model_developm When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. :::: - ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#initialize-the-validmind-datasets" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Initialize {{< var vm.product >}} datasets** -::: {.f5} +::: {.f6} Then, we'll use the [`init_dataset()` function](/validmind/validmind.qmd#init_dataset){target="_blank"} to connect the sample data with a {{< var vm.product >}} `Dataset` object in preparation for running tests: -1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} 2. Run the following cell in the Setting up section: **Initialize the {{< var vm.product >}} datasets** ::: @@ -281,20 +278,19 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to :::: - # Run ValidMind tests {background-color="#083E44" background-image="/training/assets/home-hero.svg"} ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#run-tabular-data-tests" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Run tabular data tests** -::: {.f5} -You run individual tests by calling the [`run_test` function](/validmind/validmind/tests.qmd#run_test){target="_blank"} provided by the `validmind.tests` module: +::: {.f6} +You run individual tests by calling the [`run_test()` function](/validmind/validmind/tests.qmd#run_test){target="_blank"} provided by the `validmind.tests` module: -1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} 2. Run all the cells under the Running tests section: **Run tabular data tests**. ::: @@ -305,13 +301,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#utilize-test-output" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Utilize test output** -::: {.f5} +::: {.f6} You can utilize the output from a ValidMind test for further use, for example, if you want to remove highly correlated features: -1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} 2. Run all the cells under the Running tests section: **Utilize test output**. ::: @@ -320,7 +316,6 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to :::: - # Log ValidMind tests {background-color="#083E44" background-image="/training/assets/home-hero.svg"} @@ -350,13 +345,13 @@ Every test result returned by the `run_test()` function has a `.log()` method th ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#run-and-log-multiple-tests" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Run & log multiple tests** -::: {.f5} +::: {.f6} The [`run_documentation_tests()` function](/validmind/validmind.qmd#run_documentation_tests){target="_blank"} allows you to run multiple tests at once and automatically log the results to your documentation: -1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} 2. Run the following cell in the Documenting results section: **Run and log multiple tests**. ::: @@ -366,13 +361,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#run-and-log-an-individual-test" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Run & log an individual test** -::: {.f5} +::: {.f6} Next, we'll run an individual test and log the result to the {{< var validmind.platform >}}: -1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} 2. Run the following cell in the Running tests section: **Run and log an individual test**. ::: @@ -402,13 +397,13 @@ With the test results logged, let's head to the model we connected to at the beg 1. From the **{{< fa cubes >}} Inventory** in the {{< var validmind.platform >}}, go to the model you connected to earlier. -2. In the left sidebar that appears for your model, click **Documentation**. +2. In the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**. 3. Locate the Data Preparation section and click on **2.3 Correlations and Interactions** to expand that section. -4. Hover under the Pearson Correlation Matrix content block until a horizontal dashed line with a **+** button appears, indicating that you can insert a new block. +4. Hover under the Pearson Correlation Matrix content block until a horizontal dashed line with a **{{< fa plus >}}** button appears, indicating that you can insert a new block. -5. Click **+** and then select **Test-Driven Block** under [from library]{.smallcaps}: +5. Click **{{< fa plus >}}** and then select **Test-Driven Block** under [from library]{.smallcaps}: - Click on **VM Library** under [test-driven]{.smallcaps} in the left sidebar. - In the search bar, type in `HighPearsonCorrelation`. @@ -428,7 +423,7 @@ With the test results logged, let's head to the model we connected to at the beg **Insert a test-driven block** ::: {.f6} -2.3 Correlations and Interactions — `HighPearsonCorrelation:balanced_raw_dataset` +Within your model's **{{< fa book-open >}} Documentation**: 2.3 Correlations and Interactions — `HighPearsonCorrelation:balanced_raw_dataset` ::: @@ -480,13 +475,13 @@ In this next example, we’ll focus on running the tests within the Model Develo ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#train-simple-logistic-regression-model" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Train your model** -::: {.f5} +::: {.f6} Using {{< var vm.product >}} tests, we'll train a simple logistic regression model on our dataset and evaluate its performance: -1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} 2. Run all the cells under the Model testing section: **Train simple logistic regression model**. ::: @@ -496,13 +491,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#initialize-model-evaluation-objects" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Initialize a model object** -::: {.f5} -Use the `init_dataset` and [`init_model` functions](/validmind/validmind.qmd#init_model){target="_blank"} to initialize these objects: +::: {.f6} +Use the `init_dataset()` and [`init_model()` functions](/validmind/validmind.qmd#init_model){target="_blank"} to initialize these objects: -1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} 2. Run the cell under the following Model testing section: **Initialize model evaluation objects**. ::: @@ -513,13 +508,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#assign-predictions" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Assign predictions** -::: {.f5} +::: {.f6} Use the [`assign_predictions()` method](/validmind/validmind/vm_models.qmd#assign_predictions){target="_blank"} from the `Dataset` object to link existing predictions to any number of models: -1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} 2. Run the cell under the following Model testing section: **Assign predictions**. ::: @@ -530,13 +525,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to ## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#run-the-model-evaluation-tests" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Run the model evaluation tests** -::: {.f5} +::: {.f6} Finally, we'll run only the tests within the Model Development section of the model documentation: -1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} +1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"} 2. Run the cell under the following Model testing section: **Run the model evaluation tests**. ::: @@ -574,5 +569,5 @@ In this second module, you learned how to: Continue your model development journey with:

-[Implementing custom tests](implementing-custom-tests.html){.button target="_blank"} +[Implementing Custom Tests](implementing-custom-tests.html){.button target="_blank"} ::: \ No newline at end of file diff --git a/site/training/developer-fundamentals/using-validmind-for-model-development.qmd b/site/training/developer-fundamentals/using-validmind-for-model-development.qmd index b86513fb82..8d6885e295 100644 --- a/site/training/developer-fundamentals/using-validmind-for-model-development.qmd +++ b/site/training/developer-fundamentals/using-validmind-for-model-development.qmd @@ -76,7 +76,7 @@ Already logged in and refreshed this module? Click [{{< fa chevron-right >}}]() ::: {.w-50-ns .tc} -[Log in to JupyterHub](https://jupyterhub.validmind.ai/){.button target="_blank"} +[Log in to JupyterHub]({{< var url.jupyterhub >}}/){.button target="_blank"} ::: @@ -113,7 +113,7 @@ Be sure to return to this page afterwards. Jupyter Notebook series ::: {.f3} -When you run these notebooks, they will generate a draft of model documentation and upload it to {{< var vm.product >}}, complete with test supporting test results. +When you run these notebooks, they will generate a draft of model documentation and upload it to {{< var vm.product >}}, complete with supporting test results. ::: ::: @@ -142,10 +142,10 @@ Let's start our journey with **1 — Set up the {{< var validmind.developer >}}* ## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html" background-interactive="yes" data-preload="yes"} -:::: {.absolute bottom=15 left=0 right=50 .w-100 .f4 .tc .pl4 .pr4 .overlay} +:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay} **1 — Set up the {{< var validmind.developer >}}** -::: {.f5} +::: {.f6} During this course, we'll run through these notebooks together, and at the end of your learning journey you'll have a fully documented sample model ready for review. ::: @@ -159,7 +159,7 @@ For now, **scroll through this notebook** to explore. When you are done, click [ ## {background-iframe="https://app.prod.validmind.ai" background-interactive="true" data-preload="yes"} -::: {.fr .f4 .nr5 .pa5 .overlay} +::: {.fr .f5 .mv4 .nr5 .pa5 .overlay} **Welcome to the {{< var validmind.platform >}}** From here, you can: @@ -169,7 +169,7 @@ From here, you can: - Collaborate with model validators
to get your model approved ... - ... and much more! -::: {.f5 .pl3 .pr3 .embed} +::: {.f6 .pl3 .pr3 .embed} **Can't load the {{< var validmind.platform >}}?** Make sure you're logged in and have
refreshed the page in a Chromium-based
web browser. @@ -183,22 +183,22 @@ When you're done navigating around, ## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} -:::: {.fr .f4 .mv5 .nr5 .pa4 .overlay} -**Welcome to the model inventory** +:::: {.fr .f5 .mv6 .nr5 .pa4 .overlay} +**Welcome to the {{< fa cubes >}} Inventory** Use the model inventory to track
comprehensive details for all your
-models throughout the model lifecycle. +models throughout the model lifecycle. The model inventory is customizable
and extensible, with a layout that
can be configured to suit your needs. -::: {.f5 .pl3 .pr3 .embed} +::: {.f6 .pl3 .pr3 .embed} To start the documentation process,
a model must already be registered
in the model inventory via the
-**+ Register Model** modal. +**{{< fa plus >}} Register Model** modal. ::: **Let's register a model together** on
@@ -208,20 +208,20 @@ the next page. {{< fa hand-point-right >}} ## {background-iframe="https://app.prod.validmind.ai/model-inventory/?register=open" background-interactive="true" data-preload="yes"} -:::: {.fr .f4 .mv5 .nr5 .pa4 .overlay} +:::: {.fr .f5 .mv5 .nr5 .pa4 .overlay} **Register a binary
classification model** 1. Select the option
for a new model: -::: {.f5 .nt2 .pl2} +::: {.f6 .nt2 .pl2} - **Documentation template** —
`Binary classification` - **Use case** —
`Attrition/Churn Management` You can fill in other options
according to your preference. ::: -2. Click **Register Model** to
+2. Click **{{< fa plus >}} Register Model** to
add the model to your
inventory. @@ -229,7 +229,6 @@ When you're done,
click [{{< fa chevron-right >}}]() to continue. :::: - # Install the ValidMind Library {background-color="#083E44" background-image="/training/assets/home-hero.svg"} @@ -282,13 +281,13 @@ Make sure you're logged in and have refreshed the page in a Chromium-based web b ## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html#install-the-validmind-library" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Install & initialize the {{< var validmind.developer >}}** -::: {.f5} +::: {.f6} With your code snippet copied to your clipboard: -1. Open **1 — Set up the {{< var validmind.developer >}}**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/1-set_up_validmind.ipynb){target="_blank"} +1. Open **1 — Set up the {{< var validmind.developer >}}**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/1-set_up_validmind.ipynb){target="_blank"} 2. Run all the cells in the sections under **Initializing the {{< var validmind.developer >}}**. ::: @@ -297,20 +296,19 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to :::: - # Preview model documentation {background-color="#083E44" background-image="/training/assets/home-hero.svg"} ## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html#preview-the-documentation-template" data-preload="yes"} -:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} **Preview the documentation template** -::: {.f5} +::: {.f6} You can preview your model's documentation template right from the {{< var validmind.developer >}}: -1. Continue with **1 — Set up the {{< var validmind.developer >}}**: [{{< fa square-arrow-up-right >}} JupyterHub](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/tutorials/model_development/1-set_up_validmind.ipynb){target="_blank"} +1. Continue with **1 — Set up the {{< var validmind.developer >}}**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/1-set_up_validmind.ipynb){target="_blank"} 2. Run all the cells in the sections under **Getting to know ValidMind**. ::: @@ -319,17 +317,42 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to :::: +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +**Document Overview** + +::: {.tc} +[Learn more...](/guide/model-validation/review-model-documentation.qmd){.button target="_blank"} +::: +
+ +Try it **live** on the next page. {{< fa hand-point-right >}} + +::: + +::: {.column width="70%" .bl .pl4 .f3} + +{{< include /guide/model-documentation/_model-documentation-overview.qmd >}} + +{{< include /guide/model-documentation/_review-model-documentation.qmd >}} + + +::: +:::: + ## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} -:::: {.absolute bottom=0 left=50 right=50 .w-95 .f4 .tc .pl4 .overlay} +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} **Verify the documentation template** -::: {.f5} +::: {.f6} Once you've called `preview_template()`: 1. From the model inventory, select the name of your model to open up the model details page. -2. On the left sidebar that appears for your model, click **Documentation**. -3. Note how the structure of the model documentation reflects the previewed template. +2. On the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**. +3. Note how the structure of the Document Overview reflects the previewed template. ::: @@ -363,5 +386,5 @@ In this first module, you learned how to: Continue your model development journey with:

-[Learning to run tests](learning-to-run-tests.html){.button target="_blank"} +[Learning to Run Tests](learning-to-run-tests.html){.button target="_blank"} ::: \ No newline at end of file diff --git a/site/training/training-templates/course-slides.qmd b/site/training/training-templates/course-slides.qmd index 5e0862f273..8635000fb0 100644 --- a/site/training/training-templates/course-slides.qmd +++ b/site/training/training-templates/course-slides.qmd @@ -107,7 +107,7 @@ Already logged in and refreshed this module? Click [{{< fa chevron-right >}}]() ::: {.w-50-ns .tc} -[Log in to JupyterHub](https://jupyterhub.validmind.ai/){.button target="_blank"} +[Log in to JupyterHub]({{< var url.jupyterhub >}}/){.button target="_blank"} ::: @@ -138,14 +138,14 @@ Be sure to return to this page afterwards. ## {background-iframe="https://app.prod.validmind.ai/" data-preload="yes"} -:::: {.fr .f3 .mv5 .nr4 .pa5 .overlay} +:::: {.fr .f5 .mv5 .nr4 .pa5 .overlay} This is a right-aligned modal `.overlay`
on top of the live {{< var vm.product >}} Platform. - It mimics the style of the docs site callouts. - **Bold text shows up in a different colour.** - You can use an additional callout style
with this class: `.embed` -::: {.f4 .pl3 .pr3 .embed} +::: {.f6 .pl3 .pr3 .embed} **This is the nested embed**,
which also has its own bold styling. ::: @@ -155,12 +155,12 @@ As you can see, you need to get creative
with some of your `
`s to determ # iFrame embed bottom {background-color="#083E44" background-image="/training/assets/home-hero.svg"} -## {background-iframe="/notebooks/tutorials/intro_for_model_developers_EXECUTED.html" background-interactive="yes" data-preload="yes"} +## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html" background-interactive="yes" data-preload="yes"} -:::: {.absolute bottom=15 left=0 right=50 .w-100 .f3 .tc .pl4 .pr4 .overlay} +:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay} **This is a bottom-aligned modal `.overlay` on top of a rendered site file.** -::: {.f4} +::: {.f6} You can fully interact with the page contents. ::: :::: @@ -204,7 +204,7 @@ This is a simple two-column user guide for one set of instructions.
Try it **live** on the next page. {{< fa hand-point-right >}} ::: -::: {.column width="70%" .bl .pl4 .f3} +::: {.column width="70%" .bl .pl4 .f4} ### Single `h3` header Here you would embed some single-source instructions using Quarto's includes functionality, if they were available. diff --git a/site/training/validator-fundamentals/class-imbalance-assessment.png b/site/training/validator-fundamentals/class-imbalance-assessment.png new file mode 100644 index 0000000000..589de3ecd0 Binary files /dev/null and b/site/training/validator-fundamentals/class-imbalance-assessment.png differ diff --git a/site/training/validator-fundamentals/developing-challenger-models.qmd b/site/training/validator-fundamentals/developing-challenger-models.qmd new file mode 100644 index 0000000000..c61eb49524 --- /dev/null +++ b/site/training/validator-fundamentals/developing-challenger-models.qmd @@ -0,0 +1,686 @@ +--- +title: "Developing
Challenger Models" +subtitle: "Validator Fundamentals — Module 3 of 4

_Click [{{< fa chevron-right >}}](#learning-objectives) to start_" +lightbox: true +format: + revealjs: + controls: true + controls-tutorial: true + help: true + controls-back-arrows: visible + transition: slide + theme: [default, ../assets/slides.scss] + slide-number: true + chalkboard: false + preview-links: auto + view-distance: 2 + logo: /validmind.png + footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)" + html: + # Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml + output-file: _developing-challenger-models.html + search: false +title-slide-attributes: + data-background-color: "#083E44" + data-background-image: "../assets/home-hero.svg" +skip_preview: true +--- + +# Learning objectives {.center} + +_"As a **validator** who has already run and logged data quality tests with {{< var vm.product >}}, I want to next run both out-of-the-box and custom model evaluation tests for the champion model and a potential challenger model, and use the results of my testing to log model findings."_ + +::: {.tc} +
+This third module is part of a four-part series: +

+[Validator Fundamentals](/training/validator-fundamentals/validator-fundamentals-register.qmd){.button target="_blank"} +::: + + +## Module 3 — Contents {.center} + +:::: {.columns .f3} +::: {.column width="50%" .mt4 .pr4} +### Introduction +- [{{< var vm.product >}} for model validation](#validmind-for-model-validation) +::: + +::: + +:::: {.columns .f3} + +::: {.column width="50%" .mt4 .pr4} +### Section 1 + +- [Train potential challenger model](#train-a-challenger-model) +- [Run model evaluation tests](#run-model-evaluation-tests) + +::: + +::: {.column width="50%" .mt4} +### Section 2 + +- [Implement a custom test](#implement-custom-tests) +- [Verify development test runs](#verify-test-runs) +::: + +:::: + + +
+First, let's make sure you can log in to {{< var vm.product >}}. + +{{< include /training/assets/_revealjs-navigation.qmd >}} + +## Before you begin {.center} + +::: {.panel-tabset} + +### Prerequisite course + +To continue, you need to have been [onboarded](validator-fundamentals-register.qmd#register){target="_blank"} onto {{< var validmind.training >}} with the [**{{< fa circle-check >}} Validator**]{.bubble} role and completed the first two modules of this course: + + + + +:::: {.columns} +::: {.column width="60%"} +::: {.tc} +[Using {{< var vm.product >}} for Model Validation](using-validmind-for-model-validation.html){.button target="_blank"} +::: + +::: +::: {.column width="40%"} +::: {.tc} +[Running Data Quality Tests](running-data-quality-tests.html){.button target="_blank"} +::: + +::: +:::: + +:::: {.tc .mt5 .f2 .embed} +Already logged in and refreshed this module? Click [{{< fa chevron-right >}}]() to continue. + +::: + +### Log in + +1. Log in to check your access: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .tc} + +[Log in to JupyterHub]({{< var url.jupyterhub >}}/){.button target="_blank"} + +::: + +::: {.w-50-ns .tc} +[Log in to {{< var vm.product >}}](https://app.prod.validmind.ai){.button target="_blank"} +::: + +:::: + + +::: {.tc .f3} +Be sure to return to this page afterwards. +::: + +2. After you successfully log in, refresh the page to connect this training module up to the {{< var validmind.platform >}}: + +::: {.tc} + + +::: + +::: + + + +# ValidMind for model validation {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Jupyter Notebook series + +::: {.f3} +These notebooks walk you through how to validate a model using {{< var vm.product >}}, complete with supporting test results attached as evidence to your validation report. + +::: {.f5 .nt2 .pl2 .mb4} +
+ +You will need to have already completed notebooks **1** and **2** during the first and second modules to proceed. + +::: + +::: +::: + +::: {.column width="70%" .bl .pl4 .f3} +### {{< var vm.product >}} for model validation + +Our series of four introductory notebooks for model validators include sample code and how-to information to get you started with {{< var vm.product >}}: + +1 — [Set up the {{< var validmind.developer >}} for validation](/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb){target="_blank"}
+2 — [Start the model validation process](/notebooks/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
+3 — [Developing a potential challenger model](/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"}
+4 — [Finalize testing and reporting](/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"}
+
+ +::: {.f4 .pl3 .pr3 .embed} +In this third module, we'll run through the remaining two notebooks **3** in Section 1 and **4** in Section 2 together. +::: + +::: +:::: + +Let's continue our journey with **Section 1** on the next page. {{< fa hand-point-right >}} + +# Section 1 {background-color="#083E44" background-image="/assets/img/about-us-esphere.svg"} + +## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html" background-interactive="yes" data-preload="yes"} + +:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay} +**3 — Developing a potential challenger model** + +::: {.f6} +This is the third notebook in our introductory series, which will walk you through how to evaluate your champion model against a potential challenger with {{< var vm.product >}}. + +::: + +**Scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue. + +:::: + +## Get your code snippet + +:::: {.columns} + +::: {.column width="80%"} + + + +
+ +
+ +::: + +::: {.column width="20%" .f4} + +::: {.f5} +{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your validation environment: + +1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page. +2. On the left sidebar that appears for your model, click **Getting Started**. +3. Locate the code snippet and click **Copy snippet to clipboard**. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +::: + +:::: + +:::: {.tc .f6 .embed} +**Can't load the {{< var validmind.platform >}}?** + +Make sure you're logged in and have refreshed the page in a Chromium-based web browser. + +::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#initialize-the-validmind-library" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Connect to your model** + +::: {.f6} +With your code snippet copied to your clipboard: + +1. Open **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"} +2. Run all the cells under the **Setting up** section. + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#import-the-champion-model" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Import the champion model** + +::: {.f6} +Next, let's import the champion model submitted by the model development team in the format of a `.pkl` file for evaluation: + +1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"} +2. Run the cell under the **Import the champion model** section. + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +# Train a challenger model {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="60%" .pr4 .f2} +Champion vs. challenger models + +::: + +::: {.column width="40%" .bl .pl4 .f3} +Try it **live** on the next pages. {{< fa hand-point-right >}} + +::: + +:::: + +::: {.f4} + +We're curious how an alternate model compares to our champion model, so let's train a challenger model as a basis for our testing: + +::: {.panel-tabset} + +### Champion logistic regression model + +- Our champion *logistic regression model* is a simpler, parametric model that assumes a linear relationship between the independent variables and the log-odds of the outcome. +- While logistic regression may not capture complex patterns as effectively, it offers a high degree of interpretability and is easier to explain to stakeholders. +- However, model risk is not calculated in isolation from a single factor, but rather in consideration with trade-offs in predictive performance, ease of interpretability, and overall alignment with business objectives. + +### Challenger random forest classification model + +- A *random forest classification model* is an ensemble machine learning algorithm that uses multiple decision trees to classify data. In ensemble learning, multiple models are combined to improve prediction accuracy and robustness. +- Random forest classification models generally have higher accuracy because they capture complex, non-linear relationships, but as a result they lack transparency in their predictions. + +::: + +::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#training-a-potential-challenger-model" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Random forest classification model** + +::: {.f6} +Let's train our potential challenger model: + +1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"} +2. Run the cell under the following Training a potential challenger model section: **Random forest classification model** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#initializing-the-model-objects" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Initialize the model objects** + +::: {.f6} +In addition to the initialized datasets, you'll also need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data for each of our two models using [the `vm.init_model()` method](/validmind/validmind.qmd#init_model){target="_blank"}: + +1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"} +2. Run all the cells under the section **Initializing the model objects**. + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +# Run model evaluation tests {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="50%" .pr4 .f2} +Model evaluation testing + +::: + +::: {.column width="50%" .bl .pl4 .f2} +Try it **live** on the next pages. {{< fa hand-point-right >}} + +::: + +:::: + +::: {.f4} + +With everything ready for us, let's run the rest of our validation tests. We'll focus on comprehensive testing around model performance of both the champion and challenger models going forward as we've already verified the data quality of the datasets used to train the champion model: + +::: {.panel-tabset} + +### 1. Model performance tests + +We'll start with some performance tests, beginning with independent testing of our champion logistic regression model, then moving on to our potential challenger model. + + +### 2. Model diagnostic tests + +Next, we want to inspect the robustness and stability testing comparison between our champion and challenger model. + +### 3. Feature importance tests + +Finally, we want to verify the relative influence of different input features on our models' predictions, as well as inspect the differences between our champion and challenger model to see if a certain model offers more understandable or logical importance scores for features. + +::: + +::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#run-model-performance-tests" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Run model performance tests** + +::: {.f6} +Use the [`list_tests()` function](/validmind/validmind/tests.qmd#list_tests){target="_blank"} to identify all the model performance tests for classification: + +1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"} +2. Run all the cells under the Running model evaluation tests section: **Run model performance tests** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {.scrollable} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Log a model finding + +::: {.f5 .nt2 .pl2 .mb4} +(Scroll down for the full instructions.) +::: + +::: {.tc} +[Learn more ...](/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#log-a-model-finding){.button target="_blank"} + +::: + +
Try it **live** on the next page. {{< fa hand-point-right >}} + +::: + +::: {.column width="70%" .bl .pl4 .f4} +As we can observe from the output in our notebook, our champion model doesn't pass the `MinimumAccuracy` test based on the default thresholds of the out-of-the-box test, so let's log a model finding in the {{< var validmind.platform >}}: + +#### Create a finding via a validation report + +1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier. + +2. In the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**. + +3. Locate the Data Preparation section and click on **2.2.2. Model Performance** to expand that section. + +4. Under the Model Performance Metrics section, locate Findings then click **Link Finding to Report**: + + ![Validation report with the link finding option highlighted](/notebooks/tutorials/model_validation/link-finding.png){width=90% fig-alt="Screenshot showing the validation report with the link finding option highlighted" .screenshot} + +5. Click **{{< fa plus >}} Create New Finding** to add a finding. + +6. Enter in the details for your finding, for example: + + - **[title]{.smallcaps}** — Champion Logistic Regression Model Fails Minimum Accuracy Threshold + - **[risk area]{.smallcaps}** — Model Performance + - **[documentation section]{.smallcaps}** — 3.2. Model Evaluation + - **[description]{.smallcaps}** — The logistic regression champion model was subjected to a Minimum Accuracy test to determine whether its predictive accuracy meets the predefined performance threshold of 0.7. The model achieved an accuracy score of 0.6136, which falls below the required minimum. As a result, the test produced a Fail outcome. + +7. Click **Save**. + +8. Select the finding you just added to link to your validation report. + +9. Click **Update Linked Findings** to insert your finding. + +10. Confirm that finding you inserted has been correctly inserted into section **2.2.2. Model Performance** of the report. + +11. Click on the finding to expand the finding, where you can adjust details such as severity, owner, due date, status, etc. as well as include proposed remediation plans or supporting documentation as attachments. + + +::: +:::: + + +## {background-iframe="https://app.prod.validmind.ai/model-inventory/" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Create a model finding** + +::: {.f6} +Within your **{{< fa shield >}} Validation Report**: 2.2.2. Model Performance — **Link Finding to Report** {{< fa chevron-right >}} **{{< fa plus >}} Create New Finding** {{< fa chevron-right >}} **Update Linked Findings** + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#run-diagnostic-tests" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Run diagnostic tests** + +::: {.f6} +This time, use `list_tests()` to identify all the model diagnosis tests for classification: + +1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"} +2. Run all the cells under the Running model evaluation tests section: **Run diagnostic tests** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#run-feature-importance-tests" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Run feature importance tests** + +::: {.f6} +Use `list_tests()` again to identify all the feature importance tests for classification: + +1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"} +2. Run all the cells under the Running model evaluation tests section: **Run feature importance tests** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +# Section 2 {background-color="#083E44" background-image="/assets/img/about-us-esphere.svg"} + +## {background-iframe="/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.html" background-interactive="yes" data-preload="yes"} + +:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay} +**4 — Finalize testing and reporting** + +::: {.f6} +This is the final notebook in our introductory series, which will walk you through how to supplement ValidMind tests with your own custom tests and include them as additional evidence in your validation report, and wrap up your validation testing. +::: + +**Scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue. + +:::: + +## Retrieve your code snippet + +:::: {.columns} + +::: {.column width="80%"} + + + +
+ +
+ +::: + +::: {.column width="20%"} + +::: {.f4} +As usual, let's connect back up to your model in the {{< var validmind.platform >}}: + +1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page. +2. On the left sidebar that appears for your model, click **Getting Started**. +3. Locate the code snippet and click **Copy snippet to clipboard**. + +::: + +::: + +:::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +## {background-iframe="/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.html#initialize-the-validmind-library" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Connect to your model** + +::: {.f6} +With your code snippet copied to your clipboard: + +1. Open **4 — Finalize testing and validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"} +2. Run all the cells under the **Setting up** section. + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +# Implement custom tests {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Custom tests + +
Try it **live** on the next pages. {{< fa hand-point-right >}} + +::: + +::: {.column width="70%" .bl .pl4 .f3} +Let's implement a *custom test* that calculates a confusion matrix: + +- You'll note that the custom test function is just a regular Python function that can include and require any Python library as you see fit. +- In a usual model validation situation, you would load a saved custom test provided by the model development team. In the following section, we'll have you implement the same custom test and make it available for reuse, to familiarize you with the processes. + +::: +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.html#implement-a-custom-inline-test" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Implement a custom inline test** + +::: {.f6} +An *inline test* refers to a test written and executed within the same environment as the code being tested —
in the following example, right in our Jupyter Notebook — without requiring a separate test file or framework: + +1. Continue with **4 — Finalize testing and validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"} +2. Run all the cells in the following sections under Implementing custom tests: **Implement a custom inline test** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.html#use-external-test-providers" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Use external test providers** + +::: {.f6} +Sometimes you may want to reuse the same set of custom tests across multiple models and share them with others in your organization,
like the model development team would have done with you in this example workflow featured in this series of notebooks: + +1. Continue with **4 — Finalize testing and validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"} +2. Run all the cells in the following sections under Implementing custom tests: **Use external test providers** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +# Verify test runs {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {background-iframe="/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.html#verify-test-runs" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Verify model development testing** + +::: {.f6} +Our final task is to verify that all the tests provided by the model development team were run and reported accurately: + +1. Continue with **4 — Finalize testing and validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"} +2. Run all the cells under the **Verify test runs** section. + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +# In summary {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Developing challenger models + +::: + +::: {.column width="70%" .bl .pl4 .f3} +In this third module, you learned how to: + +- [x] Initialize {{< var vm.product >}} `Model` objects +- [x] Log a model finding within your validation report +- [x] Implement, run, and log custom inline tests +- [x] Register external test providers for reusable custom tests +- [x] Verify testing done by model development using {{< var vm.product >}} + +::: +:::: + +::: {.tc} +
+Continue your model development journey with: +

+ +[Finalizing Validation Reports](finalizing-validation-reports.html){.button target="_blank"} +::: \ No newline at end of file diff --git a/site/training/validator-fundamentals/finalizing-validation-reports.qmd b/site/training/validator-fundamentals/finalizing-validation-reports.qmd new file mode 100644 index 0000000000..2d0afc88e4 --- /dev/null +++ b/site/training/validator-fundamentals/finalizing-validation-reports.qmd @@ -0,0 +1,612 @@ +--- +title: "Finalizing
Validation Reports" +subtitle: "Validator Fundamentals — Module 4 of 4

_Click [{{< fa chevron-right >}}](#learning-objectives) to start_" +lightbox: true +format: + revealjs: + controls: true + controls-tutorial: true + help: true + controls-back-arrows: visible + transition: slide + theme: [default, ../assets/slides.scss] + slide-number: true + chalkboard: false + preview-links: auto + view-distance: 2 + logo: /validmind.png + footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)" + html: + # Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml + output-file: _finalizing-validation-reports.html + search: false +title-slide-attributes: + data-background-color: "#083E44" + data-background-image: "../assets/home-hero.svg" +skip_preview: true +--- + +# Learning objectives {.center} + +_"As a **validator** who has logged validation tests with the {{< var validmind.developer >}} to the {{< var validmind.platform >}}, I want to refine my model's validation report, submit my validation report for approval, and track finding resolution and other updates to my model."_ + +::: {.tc} +
+This final module is part of a four-part series: +

+[Validator Fundamentals](/training/validator-fundamentals/validator-fundamentals-register.qmd){.button target="_blank"} +::: + + +## Module 4 — Contents {.center} + + +:::: {.columns .f3} + +::: {.column width="50%" .mt4 .pr4} +### Section 1 + +- [Refine validation report](#refine-validation-report) +- [Manage model findings](#manage-model-findings) +- [Submit validation report for approval](#submit-report-for-approval) + +::: + +::: {.column width="50%" .mt4} +### Section 2 + +- [Collaborate with others](#collaborate-with-others) +- [Track model activity](#track-model-activity) +- [View analytics](#view-analytics) + +::: + +:::: + +First, let's make sure you can log in to {{< var vm.product >}}. + +{{< include /training/assets/_revealjs-navigation.qmd >}} + +## Before you begin {.center} + +::: {.panel-tabset} + +### Prerequisite courses + +To continue, you need to have been [onboarded](validator-fundamentals-register.qmd#register){target="_blank"} onto {{< var validmind.training >}} with the [**{{< fa circle-check >}} Validator**]{.bubble} role and completed the first three modules of this course: + + + +:::: {.columns} +::: {.column width="30%"} +::: {.tc} +[Module 1](using-validmind-for-model-validation.html){.button target="_blank"} +::: + +::: +::: {.column width="30%"} +::: {.tc} +[Module 2](running-data-quality-tests.html){.button target="_blank"} +::: + +::: + +::: {.column width="30%"} +::: {.tc} +[Module 3](developing-challenger-models.html){.button target="_blank"} +::: + +::: +:::: + +:::: {.tc .mt5 .f2 .embed} +Already logged in and refreshed this module? Click [{{< fa chevron-right >}}]() to continue. + +::: + +### Log in + +1. Log in to check your access: + +::: {.tc} +[Log in to {{< var vm.product >}}](https://app.prod.validmind.ai){.button target="_blank"} +::: + +::: {.tc .f3} +Be sure to return to this page afterwards. +::: + +2. After you successfully log in, refresh the page to connect this training module up to the {{< var validmind.platform >}}: + +::: {.tc} + + +::: + + +::: + +# Section 1 {background-color="#083E44" background-image="/assets/img/about-us-esphere.svg"} + +# Refine validation report {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Make qualitative edits + +::: {.f5 .nt2 .pl2 .mb4} +(Scroll down for the full instructions.) +::: + +::: {.tc} +[Learn more ...](/guide/model-documentation/work-with-content-blocks.qmd){.button target="_blank"} + +::: + +
Try it **live** on the next page. {{< fa hand-point-right >}} +::: + +::: {.column width="70%" .bl .pl4 .f4} + +::: {.panel-tabset} + +### Add content blocks + +1. From the **{{< fa cubes >}} Inventory** in the {{< var validmind.platform >}}, go to the model you connected to earlier. + +2. In the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**. + +3. Click on **1. Executive Summary** to expand and add content to that section. + +4. Hover your mouse over the space where you want your new block to go until a horizontal dashed line with a {{< fa square-plus >}} sign appears that indicates you can insert a new block: + + ![Adding a content block in the UI](/guide/model-documentation/add-content-block.gif){width=90% fig-alt="A gif showing the process of adding a content block in the UI" .screenshot} + +After adding the block to your documentation, generate a content draft with AI using the [content editing toolbar](/guide/model-documentation/work-with-content-blocks.qmd#content-editing-toolbar){target="_blank"}: + +{{< include /guide/model-documentation/content_blocks/_generate-with-ai.qmd >}} + +### Edit test result descriptions +You can also use the content editing toolbar to revise the description of test results to explain the changes made to the raw data and the reasons behind them. + +For example: + +1. Within your validation report, click on **2.2.1. Data Quality** to expand that section and locate the linked Class Imbalance Assessment evidence. + +2. Click **See evidence details** to review the LLM-generated description that summarizes the test results, that confirm that our final preprocessed dataset actually passes our test. + + + +3. Edit the description for our individually inserted `ClassImbalance:raw_dataset_preprocessed` test: + + ![Editor for a test result description](/notebooks/tutorials/model_validation/class-imbalance-results-detail.png){fig-alt="Screenshot showing the editor for a test result description" .screenshot} + +::: + +::: +:::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Add & edit content blocks** + +::: {.f6} +Within your **{{< fa shield >}} Validation Report**: + +1. Click on **1. Executive Summary** to add and edit a content block. +2. Click on **2.2.1. Data Quality** to edit the description for the linked Class Imbalance Assessment test results. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {.scrollable} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Assess compliance + +::: {.f5 .nt2 .pl2 .mb4} +(Scroll down for the full instructions.) +::: + +::: {.tc} +[Learn more ...](/guide/model-validation/assess-compliance.qmd#provide-compliance-assessments){.button target="_blank"} + +::: + +
Try it **live** on the next page. {{< fa hand-point-right >}} +::: + +::: {.column width="70%" .bl .pl4 .f4} + +::: {.panel-tabset} + +### Add risk assessment notes + +1. From the **{{< fa cubes >}} Inventory** in the {{< var validmind.platform >}}, go to the model you connected to earlier. + +2. In the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**. + +3. Click on **2.2.1. Data Quality** to expand that section and locate the Class Imbalance Assessment sub-section. + +4. Click under **Risk Assessment Notes** to edit the content block using the content editing toolbar. + + For example, use **{{< fa diamond >}} [beta]{.smallcaps} (Generate Text with AI)** to create a draft summarizing the contents of the Class Imbalance Assessment sub-section. + + ![Sample compliance assessment](class-imbalance-assessment.png){width=90% fig-alt="A screenshot showing a sample compliance assessment" .screenshot} + +### Provide compliance assessments + +In the same Class Imbalance Assessment sub-section, above the Risk Assessment Notes, locate the Guideline with the **[assessment]{.smallcaps}** drop-down. + +{{< include /guide/model-validation/_assess-compliance-assess.qmd >}} + +::: + +::: +:::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Add your compliance assessments** + +::: {.f6} +Within your **{{< fa shield >}} Validation Report**, click on **2.2.1. Data Quality** to assess compliance for the Class Imbalance Assessment sub-section. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +::: + +# Manage model findings {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Add more findings + +::: {.f5 .nt2 .pl2 .mb4} +(Scroll down for the full instructions.) +::: + +::: {.tc} +[Learn more ...](/guide/model-validation/add-manage-model-findings.qmd){.button target="_blank"} + +::: + +
+ +Try it **live** on the next pages. {{< fa hand-point-right >}} + +::: + +::: {.column width="70%" .bl .pl4 .f4} + +Along with adding findings directly via validation reports, you can also add findings during your review of documentation: + +{{< include /guide/model-validation/_add-model-findings.qmd >}} + +::: {.panel-tabset} + +### Add finding via overview + +{{< include /guide/model-validation/_add-finding-via-overview.qmd >}} + +### Add finding via section + +{{< include /guide/model-validation/_add-finding-via-section.qmd >}} + +::: + +::: +:::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Add a finding via overview** + +::: {.f6} +Within the model's **{{< fa book-open >}} Documentation**, click **{{< fa plus >}} Add Finding** to add a finding from the overview. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Add a finding via section** + +::: {.f6} +Within the model's **{{< fa book-open >}} Documentation**, click on any section heading to expand and add a finding to that section via the **{{< var vm.product >}} Insights™** panel. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +::: + +## {.scrollable} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Track issue resolution + +::: {.f5 .nt2 .pl2 .mb4} +(Scroll down for the full instructions.) +::: + +::: {.tc} +[Learn more ...](/guide/model-validation/add-manage-model-findings.qmd#track-issue-resolution){.button target="_blank"} +::: + +
Try it **live** on the next pages. {{< fa hand-point-right >}} +::: + +::: {.column width="70%" .bl .pl4 .f4} +As you prepare your report, review open or past due findings, close resolved ones, or add a mitigation plan: + +::: {.panel-tabset} + +### Update a model finding + +{{< include /guide/model-validation/_track_issue_resolution.qmd >}} + +### View all model findings + +Along with model-specific findings, you can also view and filter a list of findings across all models within the {{< var validmind.platform >}}: + +{{< include /guide/model-validation/_view-all-findings.qmd >}} + +::: + +::: +:::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Update a model finding** + +::: {.f6} +1. From the **{{< fa cubes >}} Inventory**, select the name the model you registered to open up the model details page. +2. In the left sidebar that appears for your model, click **{{< fa triangle-exclamation >}} Model Findings**. +3. Select one of the findings you logged during this course, and make some changes to any of the fields. +4. When you are finished editing, set the finding **[status]{.smallcaps}** to `Closed`. +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +::: + +## {background-iframe="https://app.prod.validmind.ai/model-findings" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**View all {{< fa triangle-exclamation >}} Model Findings** + +::: {.f6} +Filter this list to include only findings you want to see, or toggle visibilty for column headers. +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +::: + +# Submit report for approval {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Transition model status + +::: {.f5 .nt2 .pl2 .mb4} +(Scroll down for the full instructions.) +::: + +::: {.tc} +[Learn more ...](/guide/model-documentation/submit-for-approval.qmd){.button target="_blank"} +::: + +
Try it **live** on the next page. {{< fa hand-point-right >}} +::: + +::: {.column width="70%" .bl .pl4 .f4} + +When you’re ready, verify the approval workflow, and then submit your model documentation for approval: + +::: {.panel-tabset} +### Verify workflow + +Workflow states and transitions are configured by an administrator in advance, but you should verify that the expected people are included in the approval process. + +{{< include /guide/model-workflows/_model-workflows-see.qmd >}} + +### Submit for approval + +To transition through the approval workflow, all required workflow steps must be completed. By default, a model must be in the
[In Documentation]{.bubble} state before you can submit it for validation. + +{{< include /guide/model-workflows/_model-workflows-transition.qmd >}} + +::: + +::: +:::: + + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Submit for approval** + +::: {.f6} +1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page. +2. Locate the **[model status]{.smallcaps}** section. +3. Open up the status transition panel, enter your **[notes]{.smallcaps}** and any other additional inventory fields, then click **Submit**. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + +# Section 2 {background-color="#083E44" background-image="/assets/img/about-us-esphere.svg"} + +# Collaborate with others {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Comment threads + +::: {.tc} +[Learn more ...](/guide/model-documentation/collaborate-with-others.qmd){.button target="_blank"} +::: + +
Try it **live** on the next page. {{< fa hand-point-right >}} +::: + +::: {.column width="70%" .bl .pl4 .f3} +::: {.f5 .nt2} +::: + +{{< include /guide/model-documentation/_collaborate-with-others-activity.qmd >}} + +Have a question about the model? Collaborate with your developer right in the model documentation: + +::: {.panel-tabset} + +{{< include /guide/model-documentation/_collaborate-with-others-comments.qmd >}} + +::: + +::: + +:::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Comment on a text block** + +::: {.f6} +**In the content block you added earlier** within your **{{< fa shield >}} Validation Report**: Post a comment, reply to it, and then resolve the thread. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + + +# Track model activity {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Track changes & updates + +::: {.tc} +[Learn more ...](/guide/model-inventory/view-model-activity.qmd){.button target="_blank"} +::: + +
Try it **live** on the next page. {{< fa hand-point-right >}} +::: + +::: {.column width="70%" .bl .pl4 .f4} +### View documentation activity + +{{< include /guide/model-inventory/_view-model-activity-overview.qmd >}} + +{{< include /guide/model-inventory/_view-model-activity-steps.qmd >}} + +::: + +:::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Review model activity** + +::: {.f6} +1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page. +2. In the left sidebar that appears for your model, click **{{< fa wifi >}} Model Activity**. +3. Filter the following activity: **Comments** | **Status Updates** | **Model Updates** | **Test Results** + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + +# View analytics {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {background-iframe="https://app.prod.validmind.ai/analytics" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Welcome to {{< fa square-poll-vertical >}} Analytics** + +::: {.f6} +Here, you can find executive summaries, track information on models, findings, and more. For example: + +1. Click **Findings** to review reports on findings. +3. Click into any widget to review the findings reported by that widget. +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + +# In summary {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Finalizing validation reports + + +::: {.f3} +
Want to learn more? Find your next learning resource on [{{< fa graduation-cap >}} {{< var validmind.training >}}](/training/training.qmd){target="_blank"}. + +::: + +::: + +::: {.column width="70%" .bl .pl4 .f4} +In this final module, you learned how to: + +- [x] Add or edit content blocks in your validation report +- [x] Assess the compliance of a model within your validation report +- [x] Manage model findings via multiple methods +- [x] Submit your validation report for approval +- [x] Collaborate with other stakeholders on your model +- [x] Track updates on your model +- [x] Navigate the analytics dashboard +::: +:::: + +::: {.f3} +
+There is more that {{< var vm.product >}} can do to help you validate models, from using your own template to code samples you can adapt for your own use case: +::: + +::: {.tc} +[All model validation guides](/guide/guides.qmd#model-validation){.button target="_blank"} + +::: \ No newline at end of file diff --git a/site/training/validator-fundamentals/running-data-quality-tests.qmd b/site/training/validator-fundamentals/running-data-quality-tests.qmd new file mode 100644 index 0000000000..9d66c3db25 --- /dev/null +++ b/site/training/validator-fundamentals/running-data-quality-tests.qmd @@ -0,0 +1,513 @@ +--- +title: "Running Data
Quality Tests" +subtitle: "Validator Fundamentals — Module 2 of 4

_Click [{{< fa chevron-right >}}](#learning-objectives) to start_" +lightbox: true +format: + revealjs: + controls: true + controls-tutorial: true + help: true + controls-back-arrows: visible + transition: slide + theme: [default, ../assets/slides.scss] + slide-number: true + chalkboard: false + preview-links: auto + view-distance: 2 + logo: /validmind.png + footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)" + html: + # Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml + output-file: _running-data-quality-tests.html + search: false +title-slide-attributes: + data-background-color: "#083E44" + data-background-image: "../assets/home-hero.svg" +skip_preview: true +--- + +# Learning objectives {.center} + +_"As a **validator** who has connected to a champion model via the {{< var validmind.developer >}}, I want to identify relevant tests to run from {{< var vm.product >}}'s test repository, run and log data quality tests, and insert the test results into my model's validation report."_ + +::: {.tc} +
+This second module is part of a four-part series: +

+[Validator Fundamentals](/training/validator-fundamentals/validator-fundamentals-register.qmd){.button target="_blank"} +::: + + +## Module 2 — Contents {.center} + +::: {.f2} +1. [{{< var vm.product >}} for model validation](#validmind-for-model-validation) +2. [Explore ValidMind tests](#explore-validmind-tests) +3. [Run tests with the {{< var validmind.developer >}}](#run-validmind-tests) +4. [Log tests to the {{< var validmind.platform >}}](#log-validmind-tests) +5. [Prepare datasets for model evaluation testing](#prepare-datasets-for-model-evaluation) + +::: + +First, let's make sure you can log in to {{< var vm.product >}}. + +{{< include /training/assets/_revealjs-navigation.qmd >}} + +## Before you begin {.center} + +::: {.panel-tabset} + +### Prerequisite course + +To continue, you need to have been [onboarded](validator-fundamentals-register.qmd#register){target="_blank"} onto {{< var validmind.training >}} with the [**{{< fa circle-check >}} Validator**]{.bubble} role and completed the first module of this course: + +::: {.tc} + +[Using {{< var vm.product >}} for Model Validation](using-validmind-for-model-validation.html){.button target="_blank"} +::: + +:::: {.tc .mt5 .f2 .embed} +Already logged in and refreshed this module? Click [{{< fa chevron-right >}}]() to continue. + +::: + +### Log in + +1. Log in to check your access: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .tc} + +[Log in to JupyterHub]({{< var url.jupyterhub >}}/){.button target="_blank"} + +::: + +::: {.w-50-ns .tc} +[Log in to {{< var vm.product >}}](https://app.prod.validmind.ai){.button target="_blank"} +::: + +:::: + + +::: {.tc .f3} +Be sure to return to this page afterwards. +::: + +2. After you successfully log in, refresh the page to connect this training module up to the {{< var validmind.platform >}}: + +::: {.tc} + + +::: + + +::: + + + + +# ValidMind for model validation {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Jupyter Notebook series + +::: {.f3} +These notebooks walk you through how to validate a model using {{< var vm.product >}}, complete with supporting test results attached as evidence to your validation report. + +::: {.f5 .nt2 .pl2 .mb4} +
+You will need to have already completed **1 — Set up the {{< var validmind.developer >}} for validation** during the first module to proceed. +::: + +::: +::: + +::: {.column width="70%" .bl .pl4 .f3} +### {{< var vm.product >}} for model validation + +Our series of four introductory notebooks for model validators include sample code and how-to information to get you started with {{< var vm.product >}}: + +1 — [Set up the {{< var validmind.developer >}} for validation](/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb){target="_blank"}
+2 — [Start the model validation process](/notebooks/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
+3 — [Developing a potential challenger model](/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"}
+4 — [Finalize testing and reporting](/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"}
+
+ +::: {.f4 .pl3 .pr3 .embed} +In this second module, we'll run through **2 — Start the model validation process** together. +::: + +::: +:::: + +Let's continue our journey with **2 — Start the model validation process** on the next page. {{< fa hand-point-right >}} + +## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html" background-interactive="yes" data-preload="yes"} + +:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay} +**2 — Start the model validation process** + +::: {.f6} +During this course, we'll run through these notebooks together, and at the end of your learning journey you'll have a fully supported sample validation report ready for review. + +::: + +For now, **scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue. + +:::: + + + + +# Explore ValidMind tests {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {background-iframe="/developer/model-testing/test-descriptions.html" background-interactive="true" data-preload="yes"} + +::: footer +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f4 .tc .pl4 .overlay} +**{{< var vm.product >}} test repository** + +::: {.f5} +{{< var vm.product >}} provides a wealth out-of-the-box of tests to help you ensure that your model is being built appropriately. + +In this module, you'll become familiar with the individual tests available in {{< var vm.product >}}, as well as how to run them and change parameters as necessary. + +::: + +For now, **scroll through these test descriptions** to explore. When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: +::: + +## Get your code snippet + +:::: {.columns} + +::: {.column width="80%"} + + + +
+ +
+ +::: + +::: {.column width="20%" .f4} + +::: {.f5} +{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your validation environment: + +1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page. +2. On the left sidebar that appears for your model, click **Getting Started**. +3. Locate the code snippet and click **Copy snippet to clipboard**. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +::: + +:::: + +:::: {.tc .f6 .embed} +**Can't load the {{< var validmind.platform >}}?** + +Make sure you're logged in and have refreshed the page in a Chromium-based web browser. + +::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#initialize-the-validmind-library" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Connect to your model** + +::: {.f6} +With your code snippet copied to your clipboard: + +1. Open **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"} +2. Run the cell under the **Setting up** section. + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#load-the-sample-dataset" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Load the sample dataset** + +::: {.f6} +After you've successfully initialized the {{< var validmind.developer >}}, let's import the sample dataset that was used to develop the dummy champion model: + +1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"} +2. Run the cell under the **Load the sample dataset** section. + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#identify-qualitative-tests" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Identify qualitative tests** + +::: {.f6} +Next, we'll use the [`list_tests()` function](/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.ipynb#explore-available-tests){target="_blank"} to pinpoint tests we want to run: + +1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"} +2. Run all the cells under the Verifying data quality adjustments section: **Identify qualitative tests** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#initialize-the-validmind-datasets" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Initialize {{< var vm.product >}} datasets** + +::: {.f6} +Then, we'll use the [`init_dataset()` function](/validmind/validmind.qmd#init_dataset){target="_blank"} to connect the sample data with a {{< var vm.product >}} `Dataset` object in preparation for running tests: + +1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"} +2. Run the following cell in the Verifying data quality adjustments section: **Initialize the {{< var vm.product >}} datasets** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + + + + +# Run ValidMind tests {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#run-data-quality-tests" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Run data quality tests** + +::: {.f6} +You run individual tests by calling the [`run_test()` function](/validmind/validmind/tests.qmd#run_test){target="_blank"} provided by the `validmind.tests` module: + +1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"} +2. Run all the cells under the Verifying data quality adjustments section: **Run data quality tests** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#remove-highly-correlated-features" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Remove highly correlated features** + +::: {.f6} +You can utilize the output from a ValidMind test for further use, for example, if you want to remove highly correlated features: + +1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"} +2. Run all the cells under the Verifying data quality adjustments section: **Remove highly correlated features** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + + + + +# Log ValidMind tests {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Document test results + +::: {.tc} +[Learn more ...](/validmind/validmind/vm_models.qmd#log){.button target="_blank"} + +::: + +
Try it **live** on the next page. {{< fa hand-point-right >}} +::: + +::: {.column width="70%" .bl .pl4 .f3} +Every test result returned by the `run_test()` function has a `.log()` method that can be used to send the test results to the {{< var validmind.platform >}}: + +- When logging individual test results to the platform, you'll need to manually add those results to the desired section of the validation report. +- To demonstrate how to add test results to your validation report, we'll log our data quality tests and insert the results via the {{< var validmind.platform >}}. + +::: +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#configure-and-run-comparison-tests" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Configure and run comparison tests** + +::: {.f6} +You can leverage the {{< var validmind.developer >}} to easily run comparison tests, between both datasets and models. Here, we compare the original raw dataset and the final preprocessed dataset, then log the results to the {{< var validmind.platform >}}: + +1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"} +2. Run all the cells under the Documenting test results section: **Configure and run comparison tests** +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#log-tests-with-unique-identifiers" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Log tests with unique identifiers** + +::: {.f6} +When running individual tests, you can use a custom `result_id` to tag the individual result with a unique identifier: + +1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"} +2. Run the cell under the following Documenting test results section: **Log tests with unique identifiers**. + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {.scrollable} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Add test results to reporting + +::: {.tc} +[Learn more ...](/notebooks/EXECUTED/model_validation/2-start_validation_process.ipynb#add-test-results-to-reporting){.button target="_blank"} + +::: + +
Try it **live** on the next page. {{< fa hand-point-right >}} +::: + +::: {.column width="70%" .bl .pl4 .f4} +### Link validator evidence + +With some test results logged, let's head to the model we connected to at the beginning of this notebook and insert our test results into the validation report as evidence. + +While the example below focuses on a specific test result, you can follow the same general procedure for your other results: + +::: {.panel-tabset} + +### 1. Link data quality test results +1. From the **{{< fa cubes >}} Inventory** in the {{< var validmind.platform >}}, go to the model you connected to earlier. + +2. In the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**. + +3. Locate the Data Preparation section and click on **2.2.1. Data Quality** to expand that section. + +4. Under the Class Imbalance Assessment section, locate Validator Evidence then click **Link Evidence to Report**. + +5. Select the Class Imbalance test results we logged: **ValidMind Data Validation Class Imbalance** + + ![The ClassImbalance tests selected](/notebooks/tutorials/model_validation/selecting-class-imbalance-results.png){fig-alt="Screenshot showing the ClassImbalance tests selected" .screenshot} + +6. Click **Update Linked Evidence** to add the test results to the validation report. + + Confirm that the results for the Class Imbalance test you inserted has been correctly inserted into section **2.2.1. Data Quality** of the report. + +### 2. Review Class Imbalance test results + +- Once linked as evidence to section **2.2.1. Data Quality** note that the ValidMind Data Validation Class Imbalance test results are flagged as **Requires Attention** — as they include comparative results from our initial raw dataset. +- Click **See evidence details** to review the LLM-generated description that summarizes the test results, that confirm that our final preprocessed dataset actually passes our test: + + ![ClassImbalance test generated description in the text editor](/notebooks/tutorials/model_validation/class-imbalance-results-detail.png){fig-alt="Screenshot showing the ClassImbalance test generated description in the text editor" .screenshot} + +::: + +::: +:::: + + +## {background-iframe="https://app.prod.validmind.ai/model-inventory/" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Link validator evidence** + +::: {.f6} +Within your **{{< fa shield >}} Validation Report**: 2.2.1. Data Quality — **ValidMind Data Validation Class Imbalance** {{< fa chevron-right >}} **Link Evidence to Report** + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + + +# Prepare datasets for model evaluation {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#split-the-preprocessed-dataset" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Split the preprocessed dataset** + +::: {.f6} +So far, we've rebalanced our raw dataset and used the results of {{< var vm.product >}} tests to additionally remove highly correlated features from our dataset. Next, let's now spilt our dataset into train and test in preparation for model evaluation testing: + +1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"} +2. Run all the cells under the **Split the preprocessed dataset** section. + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +# In summary {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Running data quality tests +::: + +::: {.column width="70%" .bl .pl4 .f3} +In this second module, you learned how to: + +- [x] Identify relevant tests to run from {{< var vm.product >}}'s test repository +- [x] Initialize {{< var vm.product >}} `Dataset` objects +- [x] Run out-of-the-box tests with the {{< var validmind.developer >}} +- [x] Log test results to the {{< var validmind.platform >}} +- [x] Attach logged test results onto your model's validation report + +::: +:::: + +::: {.tc} +
+Continue your model development journey with: +

+ +[Developing Challenger Models](developing-challenger-models.html){.button target="_blank"} +::: \ No newline at end of file diff --git a/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd b/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd new file mode 100644 index 0000000000..95aaa27c63 --- /dev/null +++ b/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd @@ -0,0 +1,447 @@ +--- +title: "Using {{< var vm.product >}}
for Model Validation" +subtitle: "Validator Fundamentals — Module 1 of 4

_Click [{{< fa chevron-right >}}](#learning-objectives) to start_" +lightbox: true +format: + revealjs: + controls: true + controls-tutorial: true + help: true + controls-back-arrows: visible + transition: slide + theme: [default, ../assets/slides.scss] + slide-number: true + chalkboard: false + preview-links: auto + view-distance: 2 + logo: /validmind.png + footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)" + html: + # Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml + output-file: _using-validmind-for-model-validation.html + search: false +title-slide-attributes: + data-background-color: "#083E44" + data-background-image: "../assets/home-hero.svg" +skip_preview: true +--- + +# Learning objectives {.center} + +_"As a **validator** who is new to {{< var vm.product >}}, I want to learn how to connect to a champion model in the {{< var validmind.platform >}} using the {{< var validmind.developer >}} installed in my local environment, preview the model's validation report template, and review submitted documentation for the model."_ + +::: {.tc} +
+This first module is part of a four-part series: +

+[Validator Fundamentals](/training/validator-fundamentals/validator-fundamentals-register.qmd){.button target="_blank"} +::: + + +## Module 1 — Contents {.center} + +::: {.f2} +1. [{{< var vm.product >}} for model validation](#validmind-for-model-validation) +2. [The {{< var validmind.platform >}}](#the-validmind-platform) +3. [Connect to your champion model](#connect-to-a-model) +4. [Preview your model's validation report template](#preview-validation-report) +5. [Review submitted model documentation](#review-model-documentation) + +::: + +First, let's make sure you can log in to {{< var vm.product >}}. + +{{< include /training/assets/_revealjs-navigation.qmd >}} + +## Before you begin {.center} + +::: {.panel-tabset} + +### Registration + +To continue, you need to have been [onboarded](validator-fundamentals-register.qmd#register){target="_blank"} onto {{< var validmind.training >}} with the [**{{< fa circle-check >}} Validator**]{.bubble} role. + +
+ +:::: {.tc .mt5 .f2 .embed} +Already logged in and refreshed this module? Click [{{< fa chevron-right >}}]() to continue. + +::: + + +### Log in + +1. Log in to check your access: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .tc} + +[Log in to JupyterHub]({{< var url.jupyterhub >}}/){.button target="_blank"} + +::: + +::: {.w-50-ns .tc} +[Log in to {{< var vm.product >}}](https://app.prod.validmind.ai){.button target="_blank"} +::: + +:::: + + +::: {.tc .f3} +Be sure to return to this page afterwards. +::: + +2. After you successfully log in, refresh the page to connect this training module up to the {{< var validmind.platform >}}: + +::: {.tc} + + +::: + +::: + +
+ + + +# ValidMind for model validation {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Jupyter Notebook series + +::: {.f3} +These notebooks walk you through how to validate a model using {{< var vm.product >}}, complete with supporting test results attached as evidence to your validation report. + +::: +::: + +::: {.column width="70%" .bl .pl4 .f3} +### {{< var vm.product >}} for model validation + +Our series of four introductory notebooks for model validators include sample code and how-to information to get you started with {{< var vm.product >}}: + +1 — [Set up the {{< var validmind.developer >}} for validation](/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb){target="_blank"}
+2 — [Start the model validation process](/notebooks/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
+3 — [Developing a potential challenger model](/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"}
+4 — [Finalize testing and reporting](/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"}
+
+ +::: {.f4 .pl3 .pr3 .embed} +In this first module, we'll run through **1 — Set up the {{< var validmind.developer >}} for validation** together. +::: + +::: +:::: + +
+ +Let's start our journey with **1 — Set up the {{< var validmind.developer >}} for validation** on the next page. {{< fa hand-point-right >}} + +## {background-iframe="/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.html" background-interactive="yes" data-preload="yes"} + +:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay} +**1 — Set up the {{< var validmind.developer >}} for validation** + +::: {.f6} +During this course, we'll run through these notebooks together, and at the end of your learning journey you'll have a fully supported sample validation report ready for review. +::: + +For now, **scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue. + +:::: + + + +# The ValidMind Platform {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {background-iframe="https://app.prod.validmind.ai" background-interactive="true" data-preload="yes"} + +::: {.fr .f5 .nr5 .pa5 .overlay} +**Welcome to the {{< var validmind.platform >}}** + +From here, you can: + +- Keep track of your models in the
customizable inventory ... +- Review model documentation
submitted by model developers ... +- Assess the compliance of models by
logging findings and test results as
evidence on validation reports ... +- View analytics on your models and findings,
and create custom reports ... +- ... and much more! + +::: {.f6 .pl3 .pr3 .embed} +**Can't load the {{< var validmind.platform >}}?** + +Make sure you're logged in and have refreshed
the page in a Chromium-based browser. +::: + +When you're done navigating around, +
click [{{< fa chevron-right >}}]() to continue. + +::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.fr .f5 .mv5 .nr5 .pa4 .overlay} +**Welcome to the {{< fa cubes >}} Inventory** + +Use the model inventory to track
+comprehensive details for all your
+models throughout the model lifecycle. + +The model inventory is customizable
+and extensible, with a layout that
+can be configured to suit your needs. + +::: {.f6 .pl3 .pr3 .embed} +In a usual model lifecycle,
+a champion model will have been
+independently registered in your
+model inventory and submitted to
+you for validation. + +For this training course,
+we'll have you register a dummy model
+via the **{{< fa plus >}} Register Model** modal
+and assign yourself as the validator
+to familiarize you with the
+{{< var validmind.platform >}} interface. +::: + +**Let's register a model together** for use
+with validation on the next page. {{< fa hand-point-right >}} + +:::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory/?register=open" background-interactive="true" data-preload="yes"} + +:::: {.fr .f5 .mv5 .nr5 .pa4 .overlay} +**Register a binary
classification model** + +1. Select the option
+for a new model: + +::: {.f6 .nt2 .pl2} +- **Documentation template** —
`Binary classification` +- **Use case** —
`Attrition/Churn Management` + + You can fill in other options
according to your preference. +::: + +2. Click **{{< fa plus >}} Register Model** to
+add the model to your
+inventory. + +When you're done,
click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay} +**Assign validator credentials** + +::: {.f6} +In order to log tests as a validator instead of as a developer, we'll need to adjust your **model stakeholder** permissions: + +1. From the **{{< fa cubes >}} Inventory**, select the name the model you registered to open up the model details page. +2. Remove yourself as a model owner by clicking on the **[owners]{.smallcaps}** tile, then clicking on the **x** next to your name. Click **Save** to apply your changes to that role. +3. Remove yourself as a developer by clicking on the **[developers]{.smallcaps}** tile, then clicking on the **x** next to your name. Click **Save** to apply your changes to that role. +4. Add yourself as a validator by clicking on the **[validators]{.smallcaps}** tile, and selecting your name from the drop-down menu. Click **Save** to apply your changes to that role. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + +# Connect to a model {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## Get your code snippet + +:::: {.columns} + +::: {.column width="80%"} + + + +
+ +
+ +::: + +::: {.column width="20%" .f4} + +::: {.f5} +{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your validation environment: + +1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page. +2. On the left sidebar that appears for your model, click **Getting Started**. +3. Locate the code snippet and click **Copy snippet to clipboard**. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +::: + +:::: + +:::: {.tc .f6 .embed} +**Can't load the {{< var validmind.platform >}}?** + +Make sure you're logged in and have refreshed the page in a Chromium-based web browser. + +::: + +## {background-iframe="/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.html#install-the-validmind-library" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Install & initialize the {{< var validmind.developer >}}** + +::: {.f6} +With your code snippet copied to your clipboard: + +1. Open **1 — Set up the {{< var validmind.developer >}} for validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb){target="_blank"} +2. Run all the cells under the Setting up sections: **Install the {{< var validmind.developer >}}** / **Initialize the {{< var validmind.developer >}}** + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +# Preview validation report {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {background-iframe="/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.html#preview-the-validation-report-template" data-preload="yes"} + +:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay} +**Preview the validation report template** + +::: {.f6} +You can preview your model's validation report template right from the {{< var validmind.developer >}}: + +1. Continue with **1 — Set up the {{< var validmind.developer >}} for validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb){target="_blank"} +2. Run all the cells in the sections under **Getting to know ValidMind**. + +::: + +When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue. + +:::: + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay} +**Verify the report template** + +::: {.f6} +Once you've called `preview_template()`: + +1. From the model inventory, select the name of your model to open up the model details page. +2. On the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**. +3. Note how the structure of the validation report reflects the previewed template. + +::: + +When you're done taking a look around, click [{{< fa chevron-right >}}]() to continue. + +:::: + +# Review model documentation {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +**Document Overview** + +::: {.tc} +[Learn more...](/guide/model-validation/review-model-documentation.qmd){.button target="_blank"} +::: +
+ +Try it **live** on the next page. {{< fa hand-point-right >}} + +::: + +::: {.column width="70%" .bl .pl4 .f3} + +{{< include /guide/model-documentation/_model-documentation-overview.qmd >}} + +{{< include /guide/model-documentation/_review-model-documentation.qmd >}} + + +::: +:::: + + + +## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"} + +:::: {.fr .f5 .mv6 .nr5 .pa4 .overlay} +**Explore sample model documentation** + +In a usual model lifecycle, a champion model
+will have been submitted to you for validation
+with completed model documentation. + +Here, review the empty sample model
+documentation for the model you registered
+earlier to familiarize you with what the structure
+of documentation could look like when
+presented to you for review. + +::: {.f6 .pl3 .pr3 .embed} +The structure of the model documentation reflects
+the documentation template applied to the model,
+just like the validation report template. + +::: + +When you're done taking a look
+around the model documentation,
+click [{{< fa chevron-right >}}]() to continue. + +:::: + + +# In summary {background-color="#083E44" background-image="/training/assets/home-hero.svg"} + +## {.scrollable .center} + +:::: {.columns} +::: {.column width="30%" .pr4 .f2} +Using {{< var vm.product >}} for model validation + +::: + +::: {.column width="70%" .bl .pl4 .f3} +In this first module, you learned how to: + +- [x] Install the {{< var validmind.developer >}} in your environment +- [x] Connect to a model in the {{< var validmind.platform >}} as a validator +- [x] Preview and verify your model's validation report template +- [x] View your champion model's documentation + +::: +:::: + +::: {.tc} +
+Continue your model development journey with: +

+ +[Running Data Quality Tests](running-data-quality-tests.html){.button target="_blank"} +::: \ No newline at end of file diff --git a/site/training/validator-fundamentals/validator-fundamentals-register.qmd b/site/training/validator-fundamentals/validator-fundamentals-register.qmd index 69e9119fbe..5553880d8d 100644 --- a/site/training/validator-fundamentals/validator-fundamentals-register.qmd +++ b/site/training/validator-fundamentals/validator-fundamentals-register.qmd @@ -2,8 +2,42 @@ title: "Validator Fundamentals" subtitle: For {{< var vm.product >}} date: last-modified -aliases: +aliases: - training-for-model-validators.html +listing: + - id: validator-modules + type: grid + grid-columns: 2 + max-description-length: 500 + sort: false + table-hover: true + image-align: left + contents: + - path: using-validmind-for-model-validation.html + title: "Using {{< var vm.product >}} for Model Validation" + subtitle: "Module 1" + description: "{{< fa check >}} Install the {{< var validmind.developer >}} in your environment
{{< fa check >}} Connect to a champion model as a validator
{{< fa check >}} Preview the model's validation report template
{{< fa check >}} Review submitted model documentation" + reading-time: "20" + author: "{{< var vm.product >}}" + - path: running-data-quality-tests.html + title: "Running Data Quality Tests" + subtitle: "Module 2" + description: "{{< fa check >}} Identify relevant tests to run from {{< var vm.product >}}'s test vault
{{< fa check >}} Initialize datasets for use with the {{< var validmind.developer >}}
{{< fa check >}} Run and log out-of-the-box tests on your datasets
{{< fa check >}} Insert test results into your model’s validation report" + reading-time: "45" + author: "{{< var vm.product >}}" + - path: developing-challenger-models.html + title: "Developing Challenger Models" + subtitle: "Module 3" + description: "{{< fa check >}} Initialize models for use with the {{< var validmind.developer >}}
{{< fa check >}} Run and log out-of-the box and custom tests for your models
{{< fa check >}} Use the results of tests to log model findings" + reading-time: "60" + author: "{{< var vm.product >}}" + - path: finalizing-validation-reports.html + title: "Finalizing Validation Reports" + subtitle: "Module 4" + description: "{{< fa check >}} Make qualitative edits to your validation report
{{< fa check >}} Submit your validation report for approval
{{< fa check >}} Track finding resolution and other updates to your model" + reading-time: "25" + author: "{{< var vm.product >}}" + fields: [title, subtitle, description, reading-time] --- Learn how to use {{< var vm.product >}} as a **validator** to generate validation reports, automate testing, and collaborate with your model development team. @@ -20,36 +54,23 @@ Learn how to use {{< var vm.product >}} as a **validator** to generate validatio :::: {.flex .flex-wrap .justify-around} ::: {.w-50-ns .pr3} -### Learn how to +#### Total duration -#### Part 1 +150 minutes -- Review model documentation -- Analyze test results -- Add findings, assess compliance and link evidence -- Prepare validation reports - -#### Part 2 - -- Track issue resolution -- Submit for review and approval - -#### Duration - -60 minutes - -[register](#register){.button-green .cta} ::: -::: {.w-50-ns .pa4} +::: {.w-50-ns .pa3 .tr} -::: {.preview source="validator-fundamentals.qmd"} -::: +[register now](#register){.button-green .cta} ::: :::: +:::{#validator-modules} +::: + ::: {{< include /training/_training-register.qmd >}}