diff --git a/.github/workflows/deploy-docs-prod.yaml b/.github/workflows/deploy-docs-prod.yaml
index 3e89648607..78461b0113 100644
--- a/.github/workflows/deploy-docs-prod.yaml
+++ b/.github/workflows/deploy-docs-prod.yaml
@@ -12,6 +12,10 @@ on:
jobs:
deploy:
+ if: |
+ github.event_name == 'push' ||
+ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') ||
+ github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
diff --git a/.github/workflows/deploy-docs-staging.yaml b/.github/workflows/deploy-docs-staging.yaml
index f95ad0061c..cf4fb5772e 100644
--- a/.github/workflows/deploy-docs-staging.yaml
+++ b/.github/workflows/deploy-docs-staging.yaml
@@ -12,6 +12,10 @@ on:
jobs:
deploy:
+ if: |
+ github.event_name == 'push' ||
+ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') ||
+ github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
diff --git a/.github/workflows/lighthouse-check.yaml b/.github/workflows/lighthouse-check.yaml
new file mode 100644
index 0000000000..3d8c9730ae
--- /dev/null
+++ b/.github/workflows/lighthouse-check.yaml
@@ -0,0 +1,536 @@
+name: Lighthouse check
+
+on:
+ pull_request:
+ types: [opened, synchronize, ready_for_review]
+
+permissions:
+ issues: write
+ pull-requests: write
+
+env:
+ # To change the default depth level:
+ # 0 — Top-level navigation only (e.g. /index.html, /guide/guides.html, /developer/validmind-library.html, etc.)
+ # 1 — All first-level subdirectories (e.g. /guide/*.html)
+ # 2 — All second-level subdirectories (e.g. /guide/attestation/*.html)
+ # Note: While the crawler technically supports deeper levels, expect the workflow to take >2-12 hours to complete
+ DEFAULT_DEPTH: '0'
+
+jobs:
+ lighthouse:
+ runs-on: ubuntu-latest
+ if: github.event.pull_request.draft == false
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v4
+
+ - name: Get commit SHA
+ id: get_sha
+ run: |
+ echo "COMMIT_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
+ echo "COMMIT_SHA_SHORT=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+ - name: Set PR preview URL
+ id: set_url
+ run: |
+ echo "PREVIEW_URL=https://docs-demo.vm.validmind.ai/pr_previews/${{ github.head_ref }}" >> $GITHUB_ENV
+ echo "DEPTH=${{ env.DEFAULT_DEPTH }}" >> $GITHUB_ENV
+
+ - name: Check for PR preview URL and sitemap
+ id: check_preview
+ run: |
+ # Function to check if URL exists
+ check_url() {
+ curl --head --silent --fail "$1" > /dev/null
+ return $?
+ }
+
+ # Wait for preview URL to be available (up to 60 minutes)
+ echo "Waiting for preview site to become available ..."
+ for i in {1..60}; do
+ if check_url "$PREVIEW_URL/index.html"; then
+ echo "Info: Preview site is now available"
+ break
+ fi
+
+ if [ $i -eq 60 ]; then
+ echo "Error: Preview URL did not become available after 60 minutes at $PREVIEW_URL/index.html"
+ exit 1
+ fi
+
+ echo "Attempt $i/60: Preview site not ready yet, waiting 1 minute..."
+ sleep 60
+ done
+
+ # Now check for sitemap
+ if ! check_url "$PREVIEW_URL/sitemap.xml"; then
+ echo "Error: Sitemap does not exist at $PREVIEW_URL/sitemap.xml"
+ exit 1
+ fi
+
+ # Check if installation page is accessible with auth
+ echo "Debug: Checking installation page with URL-based auth..."
+ auth_url="https://${{ secrets.INSTALLATION_USER }}:${{ secrets.INSTALLATION_PW }}@docs-demo.vm.validmind.ai/pr_previews/${{ github.head_ref }}/installation/index.html"
+ if ! curl -v --head --silent --fail --anyauth "$auth_url" 2>&1; then
+ echo "Error: Installation page is not accessible with authentication at $auth_url"
+ exit 1
+ fi
+ echo "Info: Successfully accessed password-protected installation page"
+
+ echo "preview_exists=true" >> $GITHUB_OUTPUT
+
+ - name: Install Lighthouse CI
+ if: steps.check_preview.outputs.preview_exists == 'true'
+ run: npm install -g @lhci/cli
+
+ - name: Install required Python packages
+ if: steps.check_preview.outputs.preview_exists == 'true'
+ run: |
+ python -m pip install --upgrade pip
+ pip install requests beautifulsoup4
+
+ - name: Generate URLs to check
+ if: steps.check_preview.outputs.preview_exists == 'true'
+ id: generate_urls
+ run: |
+ BASE_URL="$PREVIEW_URL"
+
+ # Create a Python script to crawl the site
+ cat > crawl.py << 'EOF'
+ import requests
+ from bs4 import BeautifulSoup
+ import sys
+ from urllib.parse import urljoin, urlparse
+ import json
+ import xml.etree.ElementTree as ET
+ import base64
+ import os
+
+ # Define root pages to check
+ ROOT_PAGES = [
+ "index.html",
+ "get-started/get-started.html",
+ "guide/guides.html",
+ "developer/validmind-library.html",
+ "support/support.html",
+ "releases/all-releases.html",
+ "training/training.html"
+ ]
+
+ def get_auth_headers():
+ # Only use auth for installation pages
+ if 'installation/' in url:
+ # Create auth headers from environment variables
+ auth_string = base64.b64encode(f"{os.environ['INSTALLATION_USER']}:{os.environ['INSTALLATION_PW']}".encode()).decode()
+ return {"Authorization": f"Basic {auth_string}"}
+ return {}
+
+ def get_url_depth(url):
+ # Parse the URL to get just the path
+ path = urlparse(url).path
+ # Remove .html extension for depth calculation
+ path = path.replace('.html', '')
+ # Remove any leading/trailing slashes
+ path = path.strip('/')
+
+ # Split into segments and count non-empty ones
+ segments = [x for x in path.split('/') if x]
+
+ # For PR preview URLs, we need to skip the first 5 segments:
+ # /pr_previews/username/branch/name/
+ if 'pr_previews' in path:
+ # Skip the first 5 segments (pr_previews/username/branch/name/)
+ segments = segments[5:]
+
+ # Debug the depth calculation
+ # print(f"URL depth calculation - Path: {path}, Segments: {segments}, Depth: {len(segments)}", file=sys.stderr)
+
+ return len(segments)
+
+ def get_urls_from_sitemap(sitemap_url, max_depth):
+ try:
+ print(f"Fetching sitemap from {sitemap_url}", file=sys.stderr)
+ # Don't use auth for sitemap
+ response = requests.get(sitemap_url)
+ print(f"Sitemap response status: {response.status_code}", file=sys.stderr)
+ if response.status_code == 200:
+ print(f"Sitemap content: {response.text[:500]}...", file=sys.stderr)
+ root = ET.fromstring(response.content)
+ # Get all URLs from sitemap
+ all_urls = set()
+
+ for url in root.findall('.//{http://www.sitemaps.org/schemas/sitemap/0.9}url'):
+ loc = url.find('{http://www.sitemaps.org/schemas/sitemap/0.9}loc')
+ if loc is not None:
+ full_url = loc.text
+ parsed_url = urlparse(full_url)
+
+ # Extract the path part after the base URL
+ path = parsed_url.path
+ # Remove leading slash if present
+ path = path.lstrip('/')
+
+ # Only include .html files
+ if path.endswith('.html'):
+ # Check depth
+ if get_url_depth(path) <= max_depth:
+ # Remove any segments that match the PR preview path
+ segments = path.split('/')
+ # Keep only the segments after the PR preview path
+ pr_preview_index = -1
+ for i, segment in enumerate(segments):
+ if segment == 'pr_previews':
+ pr_preview_index = i
+ break
+ if pr_preview_index >= 0:
+ segments = segments[pr_preview_index + 4:] # Skip pr_previews/username/branch/name
+ path = '/'.join(segments)
+ all_urls.add(path)
+ print(f"Found URL in sitemap: {path}", file=sys.stderr)
+
+ print(f"Found {len(all_urls)} URLs in sitemap:", file=sys.stderr)
+ for url in sorted(all_urls):
+ print(f" {url}", file=sys.stderr)
+ return sorted(list(all_urls))
+ else:
+ print(f"Failed to fetch sitemap: {response.status_code}", file=sys.stderr)
+ except Exception as e:
+ print(f"Error processing sitemap {sitemap_url}: {str(e)}", file=sys.stderr)
+ return []
+
+ def get_links(url, max_depth, visited=None):
+ if visited is None:
+ visited = set()
+
+ current_depth = get_url_depth(url)
+ print(f"Checking URL {url} at depth {current_depth}", file=sys.stderr)
+
+ if current_depth > max_depth or url in visited:
+ print(f"Skipping {url} - depth {current_depth} > {max_depth} or already visited", file=sys.stderr)
+ return set()
+
+ visited.add(url)
+ links = set()
+
+ try:
+ print(f"Fetching {url}", file=sys.stderr)
+ headers = get_auth_headers()
+ response = requests.get(url, headers=headers)
+ print(f"Response status: {response.status_code}", file=sys.stderr)
+ if response.status_code == 200:
+ soup = BeautifulSoup(response.text, 'html.parser')
+ print(f"Found {len(soup.find_all('a', href=True))} links on page", file=sys.stderr)
+
+ for a in soup.find_all('a', href=True):
+ href = a['href']
+ print(f"Processing link: {href}", file=sys.stderr)
+
+ # Skip external links and anchors
+ if href.startswith('#') or href.startswith('http'):
+ print(f"Skipping external/anchor link: {href}", file=sys.stderr)
+ continue
+
+ # Convert relative URLs to absolute
+ full_url = urljoin(url, href)
+ print(f"Converted to full URL: {full_url}", file=sys.stderr)
+
+ # Only include URLs from the same base domain
+ if urlparse(full_url).netloc == urlparse(url).netloc:
+ # Extract just the path part
+ path = urlparse(full_url).path
+ # Remove leading slash if present
+ path = path.lstrip('/')
+
+ # Only include .html files
+ if path.endswith('.html'):
+ print(f"Found HTML link: {path}", file=sys.stderr)
+ links.add(path)
+ # Only recursively get links if we haven't hit max depth
+ if get_url_depth(path) < max_depth:
+ print(f"Recursively checking {path} at depth {get_url_depth(path)}", file=sys.stderr)
+ links.update(get_links(full_url, max_depth, visited))
+ else:
+ print(f"Skipping recursive check for {path} - at max depth", file=sys.stderr)
+ else:
+ print(f"Skipping external domain link: {href}", file=sys.stderr)
+ except Exception as e:
+ print(f"Error processing {url}: {str(e)}", file=sys.stderr)
+
+ return links
+
+ # Get command line arguments
+ base_url = sys.argv[1]
+ max_depth = int(sys.argv[2])
+
+ print(f"Base URL: {base_url}", file=sys.stderr)
+ print(f"Max depth: {max_depth}", file=sys.stderr)
+
+ # Get all URLs
+ all_urls = set()
+
+ if max_depth == 0:
+ # For depth 0, only check ROOT_PAGES
+ print("Depth is 0, only checking ROOT_PAGES", file=sys.stderr)
+ for root in ROOT_PAGES:
+ all_urls.add(root)
+ print(f"Added root page: {root}", file=sys.stderr)
+ else:
+ # For depth > 0, use sitemap
+ print(f"Depth is {max_depth}, using sitemap", file=sys.stderr)
+ sitemap_url = f"{base_url}/sitemap.xml"
+ sitemap_urls = get_urls_from_sitemap(sitemap_url, max_depth)
+ print(f"Found {len(sitemap_urls)} URLs in sitemap", file=sys.stderr)
+ all_urls.update(sitemap_urls)
+
+ # Print URLs to stdout, ensuring proper URL construction
+ print(f"Total URLs found: {len(all_urls)}", file=sys.stderr)
+ for url in sorted(all_urls):
+ # Remove any leading slashes from the URL to avoid double slashes
+ url = url.lstrip('/')
+ # Construct the full URL by joining base_url and url with a single slash
+ full_url = f"{base_url.rstrip('/')}/{url}"
+ print(full_url)
+ print(f"Added URL: {full_url}", file=sys.stderr)
+ EOF
+
+ # Run the crawler
+ python crawl.py "$BASE_URL" "$DEPTH" > lhci-urls.txt
+
+ echo "Lighthouse will check the following URLs:"
+ cat lhci-urls.txt
+ echo -e "\nTotal number of URLs: $(wc -l < lhci-urls.txt)"
+
+ # Verify we have URLs
+ if [ ! -s lhci-urls.txt ]; then
+ echo "Error: No URLs were generated. Check the debug output above."
+ exit 1
+ fi
+
+ - name: Create Lighthouse config
+ if: steps.check_preview.outputs.preview_exists == 'true'
+ run: |
+ cat > .lighthouserc.js << 'EOF'
+ const fs = require('fs');
+ const urls = fs.readFileSync('lhci-urls.txt', 'utf-8').split('\n').filter(Boolean);
+
+ // Add auth to installation URLs using the same format as the URL check step
+ const urlsWithAuth = urls.map(url => {
+ if (url.includes('/installation/')) {
+ return `https://${process.env.INSTALLATION_USER}:${process.env.INSTALLATION_PW}@${new URL(url).host}${new URL(url).pathname}`;
+ }
+ return url;
+ });
+
+ module.exports = {
+ ci: {
+ collect: {
+ url: urlsWithAuth,
+ numberOfRuns: 1,
+ settings: {
+ formFactor: 'desktop',
+ screenEmulation: {
+ mobile: false,
+ width: 1350,
+ height: 940,
+ deviceScaleFactor: 1,
+ disabled: false,
+ },
+ throttling: {
+ rttMs: 40,
+ throughputKbps: 10240,
+ cpuSlowdownMultiplier: 1,
+ requestLatencyMs: 0,
+ downloadThroughputKbps: 0,
+ uploadThroughputKbps: 0,
+ },
+ },
+ },
+ assert: {
+ assertions: {
+ 'categories:accessibility': ['error', { minScore: 0.9 }],
+ },
+ },
+ upload: {
+ target: 'temporary-public-storage',
+ },
+ },
+ };
+ EOF
+
+ - name: Run Lighthouse audit
+ if: steps.check_preview.outputs.preview_exists == 'true'
+ uses: treosh/lighthouse-ci-action@v11
+ id: lighthouse
+ continue-on-error: true
+ env:
+ INSTALLATION_USER: ${{ secrets.INSTALLATION_USER }}
+ INSTALLATION_PW: ${{ secrets.INSTALLATION_PW }}
+ with:
+ configPath: .lighthouserc.js
+ uploadArtifacts: true
+ temporaryPublicStorage: true
+
+ - name: Check Lighthouse audit result
+ if: steps.check_preview.outputs.preview_exists == 'true'
+ run: |
+ # Check if the manifest exists and is valid JSON
+ if [ -z "${{ steps.lighthouse.outputs.manifest }}" ]; then
+ echo "Error: Lighthouse audit failed - no manifest output"
+ exit 1
+ fi
+
+ # Try to parse the manifest as JSON
+ if ! echo '${{ steps.lighthouse.outputs.manifest }}' | jq . > /dev/null 2>&1; then
+ echo "Error: Lighthouse audit failed - invalid manifest format"
+ exit 1
+ fi
+
+ # Check if any URLs were successfully audited
+ if ! echo '${{ steps.lighthouse.outputs.manifest }}' | jq 'length > 0' > /dev/null 2>&1; then
+ echo "Error: Lighthouse audit failed - no URLs were successfully audited"
+ exit 1
+ fi
+
+ - name: Post Lighthouse results comment
+ if: steps.check_preview.outputs.preview_exists == 'true'
+ uses: actions/github-script@v6
+ with:
+ script: |
+ const runId = context.runId;
+ const baseUrl = process.env.PREVIEW_URL;
+ const commitSha = process.env.COMMIT_SHA;
+ const commitShaShort = process.env.COMMIT_SHA_SHORT;
+
+ // Get artifacts for this run
+ const { data: artifacts } = await github.rest.actions.listWorkflowRunArtifacts({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ run_id: runId,
+ });
+
+ // Lighthouse artifact
+ const lighthouseArtifact = artifacts.artifacts.find(a => a.name === 'lighthouse-report');
+ const lighthouseArtifactUrl = lighthouseArtifact
+ ? `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}/artifacts/${lighthouseArtifact.id}`
+ : null;
+
+ // Lighthouse
+ const manifest = '${{ steps.lighthouse.outputs.manifest }}';
+ let manifestJson;
+ try {
+ manifestJson = JSON.parse(manifest);
+ if (!Array.isArray(manifestJson) || manifestJson.length === 0) {
+ throw new Error('Invalid manifest format or empty results');
+ }
+ } catch (error) {
+ console.error('Error parsing Lighthouse manifest:', error);
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ body: `## Lighthouse check results\n\n⚠️ WARN: Failed to parse Lighthouse results. Please check the [workflow run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}) for details.`
+ });
+ return;
+ }
+
+ // Delete old Lighthouse comments
+ const { data: comments } = await github.rest.issues.listComments({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ });
+
+ // Delete any previous comments from this workflow
+ for (const comment of comments) {
+ if (comment.user.login === 'github-actions[bot]' &&
+ comment.body.includes('## Lighthouse check results')) {
+ try {
+ console.log(`Deleting Lighthouse comment ${comment.id}`);
+ await github.rest.issues.deleteComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: comment.id,
+ });
+ console.log(`Successfully deleted Lighthouse comment ${comment.id}`);
+ } catch (error) {
+ console.error(`Failed to delete Lighthouse comment ${comment.id}:`, error);
+ }
+ }
+ }
+
+ // Calculate average accessibility score
+ const scores = manifestJson.map(run => run.summary.accessibility);
+ const avgScore = scores.reduce((a, b) => a + b, 0) / scores.length;
+ const lighthouseScore = avgScore.toFixed(2);
+
+ const lighthouseReportUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`;
+ let lighthouseComment = '';
+ if (parseFloat(lighthouseScore) >= 0.9) {
+ lighthouseComment = `✓ INFO: Average accessibility score is **${lighthouseScore}** (required: >0.9) — [View the workflow run](${lighthouseReportUrl})`;
+ } else {
+ lighthouseComment = `⚠️ WARN: Average accessibility score is **${lighthouseScore}** (required: >0.9) — [Check the workflow run](${lighthouseReportUrl})`;
+ }
+
+ const stripAuth = url => {
+ try {
+ const u = new URL(url);
+ u.username = '';
+ u.password = '';
+ return u.toString();
+ } catch {
+ return url;
+ }
+ };
+
+ // Helper to get the public report URL from htmlPath
+ const getReportUrl = (run) => {
+ if (run.report && Array.isArray(run.report)) {
+ // Find the public .report.html URL
+ const htmlReport = run.report.find(r => r.endsWith('.report.html') && r.startsWith('http'));
+ if (htmlReport) return htmlReport;
+ // Fallback: first report if available
+ if (run.report.length > 0) return run.report[0];
+ }
+ // Fallback: just show the workflow run if nothing else
+ return lighthouseReportUrl;
+ };
+
+ // Parse the links output from the Lighthouse step
+ const links = (() => {
+ try {
+ return JSON.parse(`${{ steps.lighthouse.outputs.links }}`);
+ } catch {
+ return {};
+ }
+ })();
+
+ const scoresTable = manifestJson
+ .map(run => {
+ const formatScore = (score) => score === null ? 'N/A' : score.toFixed(2);
+ const displayPath = stripAuth(run.url).replace(baseUrl, '');
+ // Use the public report URL from the links output, fallback to workflow run if missing
+ const reportUrl = links[run.url] || lighthouseReportUrl;
+ return `| [${displayPath}](${reportUrl}) | ${formatScore(run.summary.accessibility)} | ${formatScore(run.summary.performance)} | ${formatScore(run.summary['best-practices'])} | ${formatScore(run.summary.seo)} |`;
+ })
+ .join('\n');
+
+ let comment = `## Lighthouse check results\n\n`;
+ comment += `${lighthouseComment}\n\n`;
+ comment += `\nShow Lighthouse scores
\n\n`;
+ comment += `Folder depth level checked: **${process.env.DEPTH}**\n\n`;
+ comment += `Commit SHA: [${commitShaShort}](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/commit/${commitSha})\n\n`;
+ comment += `Modify the workflow to check a different depth:\n`;
+ comment += `- 0: Top-level navigation only — /index.html, /guide/guides.html, ...\n`;
+ comment += `- 1: All first-level subdirectories — /guide/\*.html, /developer/\*.html, ...\n`;
+ comment += `- 2: All second-level subdirectories — /guide/attestation/\*.html, ...\n\n`;
+ comment += `| Page | Accessibility | Performance | Best Practices | SEO |\n`;
+ comment += `|------|---------------|-------------|----------------|-----|\n`;
+ comment += `${scoresTable}\n\n`;
+ comment += ` \n\n`;
+
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ body: comment
+ });
\ No newline at end of file
diff --git a/.github/workflows/merge-main-into-staging.yaml b/.github/workflows/merge-main-into-staging.yaml
index c9a712ca40..4a6f6afb96 100644
--- a/.github/workflows/merge-main-into-staging.yaml
+++ b/.github/workflows/merge-main-into-staging.yaml
@@ -39,11 +39,16 @@ jobs:
title: 'Merge main into staging'
body: 'Automatically merge main into staging branch.'
- - name: Merge pull request
+ - name: Merge pull request (with retries)
if: ${{ steps.pr-number.outputs.pull-request-number != '' }}
- run: gh pr merge --merge --auto "${{ steps.pr-number.outputs.pull-request-number }}"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ for i in {1..5}; do
+ gh pr merge --merge --auto "${{ steps.pr-number.outputs.pull-request-number }}" && break
+ echo "Merge failed, retrying in 10 seconds ..."
+ sleep 10
+ done
- name: Delete pull request branch
if: ${{ success() && steps.pr-number.outputs.pull-request-number != '' }}
diff --git a/.github/workflows/vale-lint.yaml b/.github/workflows/vale-lint.yaml
new file mode 100644
index 0000000000..8a037dfc97
--- /dev/null
+++ b/.github/workflows/vale-lint.yaml
@@ -0,0 +1,140 @@
+name: Vale linter
+
+on:
+ pull_request:
+ types: [opened, synchronize, ready_for_review]
+
+permissions:
+ issues: write
+ pull-requests: write
+
+jobs:
+ vale:
+ runs-on: ubuntu-latest
+ if: github.event.pull_request.draft == false
+
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v4
+
+ - name: Install Vale
+ run: |
+ wget https://github.com/errata-ai/vale/releases/download/v2.28.0/vale_2.28.0_Linux_64-bit.tar.gz
+ tar -xvzf vale_2.28.0_Linux_64-bit.tar.gz
+ sudo mv vale /usr/local/bin/
+
+ - name: Run Vale
+ id: vale
+ continue-on-error: true
+ run: |
+ vale --output=JSON site/ > vale-report.json || {
+ echo "Vale linting found issues"
+ exit 1
+ }
+
+ - name: Format Vale report for PR comment
+ run: jq . vale-report.json > vale-report-pretty.json
+
+ - name: Upload Vale report as artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: vale-report
+ path: vale-report.json
+ retention-days: 1
+
+ - name: Post Vale results comment
+ uses: actions/github-script@v6
+ with:
+ script: |
+ const runId = context.runId;
+ // Get artifacts for this run
+ const { data: artifacts } = await github.rest.actions.listWorkflowRunArtifacts({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ run_id: runId,
+ });
+ const valeArtifact = artifacts.artifacts.find(a => a.name === 'vale-report');
+ const valeArtifactUrl = valeArtifact
+ ? `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}/artifacts/${valeArtifact.id}`
+ : null;
+
+ // Delete old Vale comments
+ const { data: comments } = await github.rest.issues.listComments({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ });
+
+ for (const comment of comments) {
+ if (comment.user.login === 'github-actions[bot]' && comment.body.includes('## Vale source linter')) {
+ try {
+ console.log(`Deleting Vale comment ${comment.id}`);
+ await github.rest.issues.deleteComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: comment.id,
+ });
+ console.log(`Successfully deleted Vale comment ${comment.id}`);
+ } catch (error) {
+ console.error(`Failed to delete Vale comment ${comment.id}:`, error);
+ }
+ }
+ }
+
+ const fs = require('fs');
+ const report = JSON.parse(fs.readFileSync('vale-report.json', 'utf8'));
+
+ let comment = '## Vale source linter\n\n';
+
+ if (Object.keys(report).length === 0) {
+ comment += `✓ INFO: No writing issues were found ([report](${valeArtifactUrl}))\n\n`;
+ } else {
+ // Build the summary output
+ let summaryOutput = '';
+ let totalIssues = 0;
+ let fileCount = 0;
+
+ for (const [file, issues] of Object.entries(report)) {
+ totalIssues += issues.length;
+ fileCount++;
+
+ // Include first 30 files to keep comment size manageable
+ if (fileCount <= 30) {
+ summaryOutput += `### ${file}\n`;
+ // Only include first 10 issues per file
+ const issuesToShow = issues.slice(0, 10);
+ for (const issue of issuesToShow) {
+ summaryOutput += `- Line ${issue.Line}: ${issue.Message} (${issue.Severity})\n`;
+ }
+ if (issues.length > 10) {
+ summaryOutput += `- ... and ${issues.length - 10} more issues\n`;
+ }
+ summaryOutput += '\n';
+ }
+ }
+
+ const workflowRunUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`;
+ comment += `⚠️ WARN: Found **${totalIssues}** issues across **${fileCount}** files — [Check the workflow run](${workflowRunUrl}) or [download the full report](${valeArtifactUrl})\n\n`;
+
+ // Add the summary output in a collapsed section
+ comment += `\nShow first 30 files with issues
\n\n`;
+ comment += summaryOutput;
+ if (fileCount > 30) {
+ comment += `\n... and ${fileCount - 30} more files with issues\n`;
+ }
+ comment += `\n \n\n`;
+ }
+
+ // Ensure comment doesn't exceed GitHub's limit
+ const MAX_COMMENT_LENGTH = 60000; // Leave some buffer
+ if (comment.length > MAX_COMMENT_LENGTH) {
+ comment = comment.substring(0, MAX_COMMENT_LENGTH) +
+ `\n\n... (comment truncated due to length) - See full report in artifacts`;
+ }
+
+ github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ body: comment
+ });
\ No newline at end of file
diff --git a/.github/workflows/validate-docs-site.yaml b/.github/workflows/validate-docs-site.yaml
index 2bdd873180..3c7052e4c9 100644
--- a/.github/workflows/validate-docs-site.yaml
+++ b/.github/workflows/validate-docs-site.yaml
@@ -46,7 +46,6 @@ jobs:
cat render_errors.log;
exit 1;
}
-
# See if site/notebooks/ has updates
# Checks the current PR branch against the target branch
- name: Filter changed files
@@ -102,7 +101,7 @@ jobs:
echo "No warnings or errors detected during Quarto render"
fi
- # Demo bucket is in us-east-1
+ # Demo bucket is in us-east-1
- name: Configure AWS credentials
run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} && aws configure set default.region us-east-1
@@ -113,10 +112,31 @@ jobs:
uses: actions/github-script@v6
with:
script: |
- const url = `https://docs-demo.vm.validmind.ai/pr_previews/${{ github.head_ref }}/index.html`;
+ const previewUrl = `https://docs-demo.vm.validmind.ai/pr_previews/${{ github.head_ref }}/index.html`;
+
+ // Delete old preview comments
+ const { data: comments } = await github.rest.issues.listComments({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ });
+
+ for (const comment of comments) {
+ if (comment.user.login === 'github-actions[bot]' && comment.body.includes('## Validate docs site')) {
+ await github.rest.issues.deleteComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: comment.id,
+ });
+ }
+ }
+
+ let comment = `## Validate docs site\n\n`;
+ comment += `✓ INFO: A live preview of the docs site is available — [Open the preview](${previewUrl})\n\n`;
+
github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
- body: `A PR preview is available: [Preview URL](${url})`
+ body: comment
});
diff --git a/.vale.ini b/.vale.ini
new file mode 100644
index 0000000000..2e53c3e747
--- /dev/null
+++ b/.vale.ini
@@ -0,0 +1,13 @@
+StylesPath = .vale/styles
+MinAlertLevel = suggestion
+
+Skips = _*, */_*, plugin/*
+
+Packages = Google
+
+[*.md]
+BasedOnStyles = Vale, Google
+
+[*.qmd]
+BasedOnStyles = Vale, Google
+
diff --git a/.vale/styles/Google/AMPM.yml b/.vale/styles/Google/AMPM.yml
new file mode 100644
index 0000000000..37b49edf87
--- /dev/null
+++ b/.vale/styles/Google/AMPM.yml
@@ -0,0 +1,9 @@
+extends: existence
+message: "Use 'AM' or 'PM' (preceded by a space)."
+link: "https://developers.google.com/style/word-list"
+level: error
+nonword: true
+tokens:
+ - '\d{1,2}[AP]M\b'
+ - '\d{1,2} ?[ap]m\b'
+ - '\d{1,2} ?[aApP]\.[mM]\.'
diff --git a/.vale/styles/Google/Acronyms.yml b/.vale/styles/Google/Acronyms.yml
new file mode 100644
index 0000000000..f41af0189b
--- /dev/null
+++ b/.vale/styles/Google/Acronyms.yml
@@ -0,0 +1,64 @@
+extends: conditional
+message: "Spell out '%s', if it's unfamiliar to the audience."
+link: 'https://developers.google.com/style/abbreviations'
+level: suggestion
+ignorecase: false
+# Ensures that the existence of 'first' implies the existence of 'second'.
+first: '\b([A-Z]{3,5})\b'
+second: '(?:\b[A-Z][a-z]+ )+\(([A-Z]{3,5})\)'
+# ... with the exception of these:
+exceptions:
+ - API
+ - ASP
+ - CLI
+ - CPU
+ - CSS
+ - CSV
+ - DEBUG
+ - DOM
+ - DPI
+ - FAQ
+ - GCC
+ - GDB
+ - GET
+ - GPU
+ - GTK
+ - GUI
+ - HTML
+ - HTTP
+ - HTTPS
+ - IDE
+ - JAR
+ - JSON
+ - JSX
+ - LESS
+ - LLDB
+ - NET
+ - NOTE
+ - NVDA
+ - OSS
+ - PATH
+ - PDF
+ - PHP
+ - POST
+ - RAM
+ - REPL
+ - RSA
+ - SCM
+ - SCSS
+ - SDK
+ - SQL
+ - SSH
+ - SSL
+ - SVG
+ - TBD
+ - TCP
+ - TODO
+ - URI
+ - URL
+ - USB
+ - UTF
+ - XML
+ - XSS
+ - YAML
+ - ZIP
diff --git a/.vale/styles/Google/Colons.yml b/.vale/styles/Google/Colons.yml
new file mode 100644
index 0000000000..4a027c307d
--- /dev/null
+++ b/.vale/styles/Google/Colons.yml
@@ -0,0 +1,8 @@
+extends: existence
+message: "'%s' should be in lowercase."
+link: 'https://developers.google.com/style/colons'
+nonword: true
+level: warning
+scope: sentence
+tokens:
+ - '(?=1.0.0"
+}
diff --git a/.vale/styles/Google/vocab.txt b/.vale/styles/Google/vocab.txt
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/README.md b/README.md
index c840837606..22b9fa7f9d 100644
--- a/README.md
+++ b/README.md
@@ -227,3 +227,48 @@ Configure in `config.json`, generated with the Docker image:
"JUPYTERHUB_URL": "https://your-custom-jupyter.validmind.ai"
}
```
+
+## Configuring Lighthouse checks
+
+Lighthouse is an open-source tool that audits web pages for accessibility, performance, best practices, and SEO. We automatically run Lighthouse against PR preview sites to enable a better, accessible documentation for everyone.
+
+By default, Lighthouse checks only the top-level pages in our site navigation, such as `/index.html`, `/guide/guides.html`, `/developer/validmind-library.html`, and so forth. You can configure this behavior in the workflow:
+
+```sh
+env:
+ # To change the default depth level:
+ # 0 — Top-level navigation only (e.g. /index.html, /guide/guides.html, /developer/validmind-library.html, etc.)
+ # 1 — All first-level subdirectories (e.g. /guide/*.html)
+ # 2 — All second-level subdirectories (e.g. /guide/attestation/*.html)
+ # Note: While the crawler technically supports deeper levels, expect the workflow to take >2-12 hours to complete
+ DEFAULT_DEPTH: '0'
+```
+
+**Tips:**
+
+- On the first run, the workflow waits for a preview site to become available. For subsequent runs, it checks the currently available site, which may be behind HEAD. The PR comment shows which commit SHA was checked — rerun the check if needed.
+- Use folder depths greater than zero only on working branches when you need a thorough site audit. Deeper checks take 2-12 hours to complete and significantly slow down the CI/CD pipeline. Do not merge depth changes to `main`.
+
+## Vale linter
+
+The Vale linter is used to enforce consistent writing style and catch common language issues in our documentation source. Vale runs automatically on pull requests but can also be run locally when addressing source issues.
+
+### Running Vale locally
+
+```sh
+brew install vale
+vale site/
+```
+
+**Tip:** Locally, you can use Vale to check specific content areas you are working on, such as `site/guides/`.
+
+### Configuring Vale
+
+- The linter is configured via a `vale.ini` file in the root of the repository. This file specifies which styles to use and which files or directories to skip.
+- Community styles such as `Vale` and `Google` are installed automatically in the CI workflow.
+- The workflow is set up to ignore files and folders starting with an underscore (`_`) and the `site/plugin` directory.
+
+### FUTURE: Customizing rules
+
+- To add or remove styles, edit the `BasedOnStyles` lines in your `vale.ini`.
+- To skip additional files or folders, update the `Skips` setting in `vale.ini` or adjust the workflow globs.
diff --git a/site/Makefile b/site/Makefile
index e4a0c54b93..49087d371e 100644
--- a/site/Makefile
+++ b/site/Makefile
@@ -6,14 +6,14 @@ SRC_DIR := $(SRC_ROOT)/validmind-library
DEST_DIR_NB := notebooks
DEST_DIR_PYTHON := validmind
DEST_DIR_TESTS := tests
-GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
+GIT_BRANCH := $(or $(GITHUB_HEAD_REF),$(GITHUB_REF_NAME),$(shell git rev-parse --abbrev-ref HEAD))
INSTALLATION_BRANCH := main
LIBRARY_BRANCH ?= $(or $(BRANCH),main)
SRC_ROOT := _source
SRC_DIR := $(SRC_ROOT)/validmind-library
# Define .PHONY target for help section
-.PHONY: help clean clone copy-installation delete-demo-branch deploy-demo-branch deploy-prod deploy-staging docker-build docker-serve docker-site docker-site-lite docs-site execute get-source notebooks python-docs release-notes test-descriptions yearly-releases
+.PHONY: help clean clone copy-installation delete-demo-branch deploy-demo-branch deploy-prod deploy-staging docker-build docker-serve docker-site docker-site-lite docs-site execute generate-sitemap get-source notebooks python-docs release-notes test-descriptions yearly-releases
# Help section
help:
@@ -32,6 +32,7 @@ help:
@echo " docker-site Get source, render site with Docker profile, execute notebooks"
@echo " docker-site-lite Get source and render site with Docker profile (skips notebook execution)"
@echo " docs-site Get all source files and render the production docs site with Quarto"
+ @echo " generate-sitemap Generate a sitemap for the static HTML site"
@echo " execute Execute a Jupyter Notebook or notebook directory"
@echo " get-source Get all source files (clean, clone, copy-installation, notebooks, python-docs, test-descriptions)"
@echo " notebooks Copy Jupyter notebooks into notebooks/"
@@ -164,6 +165,74 @@ execute:
quarto render --profile "$$PROFILE" "$$FILE_PATH"; \
rm $$env_file '
+# Generate sitemap for the site
+generate-sitemap:
+ @echo "Generating sitemaps ..."
+ @find _site -name "*.html" -not -path "*/internal/*" -not -path "*/site_libs/*" -not -path "*/sitemap.html/*" -not -path "*/training.html/*" | while read -r file; do \
+ url_path=$$(echo "$$file" | sed 's|^_site/||'); \
+ if [ "$$(uname)" = "Darwin" ]; then \
+ lastmod=$$(stat -f "%Sm" -t "%Y-%m-%dT%H:%M:%SZ" "$$file"); \
+ else \
+ lastmod=$$(stat -c "%y" "$$file" | sed 's/ /T/;s/$$/Z/'); \
+ fi; \
+ printf " \n https://docs-demo.vm.validmind.ai/pr_previews/$(GIT_BRANCH)/%s\n %s\n weekly\n 0.8\n \n" "$$url_path" "$$lastmod" >> sitemap.xml.tmp; \
+ printf "%s\n" "$$url_path" >> sitemap.urls.tmp; \
+ done
+ @printf '\n\n' > _site/sitemap.xml
+ @cat sitemap.xml.tmp >> _site/sitemap.xml
+ @printf '\n' >> _site/sitemap.xml
+ @rm sitemap.xml.tmp
+ @printf '\n\n\n \n ValidMind Documentation & Training Sitemap\n \n\n\n \n
Root Pages
\n
\n' > _site/sitemap.html
+ @# Deduplicate URLs before processing
+ @sort -u sitemap.urls.tmp > sitemap.urls.unique.tmp
+ @mv sitemap.urls.unique.tmp sitemap.urls.tmp
+ @grep -E '^(index|404)\.html$$' sitemap.urls.tmp | while read -r url; do \
+ file="_site/$$url"; \
+ if [ -f "$$file" ] && grep -q '' "$$file"; then \
+ title=$$(grep -m1 '<title>' "$$file" | sed 's/.*<title>\(.*\)<\/title>.*/\1/'); \
+ else \
+ title="$$url"; \
+ fi; \
+ if [ "$$title" != "Redirect" ]; then \
+ printf " <li><a href=\"https://docs-demo.vm.validmind.ai/pr_previews/$(GIT_BRANCH)/%s\">%s</a></li>\n" "$$url" "$$title" >> _site/sitemap.html; \
+ fi; \
+ done
+ @printf ' </ul>\n </div>\n' >> _site/sitemap.html
+ @# Group remaining pages by top-level directory
+ @for dir in $$(grep -v -E '^(index|404)\.html$$' sitemap.urls.tmp | cut -d/ -f1 | grep -vE '\.(html|xml)$$' | sort -u); do \
+ printf ' <h2>%s/</h2>\n <ul>\n' "$$dir" >> _site/sitemap.html; \
+ grep -E "^$$dir/[^/]+\.html$$" sitemap.urls.tmp | while read -r url; do \
+ file="_site/$$url"; \
+ if [ -f "$$file" ] && grep -q '<title>' "$$file"; then \
+ title=$$(grep -m1 '<title>' "$$file" | sed 's/.*<title>\(.*\)<\/title>.*/\1/'); \
+ else \
+ title="$$url"; \
+ fi; \
+ if [ "$$title" != "Redirect" ]; then \
+ printf " <li><a href=\"https://docs-demo.vm.validmind.ai/pr_previews/$(GIT_BRANCH)/%s\">%s</a></li>\n" "$$url" "$$title" >> _site/sitemap.html; \
+ fi; \
+ done; \
+ for sub in $$(grep -E "^$$dir/" sitemap.urls.tmp | grep -o -E "^$$dir/[^/]+/" | sed "s|^$$dir/||;s|/||" | sort -u); do \
+ printf ' <li>%s/\n <ul>\n' "$$sub" >> _site/sitemap.html; \
+ grep -E "^$$dir/$$sub/" sitemap.urls.tmp | while read -r url; do \
+ file="_site/$$url"; \
+ if [ -f "$$file" ] && grep -q '<title>' "$$file"; then \
+ title=$$(grep -m1 '<title>' "$$file" | sed 's/.*<title>\(.*\)<\/title>.*/\1/'); \
+ else \
+ title="$$url"; \
+ fi; \
+ if [ "$$title" != "Redirect" ]; then \
+ printf " <li><a href=\"https://docs-demo.vm.validmind.ai/pr_previews/$(GIT_BRANCH)/%s\">%s</a></li>\n" "$$url" "$$title" >> _site/sitemap.html; \
+ fi; \
+ done; \
+ printf ' </ul>\n </li>\n' >> _site/sitemap.html; \
+ done; \
+ printf ' </ul>\n' >> _site/sitemap.html; \
+ done
+ @printf '</body>\n</html>\n' >> _site/sitemap.html
+ @rm sitemap.urls.tmp
+ @echo "Sitemaps generated at _site/sitemap.xml and _site/sitemap.html"
+
# Get all source files
get-source: clean clone copy-installation notebooks python-docs test-descriptions
@@ -182,8 +251,6 @@ notebooks:
@echo "Copying LICENSE into notebooks ..."
@cp -r $(SRC_DIR)/LICENSE $(DEST_DIR_NB)/
@rm -rf $(DEST_DIR_NB)/code_sharing
- @echo "Moving Quickstart notebook into the root of notebooks/ ..."
- @if [ -f "$(DEST_DIR_NB)/code_samples/quickstart_customer_churn_full_suite.ipynb" ]; then mv $(DEST_DIR_NB)/code_samples/quickstart_customer_churn_full_suite.ipynb $(DEST_DIR_NB)/quickstart_customer_churn_full_suite.ipynb; fi
@echo "Copying _metadata.yml into notebooks/ ..."
@cp developer/_metadata.yml $(DEST_DIR_NB)/_metadata.yml
@echo "Zip up notebooks.zip ..."
diff --git a/site/_extensions/nrichers/slideover/_extension.yml b/site/_extensions/nrichers/slideover/_extension.yml
new file mode 100644
index 0000000000..e942220563
--- /dev/null
+++ b/site/_extensions/nrichers/slideover/_extension.yml
@@ -0,0 +1,12 @@
+title: Reveal.js Slideover
+author: Nik Richers
+version: 1.3.1
+quarto-required: ">=1.6.0"
+contributes:
+ revealjs-plugins:
+ - name: slideover
+ script:
+ - slideover.js
+ stylesheet:
+ - slideover.css
+ - custom-slideover.css
\ No newline at end of file
diff --git a/site/_extensions/nrichers/slideover/custom-slideover.css b/site/_extensions/nrichers/slideover/custom-slideover.css
new file mode 100644
index 0000000000..06b9be80eb
--- /dev/null
+++ b/site/_extensions/nrichers/slideover/custom-slideover.css
@@ -0,0 +1,80 @@
+/* Global variables */
+:root {
+ --slideover-default-bg: #FAFAFA;
+ --slideover-default-text: #083e44;
+ --slideover-default-border: 1px solid #083e44;
+ --slideover-default-links: #DE257E;
+ --slideover-default-accent: #3E6C69;
+}
+
+/* Global slideover styling */
+.slideover__content {
+ background: var(--slideover-default-bg) !important;
+ color: var(--slideover-default-text) !important;
+ border: var(--slideover-default-border) !important;
+}
+
+/* Right-aligned emphasis border */
+.slideover--r.slideover__content {
+ border-left-width: 5px !important;
+}
+
+/* Left-aligned emphasis border */
+.slideover--l.slideover__content {
+ border-right-width: 5px !important;
+}
+
+/* Top-aligned emphasis border */
+.slideover--t.slideover__content {
+ border-bottom-width: 5px !important;
+}
+
+/* Bottom-aligned emphasis border */
+.slideover--b.slideover__content {
+ border-top-width: 5px !important;
+}
+
+/* Link styling */
+.slideover__content-area a {
+ color: var(--slideover-default-links);
+ text-decoration: none;
+}
+
+.slideover__content-area a:hover {
+ text-decoration: underline 2px solid var(--slideover-default-text);
+}
+
+/* Callout embed styling */
+.slideover__content-area .embed {
+ background-color: #f5fcfd;
+ border: 1px solid rgba(8, 62, 68, 0.2);
+}
+
+/* Emphasis styling */
+.slideover__content-area strong,
+.slideover__content-area b,
+.slideover__content-area i,
+.slideover__content-area em {
+ color: var(--slideover-default-accent) !important;
+}
+
+/* Quotation styling */
+q {
+ color: var(--slideover-default-accent) !important;
+}
+
+.slideover__content-area blockquote {
+ border-left: 4px solid var(--slideover-default-accent) !important;
+ color: var(--slideover-default-accent) !important;
+}
+
+/* Code styling */
+.slideover__content-area div.sourceCode {
+ background-color: var(--slideover-default-bg) !important;
+}
+
+.slideover__content-area code {
+ color: var(--slideover-default-text) !important;
+ background-color: #EAF8FA;
+ border: 1px solid rgba(8, 62, 68, 0.2);
+}
diff --git a/site/_extensions/nrichers/slideover/slideover.css b/site/_extensions/nrichers/slideover/slideover.css
new file mode 100644
index 0000000000..9a83d087c3
--- /dev/null
+++ b/site/_extensions/nrichers/slideover/slideover.css
@@ -0,0 +1,426 @@
+/* Import Inter font from Google Fonts */
+@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap');
+
+/* Define the default slideover widths */
+:root {
+ --slideover-default-lr: 40vw;
+ --slideover-default-tb: 80vw;
+
+ --slideover-3quarters-lr: calc(var(--slideover-default-lr) * 0.75);
+ --slideover-3quarters-tb: calc(var(--slideover-default-tb) * 0.75);
+
+ --slideover-half-lr: calc(var(--slideover-default-lr) / 2);
+ --slideover-half-tb: calc(var(--slideover-default-tb) / 2);
+
+ --slideover-third-lr: calc(var(--slideover-default-lr) / 3);
+ --slideover-third-tb: calc(var(--slideover-default-tb) / 3);
+}
+
+/* Modifiers for half, third, three quarters (LR) */
+.slideover--l.slideover__content.half,
+.slideover--r.slideover__content.half {
+ width: var(--slideover-half-lr) !important;
+}
+
+.slideover--l.slideover__content.third,
+.slideover--r.slideover__content.third {
+ width: var(--slideover-third-lr) !important;
+}
+
+.slideover--l.slideover__content.three-quarters,
+.slideover--r.slideover__content.three-quarters {
+ width: var(--slideover-3quarters-lr) !important;
+}
+
+/* Modifiers for half, third, three quarters (TB) */
+.slideover--t.slideover__content.half,
+.slideover--b.slideover__content.half {
+ width: var(--slideover-half-tb) !important;
+}
+
+.slideover--t.slideover__content.third,
+.slideover--b.slideover__content.third {
+ width: var(--slideover-third-tb) !important;
+}
+
+.slideover--t.slideover__content.three-quarters,
+.slideover--b.slideover__content.three-quarters {
+ width: var(--slideover-3quarters-tb) !important;
+}
+
+.slideover__presentation {
+ position: relative;
+}
+
+.slideover__container {
+ position: fixed;
+ top: 0;
+ left: 0;
+ width: 100%;
+ height: 100%;
+ pointer-events: none;
+ z-index: 1;
+ display: flex;
+ justify-content: left;
+ align-items: left;
+}
+
+/* Global slideover container styling */
+.slideover__content {
+ position: fixed;
+ box-shadow: -2px 0 4px rgba(8, 62, 68, 0.1), 0 0 4px rgba(8, 62, 68, 0.1);
+ transition: transform 0.3s ease-in-out;
+ pointer-events: auto;
+ z-index: 1001;
+ display: flex;
+ flex-direction: column;
+ font-family: 'Inter', -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
+ font-size: 1.25em;
+ margin: 0;
+}
+
+/* Slideover header global styling */
+.slideover__header {
+ height: 40px;
+ display: flex;
+ align-items: left;
+ justify-content: left;
+ cursor: pointer;
+ flex-shrink: 0;
+ padding: 0 8px;
+ margin: 0;
+ position: relative;
+ z-index: 3;
+ pointer-events: auto;
+}
+
+/* Global slideover content area styling */
+.slideover__content-area {
+ padding: 0 1.5rem 1.5rem;
+ overflow-y: auto;
+ line-height: 1.5;
+ margin: -40px 0 0 0;
+ position: relative;
+ z-index: 2;
+ padding-top: 25px;
+ opacity: 1;
+ visibility: visible;
+ transition: opacity 0.3s ease-in-out, visibility 0.3s ease-in-out;
+}
+
+.slideover__content--active .slideover__content-area {
+ z-index: 3;
+}
+
+.slideover__content:not(.slideover__content--active) .slideover__content-area {
+ opacity: 0;
+ visibility: hidden;
+}
+
+.slideover__content-area p,
+.slideover__content-area div {
+ padding-block-start: 0.5em;
+}
+
+.slideover__content-area > *:first-child {
+ margin-top: 0;
+ padding-top: 0;
+ padding-block-start: 0;
+}
+
+.slideover__content-area p {
+ margin: 1em 0;
+}
+
+.slideover__content-area ul,
+.slideover__content-area ol {
+ margin: 1em 0;
+ padding-left: 2em;
+}
+
+/* Right-aligned slideover effects */
+.slideover--r.slideover__content {
+ top: 50%;
+ right: 0;
+ left: auto;
+ height: auto;
+ max-height: 80vh;
+ width: var(--slideover-default-lr);
+ max-width: 100vw;
+ box-sizing: border-box;
+ transform: translateY(-50%) translateX(calc(100% - 60px));
+ transition: transform 0.3s cubic-bezier(.4,0,.2,1);
+ display: flex;
+ flex-direction: column;
+ padding: 20px;
+ border-radius: 8px 0 0 8px;
+ overflow: hidden;
+}
+
+.slideover--r .slideover__content-area {
+ flex: 1;
+ overflow-y: auto;
+ margin-top: -20px;
+ margin-bottom: -20px;
+ padding: 40px 1.5rem 1.5rem;
+ margin-top: -20px;
+ margin-bottom: -20px;
+}
+
+.slideover--r.slideover__content.slideover__content--active {
+ transform: translateY(-50%) translateX(0);
+}
+
+.slideover--r .slideover__header {
+ margin: -20px 0;
+}
+
+/* Left-aligned slideover effects */
+.slideover--l.slideover__content {
+ top: 50%;
+ left: 0;
+ right: auto;
+ height: auto;
+ max-height: 80vh;
+ width: var(--slideover-default-lr);
+ max-width: 100vw;
+ box-sizing: border-box;
+ transform: translateY(-50%) translateX(calc(-100% + 60px));
+ transition: transform 0.3s cubic-bezier(.4,0,.2,1);
+ display: flex;
+ flex-direction: column;
+ padding: 20px;
+ border-radius: 0 8px 8px 0;
+ overflow: hidden;
+}
+
+.slideover--l .slideover__content-area {
+ flex: 1;
+ overflow-y: auto;
+ margin-top: -20px;
+ margin-bottom: -20px;
+ padding: 40px 1.5rem 1.5rem;
+}
+
+.slideover--l.slideover__content.slideover__content--active {
+ transform: translateY(-50%) translateX(0);
+}
+
+
+.slideover--l .slideover__header {
+ margin: -20px 0;
+}
+
+/* Bottom-aligned slideover effects */
+.slideover--b.slideover__content {
+ bottom: 0;
+ left: 50%;
+ transform: translateX(-50%) translateY(calc(100% - 70px));
+ top: auto;
+ width: var(--slideover-default-tb);
+ height: auto;
+ max-height: 40vh;
+ border-radius: 8px 8px 0 0;
+ box-shadow: 0 -2px 4px rgba(8, 62, 68, 0.1), -2px 0 4px rgba(8, 62, 68, 0.1), 2px 0 4px rgba(8, 62, 68, 0.1);
+ padding: 0 20px;
+ text-align: left;
+ overflow: hidden;
+ padding-bottom: 20px;
+}
+
+.slideover--b .slideover__content-area {
+ margin-left: -20px;
+ margin-right: -20px;
+ padding-left: 3rem;
+ padding-right: 3rem;
+ text-align: left;
+}
+
+.slideover--b.slideover__content.slideover__content--active {
+ transform: translateX(-50%) translateY(0);
+}
+
+.slideover--b .slideover__header {
+ justify-content: flex-end;
+ padding-right: 16px;
+ margin: 0 -20px;
+ border-radius: 8px 8px 0 0;
+ background: transparent;
+}
+
+/* Top-aligned slideover effects */
+.slideover--t.slideover__content {
+ top: 0;
+ left: 50%;
+ transform: translateX(-50%) translateY(calc(-100% + 70px));
+ bottom: auto;
+ width: var(--slideover-default-tb);
+ height: auto;
+ max-height: 40vh;
+ border-radius: 0 0 8px 8px;
+ box-shadow: 0 2px 4px rgba(8, 62, 68, 0.1), -2px 0 4px rgba(8, 62, 68, 0.1), 2px 0 4px rgba(8, 62, 68, 0.1);
+ padding: 20px;
+ text-align: left;
+ overflow: hidden;
+}
+
+.slideover--t .slideover__content-area {
+ margin-left: -20px;
+ margin-right: -20px;
+ padding-left: 3rem;
+ padding-right: 3rem;
+ padding-top: 0px;
+ text-align: left;
+}
+
+.slideover--t.slideover__content.slideover__content--active {
+ transform: translateX(-50%) translateY(0);
+}
+
+.slideover--t .slideover__header {
+ justify-content: flex-end;
+ padding-right: 16px;
+ margin: 0 -20px;
+ border-radius: 0 0 8px 8px;
+ background: transparent;
+}
+
+/* Toggle-arrow styling */
+.slideover__toggle {
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ width: 24px;
+ height: 24px;
+ transition: transform 0.3s ease;
+ cursor: pointer;
+ position: absolute;
+ top: 8px;
+ right: 8px;
+ z-index: 10;
+}
+
+.slideover__toggle svg {
+ width: 24px;
+ height: 24px;
+ fill: currentColor;
+ stroke: currentColor;
+ stroke-width: 1px;
+}
+
+.slideover--r .slideover__toggle {
+ left: 8px;
+ right: auto;
+}
+
+.slideover--l .slideover__toggle {
+ right: 8px;
+ left: auto;
+}
+
+.slideover--t .slideover__toggle {
+ top: auto;
+ bottom: 8px;
+ right: 8px;
+ left: auto;
+}
+
+/* Toggle rotation logic: default (open), --active for closed */
+.slideover__content.slideover--r .slideover__toggle {
+ transform: rotate(180deg); /* Open: left arrow */
+}
+.slideover__content.slideover--r .slideover__toggle--active {
+ transform: rotate(0deg); /* Closed: right arrow */
+}
+
+.slideover__content.slideover--l .slideover__toggle {
+ transform: rotate(0deg); /* Open: right arrow */
+}
+.slideover__content.slideover--l .slideover__toggle--active {
+ transform: rotate(180deg); /* Closed: left arrow */
+}
+
+.slideover__content.slideover--t .slideover__toggle {
+ transform: rotate(0deg); /* Open: down arrow */
+}
+.slideover__content.slideover--t .slideover__toggle--active {
+ transform: rotate(180deg); /* Closed: up arrow */
+}
+
+.slideover__content.slideover--b .slideover__toggle {
+ transform: rotate(180deg); /* Open: up arrow */
+}
+.slideover__content.slideover--b .slideover__toggle--active {
+ transform: rotate(0deg); /* Closed: down arrow */
+}
+
+
+/* Mobile widths */
+@media screen and (max-width: 800px) {
+ .slideover--r.slideover__content,
+ .slideover--l.slideover__content {
+ max-height: 90vh;
+ width: 90vw;
+ }
+ .slideover--b.slideover__content,
+ .slideover--t.slideover__content {
+ width: 90%;
+ max-height: 60vh;
+ }
+}
+
+/* Other elements */
+
+/* Callout embed */
+.slideover__content-area .embed {
+ margin-left: 25px;
+ padding-left: 20px;
+ padding-top: 0px;
+ font-size: 0.9em;
+ border-radius: 5px;
+ padding-right: 1rem !important;
+}
+
+/* Quotations */
+q {
+ font-style: italic;
+ padding-left: 1em;
+ margin: 0.5em 0;
+ display: inline-block;
+}
+
+.slideover__content-area blockquote {
+ margin: 1em 0;
+ padding-left: 1em;
+}
+
+/* Emphasis */
+.slideover__content-area strong,
+.slideover__content-area b {
+ font-weight: 700;
+}
+
+.slideover__content-area em {
+ font-style: italic;
+}
+
+/* Code blocks */
+.slideover__content-area div.sourceCode {
+ font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+ font-size: 0.9em;
+ padding: 0.4em 0.6em;
+}
+
+.slideover__content-area code {
+ font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+ font-size: 0.9em;
+ padding: 0.1em 0.2em;
+ border-radius: 5px;
+}
+
+.slideover__content-area pre.numberSource {
+ margin-left: 1em;
+ border-left: none;
+ padding-left: 4px;
+}
+
diff --git a/site/_extensions/nrichers/slideover/slideover.js b/site/_extensions/nrichers/slideover/slideover.js
new file mode 100644
index 0000000000..6126c42515
--- /dev/null
+++ b/site/_extensions/nrichers/slideover/slideover.js
@@ -0,0 +1,129 @@
+window.slideover = {};
+
+var Plugin = {
+ id: 'slideover',
+ init: function(reveal) {
+ var holders = { presentation: undefined, slideover: undefined };
+
+ function loadStylesheet() {
+ var path = undefined;
+ [].slice.call(document.getElementsByTagName('script')).forEach(function(script) {
+ if (script.src.indexOf('slideover.js') > -1) {
+ path = script.src.split('/').slice(0, -1).join('/') + '/';
+ }
+ });
+ var link = document.createElement('link');
+ link.rel = 'stylesheet';
+ link.href = path + 'slideover.css';
+ document.head.appendChild(link);
+ }
+
+ function setup() {
+ loadStylesheet();
+ holders.presentation = document.querySelector('.reveal');
+ if (!holders.presentation) return false;
+ holders.presentation.classList.add('slideover__presentation');
+
+ var slideover = document.createElement('div');
+ slideover.classList.add('slideover__container');
+ holders.presentation.parentNode.insertBefore(slideover, holders.presentation.nextElementSibling);
+ holders.slideover = slideover;
+ return true;
+ }
+
+ function processMarkdownContent(content) {
+ const tempDiv = document.createElement('div');
+ tempDiv.innerHTML = content;
+ return tempDiv.innerHTML;
+ }
+
+ function handleOverlays() {
+ if (!holders.slideover) return;
+ holders.slideover.innerHTML = '';
+ const currentSlide = reveal.getCurrentSlide();
+ if (!currentSlide) return;
+ const overlays = currentSlide.querySelectorAll('.slideover--r, .slideover--b, .slideover--l, .slideover--t');
+
+ overlays.forEach((overlay) => {
+ const originalContent = overlay.innerHTML;
+ const content = document.createElement('div');
+ content.classList.add('slideover__content');
+ ['slideover--b', 'slideover--r', 'slideover--l', 'slideover--t'].forEach(dir => {
+ if (overlay.classList.contains(dir)) content.classList.add(dir);
+ });
+
+ // Add any additional modifier classes
+ overlay.classList.forEach(cls => {
+ if (!['slideover--r', 'slideover--b', 'slideover--l', 'slideover--t'].includes(cls)) {
+ content.classList.add(cls);
+ }
+ });
+
+ const header = document.createElement('div');
+ header.classList.add('slideover__header');
+ const toggle = document.createElement('div');
+ toggle.classList.add('slideover__toggle');
+ toggle.innerHTML = (overlay.classList.contains('slideover--r') || overlay.classList.contains('slideover--l'))
+ ? '<svg viewBox="0 0 24 24"><path fill="currentColor" d="M4.84 7.41L9.42 12l-4.58 4.59L6.25 18l6-6-6-6z M15.84 7.41L20.42 12l-4.58 4.59L17.25 18l6-6-6-6z"/></svg>'
+ : '<svg viewBox="0 0 24 24"><path fill="currentColor" d="M7.41 4.84L12 9.42l4.59-4.58L18 6.25l-6 6-6-6z M7.41 15.84L12 20.42l4.59-4.58L18 17.25l-6 6-6-6z"/></svg>';
+ content.appendChild(toggle);
+ content.appendChild(header);
+ const contentArea = document.createElement('div');
+ contentArea.classList.add('slideover__content-area');
+ contentArea.innerHTML = processMarkdownContent(originalContent);
+ content.appendChild(contentArea);
+ holders.slideover.appendChild(content);
+
+ // Start open, with toggle matching open state
+ content.classList.add('slideover__content--active');
+ toggle.classList.add('slideover__toggle--active');
+ content.dataset.slideoverState = 'open';
+
+ // Toggle click
+ toggle.addEventListener('click', (e) => {
+ e.stopPropagation();
+ const isActive = content.classList.toggle('slideover__content--active');
+ toggle.classList.toggle('slideover__toggle--active', isActive);
+ content.dataset.slideoverState = isActive ? 'open' : 'closed';
+ });
+
+ // Auto-collapse
+ if (overlay.classList.contains('auto-collapse') || [...overlay.classList].some(cls => cls.startsWith('auto-collapse-'))) {
+ let autoCollapseTimer;
+ let userExpanded = false;
+ const delay = [...overlay.classList].find(cls => cls.startsWith('auto-collapse-')) ?
+ parseInt([...overlay.classList].find(cls => cls.startsWith('auto-collapse-')).replace('auto-collapse-', '')) * 1000 : 5000;
+ function resetAutoCollapseTimer(element) {
+ clearTimeout(autoCollapseTimer);
+ autoCollapseTimer = setTimeout(() => {
+ if (element.classList.contains('slideover__content--active') && !userExpanded) {
+ element.classList.remove('slideover__content--active');
+ const toggle = element.querySelector('.slideover__toggle');
+ if (toggle) {
+ toggle.classList.remove('slideover__toggle--active');
+ element.dataset.slideoverState = 'closed';
+ }
+ }
+ }, delay);
+ }
+ resetAutoCollapseTimer(content);
+ content.addEventListener('mouseenter', () => resetAutoCollapseTimer(content));
+ content.addEventListener('click', () => resetAutoCollapseTimer(content));
+ content.addEventListener('keydown', () => resetAutoCollapseTimer(content));
+ toggle.addEventListener('click', () => { if (content.classList.contains('slideover__content--active')) userExpanded = true; });
+ }
+
+ overlay.style.display = 'none';
+ });
+ }
+
+ reveal.addEventListener('ready', () => setup() && handleOverlays());
+ reveal.addEventListener('slidechanged', handleOverlays);
+ reveal.addEventListener('fragmentshown', handleOverlays);
+ reveal.addEventListener('fragmenthidden', handleOverlays);
+ window.slideover = { setup, handleOverlays };
+ }
+};
+
+if (typeof window.Reveal === 'undefined') throw new Error('Reveal.js slideover plugin requires Reveal.js');
+Reveal.registerPlugin('slideover', Plugin);
\ No newline at end of file
diff --git a/site/_quarto.yml b/site/_quarto.yml
index 099b0fb147..37752a76fd 100644
--- a/site/_quarto.yml
+++ b/site/_quarto.yml
@@ -1,5 +1,6 @@
project:
type: website
+ post-render: make generate-sitemap
metadata-files:
- developer/_sidebar.yaml
@@ -46,8 +47,12 @@ website:
file: developer/validmind-library.qmd
- text: "{{< fa cubes >}} Supported Models"
file: developer/supported-models.qmd
- - text: "{{< fa rocket >}} QuickStart Notebook"
- file: notebooks/quickstart_customer_churn_full_suite.ipynb
+ - text: "---"
+ - text: "{{< fa rocket >}} QUICKSTART"
+ - text: "{{< fa file-pen >}} For Model Documentation"
+ file: notebooks/quickstart/quickstart_model_documentation.ipynb
+ - text: "{{< fa clipboard-check >}} For Model Validation"
+ file: notebooks/quickstart/quickstart_model_validation.ipynb
- text: "---"
- text: "{{< fa vial >}} TESTING"
- text: "{{< fa flask-vial >}} Run Tests & Test Suites"
diff --git a/site/developer/_sidebar.yaml b/site/developer/_sidebar.yaml
index 58a7b0a24e..53aea19bbe 100644
--- a/site/developer/_sidebar.yaml
+++ b/site/developer/_sidebar.yaml
@@ -8,8 +8,9 @@ website:
file: developer/validmind-library.qmd
- developer/supported-models.qmd
- text: "---"
- - text: "QuickStart"
- - notebooks/quickstart_customer_churn_full_suite.ipynb
+ - text: "Quickstart"
+ - notebooks/quickstart/quickstart_model_documentation.ipynb
+ - notebooks/quickstart/quickstart_model_validation.ipynb
- text: "Install and initialize ValidMind Library"
file: developer/model-documentation/install-and-initialize-validmind-library.qmd
- developer/model-documentation/store-credentials-in-env-file.qmd
diff --git a/site/developer/samples-jupyter-notebooks.qmd b/site/developer/samples-jupyter-notebooks.qmd
index f191027cf7..bdaf34543a 100644
--- a/site/developer/samples-jupyter-notebooks.qmd
+++ b/site/developer/samples-jupyter-notebooks.qmd
@@ -52,7 +52,7 @@ JupyterHub
: A a web-based platform when you can interact with Jupyter Notebook instances on a shared server. It is commonly used as a collaborative and interactive computing environment for data analysis, scientific research, and programming.
::: {.tc}
-[{{< fa code >}} Quickstart on JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/quickstart_customer_churn_full_suite.ipynb){.button target="_blank"}
+[{{< fa code >}} Quickstart on JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/quickstart/quickstart_model_documentation.ipynb){.button target="_blank"}
:::
diff --git a/site/developer/validmind-library.qmd b/site/developer/validmind-library.qmd
index 23e809b765..0614d6ea89 100644
--- a/site/developer/validmind-library.qmd
+++ b/site/developer/validmind-library.qmd
@@ -15,10 +15,11 @@ listing:
fields: [title, description]
grid-columns: 2
contents:
- - ../notebooks/quickstart_customer_churn_full_suite.ipynb
- - path: https://youtu.be/rIR8Mql7eGs
- title: "{{< fa brands youtube >}} {{< var vm.product >}} QuickStart"
- description: "Watch the walkthrough on YouTube: `https://youtu.be/rIR8Mql7eGs`"
+ - ../notebooks/quickstart/quickstart_model_documentation.ipynb
+ - ../notebooks/quickstart/quickstart_model_validation.ipynb
+ # - path: https://youtu.be/rIR8Mql7eGs
+ # title: "{{< fa brands youtube >}} {{< var vm.product >}} QuickStart"
+ # description: "Watch the walkthrough on YouTube: `https://youtu.be/rIR8Mql7eGs`"
- id: model-development
type: grid
grid-columns: 2
@@ -106,9 +107,9 @@ The {{< var validmind.developer >}} is designed to be model agnostic. If your mo
:::
-## QuickStart
+## Quickstart
-After you [**sign up**](/guide/configuration/accessing-validmind.qmd) for {{< var vm.product >}} to get access, try our QuickStart:
+After you [**sign up**](/guide/configuration/accessing-validmind.qmd) for {{< var vm.product >}} to get access, try our quickstarts for model documentation or validation:
:::{#library-quickstart}
:::
diff --git a/site/get-started/developer/try-in-your-own-environment.qmd b/site/get-started/developer/try-in-your-own-environment.qmd
index d852a35bc3..091de28004 100644
--- a/site/get-started/developer/try-in-your-own-environment.qmd
+++ b/site/get-started/developer/try-in-your-own-environment.qmd
@@ -41,7 +41,7 @@ Start by retrieving our notebook samples from GitHub, or download them from our
git clone https://github.com/validmind/validmind-library.git
```
-4. After the cloning process is complete, open `notebooks/code_samples/quickstart_customer_churn_full_suite.ipynb` in your developer environment and [run the notebook](#run-the-notebook).
+4. After the cloning process is complete, open `notebooks/quickstart/quickstart_model_documentation.ipynb` in your developer environment and [run the notebook](#run-the-notebook).
The notebook will guide you through installing the {{< var validmind.developer >}}, initializing the Python environment, and finally connecting the {{< var vm.developer >}} to the {{< var validmind.platform >}}.
@@ -55,7 +55,7 @@ Start by retrieving our notebook samples from GitHub, or download them from our
3. Navigate to the folder where you extracted the files.
-4. Open `notebooks/code_samples/quickstart_customer_churn_full_suite.ipynb` in your developer environment and [run the notebook](#run-the-notebook).
+4. Open `notebooks/quickstart/quickstart_model_documentation.ipynb` in your developer environment and [run the notebook](#run-the-notebook).
The notebook will guide you through installing the {{< var validmind.developer >}}, initializing the Python environment, and finally connecting the {{< var vm.developer >}} to the {{< var validmind.platform >}}.
diff --git a/site/get-started/developer/try-with-jupyterhub.qmd b/site/get-started/developer/try-with-jupyterhub.qmd
index 7ffcb4729d..78ce32bae8 100644
--- a/site/get-started/developer/try-with-jupyterhub.qmd
+++ b/site/get-started/developer/try-with-jupyterhub.qmd
@@ -22,7 +22,7 @@ Signing up is FREE — {{< var link.register >}}
## Steps
-1. In a web browser, open [Quickstart for model documentation]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/quickstart_customer_churn_full_suite.ipynb).
+1. In a web browser, open [Quickstart for model documentation]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/quickstart/quickstart_model_documentation.ipynb).
This link takes you to {{< var vm.product >}}'s JupyterHub instance where you can log in with the Auth0 credentials for your {{< var vm.product >}} account to access the **QuickStart for model documentation** notebook:
diff --git a/site/guide/model-validation/_add-model-findings.qmd b/site/guide/model-validation/_add-model-findings.qmd
index 9a3c8078f2..027496f15d 100644
--- a/site/guide/model-validation/_add-model-findings.qmd
+++ b/site/guide/model-validation/_add-model-findings.qmd
@@ -10,6 +10,8 @@
::::
:::: {.content-hidden unless-format="revealjs"}
+1. In the left sidebar, click **{{< fa cubes >}} Inventory**.
+
1. Select a model or [find your model by applying a filter or searching for it](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models){target="_blank"}.
1. In the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**.
diff --git a/site/guide/model-validation/_track_issue_resolution.qmd b/site/guide/model-validation/_track_issue_resolution.qmd
index a83858da5d..42f3a13c0d 100644
--- a/site/guide/model-validation/_track_issue_resolution.qmd
+++ b/site/guide/model-validation/_track_issue_resolution.qmd
@@ -41,6 +41,8 @@ Low-severity findings (Level 3)
::::
:::: {.content-hidden unless-format="revealjs"}
+1. In the left sidebar, click **{{< fa cubes >}} Inventory**.
+
1. Select a model or [find your model by applying a filter or searching for it](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models){target="_blank"}.
1. In the left sidebar that appears for your model, click **{{< fa triangle-exclamation >}} Model Findings**.
diff --git a/site/guide/model-workflows/_model-workflows-see.qmd b/site/guide/model-workflows/_model-workflows-see.qmd
index 8fc77b1d9a..f004ba04cd 100644
--- a/site/guide/model-workflows/_model-workflows-see.qmd
+++ b/site/guide/model-workflows/_model-workflows-see.qmd
@@ -25,6 +25,8 @@ While your lifecycle statuses and workflows are custom to your organization, som
::::
:::: {.content-hidden unless-format="revealjs"}
+1. In the left sidebar, click **{{< fa cubes >}} Inventory**.
+
1. Select a model or [find your model by applying a filter or searching for it](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models){target="_blank"}.
1. On the landing page of your model, locate the [model status]{.smallcaps} section:
diff --git a/site/guide/model-workflows/_model-workflows-transition.qmd b/site/guide/model-workflows/_model-workflows-transition.qmd
index dbefa12c90..4a8351f270 100644
--- a/site/guide/model-workflows/_model-workflows-transition.qmd
+++ b/site/guide/model-workflows/_model-workflows-transition.qmd
@@ -11,6 +11,8 @@
::::
:::: {.content-hidden unless-format="revealjs"}
+1. In the left sidebar, click **{{< fa cubes >}} Inventory**.
+
1. Select a model or [find your model by applying a filter or searching for it](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models){target="_blank"}.
1. If an action is available to your role, you'll see it listed under your model status on the model's landing page.
diff --git a/site/notebooks.zip b/site/notebooks.zip
index e37aae0467..661d08d0e4 100644
Binary files a/site/notebooks.zip and b/site/notebooks.zip differ
diff --git a/site/notebooks/EXECUTED/model_development/2-start_development_process.ipynb b/site/notebooks/EXECUTED/model_development/2-start_development_process.ipynb
index 939b907661..31bfd062e3 100644
--- a/site/notebooks/EXECUTED/model_development/2-start_development_process.ipynb
+++ b/site/notebooks/EXECUTED/model_development/2-start_development_process.ipynb
@@ -142,7 +142,7 @@
"In our below example, note that: \n",
"\n",
"- The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n",
- "- The ValidMind Library provides a wrapper to automatically load the dataset as a Pandas DataFrame object."
+ "- The ValidMind Library provides a wrapper to automatically load the dataset as a [Pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object. A Pandas Dataframe is a two-dimensional tabular data structure that makes use of rows and columns."
]
},
{
@@ -887,7 +887,7 @@
"\n",
"### Assign predictions\n",
"\n",
- "Once the model has been registered you can assign model predictions to the training and test datasets.\n",
+ "Once the model has been registered you can assign model predictions to the training and testing datasets.\n",
"\n",
"- The [`assign_predictions()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#assign_predictions) from the `Dataset` object can link existing predictions to any number of models.\n",
"- This method links the model's class prediction values and probabilities to our `vm_train_ds` and `vm_test_ds` datasets.\n",
@@ -959,7 +959,7 @@
"\n",
"- [x] Import a sample dataset\n",
"- [x] Identify which tests you might want to run with ValidMind\n",
- "- [x] Initialize ValidMind datasets\n",
+ "- [x] Initialize ValidMind datasets and model objects\n",
"- [x] Run individual tests\n",
"- [x] Utilize the output from tests you've run\n",
"- [x] Log test results from sets of or individual tests as evidence to the ValidMind Platform\n",
diff --git a/site/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.ipynb b/site/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.ipynb
index e38978aec3..58e4016929 100644
--- a/site/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.ipynb
+++ b/site/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.ipynb
@@ -896,7 +896,19 @@
"\n",
"### Work with your model documentation\n",
"\n",
- "Now that you've logged all your test results and generated a draft for your model documentation, head to the ValidMind Platform to make qualitative edits, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. **Learn more:** [Working with model documentation](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)"
+ "Now that you've logged all your test results and generated a draft for your model documentation, head to the ValidMind Platform to wrap up your model documentation. Continue to work on your model documentation by:\n",
+ "\n",
+ "- **Run and log more tests:** Use the skills you learned in this series of notebooks to run and log more individual tests, including custom tests, then insert them into your documentation as supplementary evidence. (Learn more: [`validmind.tests`](https://docs.validmind.ai/validmind/validmind/tests.html))\n",
+ "\n",
+ "- **Inserting additional test results:** Add **Test-Driven Blocks** under any relevant section of your model documentation. (Learn more: [Work with test results](https://docs.validmind.ai/guide/model-documentation/work-with-test-results.html))\n",
+ "\n",
+ "- **Making qualitative edits to your test descriptions:** Click on the description of any inserted test results to review and edit the ValidMind-generated test descriptions for quality and accuracy. (Learn more: [Working with model documentation](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html#add-or-edit-documentation))\n",
+ "\n",
+ "- **View guidelines:** In any section of your model documentation, click **ValidMind Insights** in the top right corner to reveal the Documentation Guidelines for each section to help guide the contents of your model documentation. (Learn more: [View documentation guidelines](https://docs.validmind.ai/guide/model-documentation/view-documentation-guidelines.html))\n",
+ "\n",
+ "- **Collaborate with other stakeholders:** Use the ValidMind Platform's real-time collaborative features to work seamlessly together with the rest of your organization, including model validators. Review suggested changes in your content blocks, work with versioned history, and use comments to discuss specific portions of your model documentation. (Learn more: [Collaborate with others](https://docs.validmind.ai/guide/model-documentation/collaborate-with-others.html))\n",
+ "\n",
+ "When your model documentation is complete and ready for review, submit it for approval from the same ValidMind Platform where you made your edits and collaborated with the rest of your organization, ensuring transparency and a thorough model development history. (Learn more: [Submit for approval](https://docs.validmind.ai/guide/model-documentation/submit-for-approval.html))"
]
},
{
diff --git a/site/notebooks/EXECUTED/model_validation/2-start_validation_process.ipynb b/site/notebooks/EXECUTED/model_validation/2-start_validation_process.ipynb
index 51bb9dc97b..747a5dd1e4 100644
--- a/site/notebooks/EXECUTED/model_validation/2-start_validation_process.ipynb
+++ b/site/notebooks/EXECUTED/model_validation/2-start_validation_process.ipynb
@@ -147,7 +147,7 @@
"In our below example, note that:\n",
"\n",
"- The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n",
- "- The ValidMind Library provides a wrapper to automatically load the dataset as a Pandas [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object."
+ "- The ValidMind Library provides a wrapper to automatically load the dataset as a [Pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object. A Pandas Dataframe is a two-dimensional tabular data structure that makes use of rows and columns."
]
},
{
@@ -564,7 +564,7 @@
"\n",
"## Documenting test results\n",
"\n",
- "Now that we've done some analysis on two different datasets, we can use ValidMind to easily document why certain things were done to our raw data with testing to support it. As we learned above, every test result returned by the `run_test()` function has a `.log()` method that can be used to send the test results to the ValidMind Platform.\n",
+ "Now that we've done some analysis on two different datasets, we can use ValidMind to easily document why certain things were done to our raw data with testing to support it. Every test result returned by the `run_test()` function has a `.log()` method that can be used to send the test results to the ValidMind Platform.\n",
"\n",
"When logging validation test results to the platform, you'll need to manually add those results to the desired section of the validation report. To demonstrate how to add test results to your validation report, we'll log our data quality tests and insert the results via the ValidMind Platform."
]
diff --git a/site/notebooks/EXECUTED/model_validation/3-developing_challenger_model.ipynb b/site/notebooks/EXECUTED/model_validation/3-developing_challenger_model.ipynb
index ba6958f286..83061a8273 100644
--- a/site/notebooks/EXECUTED/model_validation/3-developing_challenger_model.ipynb
+++ b/site/notebooks/EXECUTED/model_validation/3-developing_challenger_model.ipynb
@@ -520,7 +520,7 @@
"\n",
"## Running model evaluation tests\n",
"\n",
- "With everything ready for us, let's run the rest of our validation tests. We'll focus on comprehensive testing around model performance of both the champion and challenger models going forward as we've already verified the data quality of the datasets used to train the champion model."
+ "With our setup complete, let's run the rest of our validation tests. Since we have already verified the data quality of the dataset used to train our champion model, we will now focus on comprehensive performance evaluations of both the champion and challenger models."
]
},
{
@@ -584,7 +584,10 @@
"\n",
"#### Evaluate performance of the champion model\n",
"\n",
- "Now, let's run and log our batch of model performance tests using our testing dataset (`vm_test_ds`) for our champion model:"
+ "Now, let's run and log our batch of model performance tests using our testing dataset (`vm_test_ds`) for our champion model:\n",
+ "\n",
+ "- The test set serves as a proxy for real-world data, providing an unbiased estimate of model performance since it was not used during training or tuning.\n",
+ "- The test set also acts as protection against selection bias and model tweaking, giving a final, more unbiased checkpoint."
]
},
{
@@ -725,9 +728,9 @@
"\n",
"### Run diagnostic tests\n",
"\n",
- "Next we want to inspect the robustness and stability testing comparison between our champion and challenger model.\n",
+ "Next, we want to inspect the robustness and stability testing comparison between our champion and challenger model.\n",
"\n",
- "Use `list_tests()` to identify all the model diagnosis tests for classification:"
+ "Use `list_tests()` to list all available diagnosis tests applicable to classification tasks:"
]
},
{
@@ -743,9 +746,12 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Let's see if models suffer from any *overfit* potentials and also where there are potential sub-segments of issues with the [`OverfitDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/OverfitDiagnosis.html). \n",
+ "Let’s now assess the models for potential signs of *overfitting* and identify any sub-segments where performance may inconsistent with the [`OverfitDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/OverfitDiagnosis.html).\n",
+ "\n",
+ "Overfitting occurs when a model learns the training data too well, capturing not only the true pattern but noise and random fluctuations resulting in excellent performance on the training dataset but poor generalization to new, unseen data:\n",
"\n",
- "Overfitting occurs when a model learns the training data too well, capturing not only the true pattern but noise and random fluctuations resulting in excellent performance on the training dataset but poor generalization to new, unseen data."
+ "- Since the training dataset (`vm_train_ds`) was used to fit the model, we use this set to establish a baseline performance for how well the model performs on data it has already seen.\n",
+ "- The testing dataset (`vm_test_ds`) was never seen during training, and here simulates real-world generalization, or how well the model performs on new, unseen data. "
]
},
{
@@ -767,9 +773,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Let's also conduct *robustness* and *stability* testing of the two models with the [`RobustnessDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/RobustnessDiagnosis.html).\n",
+ "Let's also conduct *robustness* and *stability* testing of the two models with the [`RobustnessDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/RobustnessDiagnosis.html). Robustness refers to a model's ability to maintain consistent performance, and stability refers to a model's ability to produce consistent outputs over time across different data subsets.\n",
"\n",
- "Robustness refers to a model's ability to maintain consistent performance, and stability refers to a model's ability to produce consistent outputs over time across different data subsets."
+ "Again, we'll use both the training and testing datasets to establish baseline performance and to simulate real-world generalization:"
]
},
{
@@ -811,6 +817,13 @@
"FI"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We'll only use our testing dataset (`vm_test_ds`) here, to provide a realistic, unseen sample that mimic future or production data, as the training dataset has already influenced our model during learning:"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/site/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.ipynb b/site/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.ipynb
index 886f9e9061..ffdc4522a2 100644
--- a/site/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.ipynb
+++ b/site/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.ipynb
@@ -1144,13 +1144,17 @@
"\n",
"- **Inserting additional test results:** Click **Link Evidence to Report** under any section of 2. Validation in your validation report. (Learn more: [Link evidence to reports](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#link-evidence-to-reports))\n",
"\n",
- "- **Making qualitative edits to your test descriptions:** Expand any linked evidence under Validator Evidence and click **See evidence details** to review and edit the ValidMind-generated test descriptions for quality and accuracy.\n",
+ "- **Making qualitative edits to your test descriptions:** Expand any linked evidence under Validator Evidence and click **See evidence details** to review and edit the ValidMind-generated test descriptions for quality and accuracy. (Learn more: [Preparing validation reports](https://docs.validmind.ai/guide/model-validation/preparing-validation-reports.html#get-started))\n",
"\n",
"- **Adding more findings:** Click **Link Finding to Report** in any validation report section, then click **+ Create New Finding**. (Learn more: [Add and manage model findings](https://docs.validmind.ai/guide/model-validation/add-manage-model-findings.html))\n",
"\n",
- "- **Adding risk assessment notes:** Click under **Risk Assessment Notes** in any validation report section to access the text editor and content editing toolbar, including an option to generate a draft with AI. Edit your ValidMind-generated test descriptions (Learn more: [Work with content blocks](https://docs.validmind.ai/guide/model-documentation/work-with-content-blocks.html#content-editing-toolbar))\n",
+ "- **Adding risk assessment notes:** Click under **Risk Assessment Notes** in any validation report section to access the text editor and content editing toolbar, including an option to generate a draft with AI. Once generated, edit your ValidMind-generated test descriptions to adhere to your organization's requirements. (Learn more: [Work with content blocks](https://docs.validmind.ai/guide/model-documentation/work-with-content-blocks.html#content-editing-toolbar))\n",
"\n",
- "- **Assessing compliance:** Under the Guideline for any validation report section, click **ASSESSMENT** and select the compliance status from the drop-down menu. (Learn more: [Provide compliance assessments](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#provide-compliance-assessments))"
+ "- **Assessing compliance:** Under the Guideline for any validation report section, click **ASSESSMENT** and select the compliance status from the drop-down menu. (Learn more: [Provide compliance assessments](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#provide-compliance-assessments))\n",
+ "\n",
+ "- **Collaborate with other stakeholders:** Use the ValidMind Platform's real-time collaborative features to work seamlessly together with the rest of your organization, including model developers. Propose suggested changes in the model documentation, work with versioned history, and use comments to discuss specific portions of the model documentation. (Learn more: [Collaborate with others](https://docs.validmind.ai/guide/model-documentation/collaborate-with-others.html))\n",
+ "\n",
+ "When your validation report is complete and ready for review, submit it for approval from the same ValidMind Platform where you made your edits and collaborated with the rest of your organization, ensuring transparency and a thorough model validation history. (Learn more: [Submit for approval](https://docs.validmind.ai/guide/model-documentation/submit-for-approval.html))"
]
},
{
diff --git a/site/notebooks/README.md b/site/notebooks/README.md
index d4b8b9dbc6..a641f3e276 100644
--- a/site/notebooks/README.md
+++ b/site/notebooks/README.md
@@ -4,9 +4,10 @@ Our [Jupyter Notebook](https://jupyter.org/) code samples are designed to showca
Sample notebooks are organized into the following folders:
-* `notebooks/code_samples` — Showcase end-to-end functionality for documenting models
-* `notebooks/how_to` — Learn how to use specific ValidMind features, e.g. how to list all test suites
+* `notebooks/quickstart` — Quick guides to get you started with ValidMind
* `notebooks/tutorials` — Get step-by-step instructions and learn about ValidMind concepts in depth
+* `notebooks/how_to` — Learn how to use specific ValidMind features, for example how to list all test suites
+* `notebooks/code_samples` — Showcase end-to-end functionality for documenting or validating models
* `notebooks/code_sharing` — Share your own notebooks or document code internally
## Getting started
diff --git a/site/notebooks/code_samples/code_explainer/customer_churn_full_suite.py b/site/notebooks/code_samples/code_explainer/customer_churn_full_suite.py
new file mode 100644
index 0000000000..6bd9c2a8ab
--- /dev/null
+++ b/site/notebooks/code_samples/code_explainer/customer_churn_full_suite.py
@@ -0,0 +1,240 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""
+Quickstart for model documentation
+
+Welcome! Let's get you started with the basic process of documenting models with ValidMind.
+
+You will learn how to initialize the ValidMind Library, load a sample dataset to train a simple classification model,
+and then run a ValidMind test suite to quickly generate documentation about the data and model.
+
+This script uses the Bank Customer Churn Prediction sample dataset from Kaggle to train the classification model.
+"""
+
+# Import required libraries
+import validmind as vm
+import xgboost as xgb
+import logging
+import os
+import yaml
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Any
+from validmind.datasets.classification import customer_churn
+
+
+# Configure logging
+logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+ handlers=[
+ logging.FileHandler('model_development.log'),
+ logging.StreamHandler()
+ ]
+)
+logger = logging.getLogger(__name__)
+
+
+# Configuration management
+class ConfigManager:
+ def __init__(self, config_path: str = "config.yaml"):
+ self.config_path = config_path
+ self.config = self._load_config()
+
+ def _load_config(self) -> Dict[str, Any]:
+ """Load configuration from YAML file"""
+ if os.path.exists(self.config_path):
+ with open(self.config_path, 'r') as f:
+ return yaml.safe_load(f)
+ return {}
+
+ def save_config(self, config: Dict[str, Any]):
+ """Save configuration to YAML file"""
+ with open(self.config_path, 'w') as f:
+ yaml.dump(config, f)
+ self.config = config
+
+
+# Initialize configuration
+config_manager = ConfigManager()
+model_config = {
+ "model_name": "customer_churn_xgboost",
+ "version": "1.0.0",
+ "parameters": {
+ "early_stopping_rounds": 10,
+ "eval_metric": ["error", "logloss", "auc"]
+ },
+ "security": {
+ "api_key_rotation_days": 30,
+ "data_encryption": True
+ }
+}
+config_manager.save_config(model_config)
+
+# Initialize the ValidMind Library
+vm.init(
+ api_host="...",
+ api_key="...",
+ api_secret="...",
+ model="...",
+)
+
+# Preview the documentation template
+vm.preview_template()
+
+# Load the sample dataset
+logger.info("Loading demo dataset")
+print(
+ f"Loaded demo dataset with: \n\n\t• Target column: '{customer_churn.target_column}' \n\t• Class labels: {customer_churn.class_labels}"
+)
+
+raw_df = customer_churn.load_data()
+print("\nFirst few rows of the dataset:")
+print(raw_df.head())
+
+# Preprocess the raw dataset
+logger.info("Preprocessing dataset")
+train_df, validation_df, test_df = customer_churn.preprocess(raw_df)
+
+x_train = train_df.drop(customer_churn.target_column, axis=1)
+y_train = train_df[customer_churn.target_column]
+x_val = validation_df.drop(customer_churn.target_column, axis=1)
+y_val = validation_df[customer_churn.target_column]
+x_test = test_df.drop(customer_churn.target_column, axis=1)
+y_test = test_df[customer_churn.target_column]
+
+# Initialize and train the XGBoost model
+logger.info("Training XGBoost model")
+model = xgb.XGBClassifier(early_stopping_rounds=10)
+model.set_params(
+ eval_metric=["error", "logloss", "auc"],
+)
+model.fit(
+ x_train,
+ y_train,
+ eval_set=[(x_val, y_val)],
+ verbose=False,
+)
+
+
+# Model versioning and artifact management
+class ModelVersioning:
+ def __init__(self, model_dir: str = "model_artifacts"):
+ self.model_dir = Path(model_dir)
+ self.model_dir.mkdir(exist_ok=True)
+
+ def save_model(self, model: Any, version: str):
+ """Save model artifacts with versioning"""
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ model_path = self.model_dir / f"model_v{version}_{timestamp}.json"
+ model.save_model(str(model_path))
+ logger.info(f"Model saved to {model_path}")
+
+ def load_model(self, version: str) -> Any:
+ """Load model artifacts by version"""
+ model_files = list(self.model_dir.glob(f"model_v{version}_*.json"))
+ if not model_files:
+ raise ValueError(f"No model found for version {version}")
+ latest_model = max(model_files, key=lambda x: x.stat().st_mtime)
+ model = xgb.XGBClassifier()
+ model.load_model(str(latest_model))
+ return model
+
+
+# Initialize model versioning
+model_versioning = ModelVersioning()
+model_versioning.save_model(model, model_config["version"])
+
+# Initialize ValidMind datasets
+vm_raw_dataset = vm.init_dataset(
+ dataset=raw_df,
+ input_id="raw_dataset",
+ target_column=customer_churn.target_column,
+ class_labels=customer_churn.class_labels,
+)
+
+vm_train_ds = vm.init_dataset(
+ dataset=train_df,
+ input_id="train_dataset",
+ target_column=customer_churn.target_column,
+)
+
+vm_test_ds = vm.init_dataset(
+ dataset=test_df,
+ input_id="test_dataset",
+ target_column=customer_churn.target_column
+)
+
+# Initialize ValidMind model
+vm_model = vm.init_model(
+ model,
+ input_id="model",
+)
+
+# Assign predictions to the datasets
+vm_train_ds.assign_predictions(
+ model=vm_model,
+)
+
+vm_test_ds.assign_predictions(
+ model=vm_model,
+)
+
+# Define test configuration
+test_config = customer_churn.get_demo_test_config()
+
+# Run the full suite of tests
+logger.info("Running documentation tests")
+full_suite = vm.run_documentation_tests(config=test_config)
+
+
+# Model inference and scoring
+class ModelInference:
+ def __init__(self, model: Any):
+ self.model = model
+
+ def predict(self, data: Any) -> Any:
+ """Make predictions on new data"""
+ return self.model.predict(data)
+
+ def predict_proba(self, data: Any) -> Any:
+ """Get prediction probabilities"""
+ return self.model.predict_proba(data)
+
+ def score(self, data: Any, target: Any) -> float:
+ """Calculate model score"""
+ return self.model.score(data, target)
+
+
+# Initialize inference
+inference = ModelInference(model)
+
+
+# Example usage
+def example_usage():
+ """Example usage of the model"""
+ # Load new data
+ new_data = x_test[:5] # Example with first 5 test samples
+
+ # Make predictions
+ predictions = inference.predict(new_data)
+ probabilities = inference.predict_proba(new_data)
+
+ # Calculate score
+ score = inference.score(x_test, y_test)
+
+ logger.info(f"Model score: {score:.4f}")
+ logger.info(f"Predictions: {predictions}")
+ logger.info(f"Probabilities: {probabilities}")
+
+
+# Run example
+example_usage()
+
+# Note: After running this script, you can view the results in the ValidMind Platform
+# by going to the Model Inventory and selecting your model.
+
+if __name__ == "__main__":
+ print("\nScript execution completed. Check the ValidMind Platform for results.")
diff --git a/site/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb b/site/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb
new file mode 100644
index 0000000000..b0c874d917
--- /dev/null
+++ b/site/notebooks/code_samples/code_explainer/quickstart_code_explainer_demo.ipynb
@@ -0,0 +1,747 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Quickstart for model code documentation\n",
+ "\n",
+ "Welcome! This notebook demonstrates how to use the ValidMind code explainer to automatically generate comprehensive documentation for your codebase. The code explainer analyzes your source code and provides detailed explanations across various aspects of your implementation.\n",
+ "\n",
+ "## About Code Explainer\n",
+ "The ValidMind code explainer is a powerful tool that automatically analyzes your source code and generates comprehensive documentation. It helps you:\n",
+ "\n",
+ "- Understand the structure and organization of your codebase\n",
+ "- Document dependencies and environment setup\n",
+ "- Explain data processing and model implementation details\n",
+ "- Document training, evaluation, and inference pipelines\n",
+ "- Track configuration, testing, and security measures\n",
+ "\n",
+ "This tool is particularly useful for:\n",
+ "- Onboarding new team members\n",
+ "- Maintaining up-to-date documentation\n",
+ "- Ensuring code quality and best practices\n",
+ "- Facilitating code reviews and audits\n",
+ "\n",
+ "\n",
+ "## Contents\n",
+ "- [About ValidMind](#toc1_) \n",
+ " - [Before you begin](#toc1_1_) \n",
+ " - [New to ValidMind?](#toc1_2_) \n",
+ " - [Key concepts](#toc1_3_) \n",
+ "- [Install the ValidMind Library](#toc2_)\n",
+ "- [Initialize the client library](#toc3_) \n",
+ " - [Get your code snippet](#toc3_1_)\n",
+ "- [Preview the documentation template](#toc4_) \n",
+ "- [Code Analysis Sections](#sections)\n",
+ " - [Default Behavior](#defaultBehavior)\n",
+ " - [Codebase Overview](#overview)\n",
+ " - [Environment and Dependencies](#environment)\n",
+ " - [Data Handling](#data)\n",
+ " - [Model Implementation](#model)\n",
+ " - [Training Pipeline](#training)\n",
+ " - [Evaluation and Validation](#evaluation)\n",
+ " - [Inference and Scoring](#inference)\n",
+ " - [Configuration Management](#config)\n",
+ " - [Testing Strategy](#testing)\n",
+ " - [Logging and Monitoring](#logging)\n",
+ " - [Version Control](#versioning)\n",
+ " - [Security Measures](#security)\n",
+ " - [Usage Examples](#examples)\n",
+ " - [Known Issues and Improvements](#issues)\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "<a id='toc1_'></a>\n",
+ "\n",
+ "## About ValidMind\n",
+ "\n",
+ "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n",
+ "\n",
+ "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n",
+ "\n",
+ "<a id='toc1_1_'></a>\n",
+ "\n",
+ "### Before you begin\n",
+ "\n",
+ "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n",
+ "\n",
+ "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n",
+ "\n",
+ "<a id='toc1_2_'></a>\n",
+ "\n",
+ "### New to ValidMind?\n",
+ "\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
+ "\n",
+ "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\"><span style=\"color: #083E44;\"><b>For access to all features available in this notebook, create a free ValidMind account.</b></span>\n",
+ "<br></br>\n",
+ "Signing up is FREE — <a href=\"https://docs.validmind.ai/guide/configuration/register-with-validmind.html\" style=\"color: #DE257E;\"><b>Register with ValidMind</b></a></div>\n",
+ "\n",
+ "<a id='toc1_3_'></a>\n",
+ "\n",
+ "### Key concepts\n",
+ "\n",
+ "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n",
+ "\n",
+ "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n",
+ "\n",
+ "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n",
+ "\n",
+ "**Custom tests**: Custom tests are functions that you define to evaluate your model or dataset. These functions can be registered via the ValidMind Library to be used with the ValidMind Platform.\n",
+ "\n",
+ "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n",
+ "\n",
+ " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n",
+ " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n",
+ " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom test.\n",
+ " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom test. See this [example](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html) for more information.\n",
+ "\n",
+ "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a test, customize its behavior, or provide additional context.\n",
+ "\n",
+ "**Outputs**: Custom tests can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n",
+ "\n",
+ "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n",
+ "\n",
+ "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc2_'></a>\n",
+ "\n",
+ "## Install the ValidMind Library\n",
+ "\n",
+ "To install the library:\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip install -q validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_'></a>\n",
+ "\n",
+ "## Initialize the ValidMind Library\n",
+ "\n",
+ "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n",
+ "\n",
+ "<a id='toc3_1_'></a>\n",
+ "\n",
+ "### Get your code snippet\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Model Inventory** and click **+ Register Model**.\n",
+ "\n",
+ "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n",
+ "\n",
+ " For example, to register a model for use with this notebook, select:\n",
+ "\n",
+ " - Documentation template: `Model Source Code Documentation`\n",
+ "\n",
+ " You can fill in other options according to your preference.\n",
+ "\n",
+ "4. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
+ "\n",
+ "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load your model identifier credentials from an `.env` file\n",
+ "\n",
+ "%load_ext dotenv\n",
+ "%dotenv .env\n",
+ "\n",
+ "# Or replace with your code snippet\n",
+ "\n",
+ "import validmind as vm\n",
+ "\n",
+ "vm.init(\n",
+ " # api_host=\"...\",\n",
+ " # api_key=\"...\",\n",
+ " # api_secret=\"...\",\n",
+ " # model=\"...\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc4_'></a>\n",
+ "\n",
+ "## Preview the documentation template\n",
+ "\n",
+ "A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n",
+ "\n",
+ "You will upload documentation and test results into this template later on. For now, take a look at the structure that the template provides with the `vm.preview_template()` function from the ValidMind library and note the empty sections:\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.preview_template()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Common function\n",
+ "The code above defines two key functions:\n",
+ "1. A function to read source code from 'customer_churn_full_suite.py' file\n",
+ "2. An 'explain_code' function that uses ValidMind's experimental agents to analyze and explain code.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "source_code=\"\"\n",
+ "with open(\"customer_churn_full_suite.py\", \"r\") as f:\n",
+ " source_code = f.read()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The `vm.experimental.agents.run_task` function is used to execute AI agent tasks.\n",
+ "\n",
+ "It requires:\n",
+ "- task: The type of task to run (e.g. `code_explainer`)\n",
+ "- input: A dictionary containing task-specific parameters\n",
+ " - For `code_explainer`, this includes:\n",
+ " - **source_code** (str): The code to be analyzed\n",
+ " - **user_instructions** (str): Instructions for how to analyze the code"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def explain_code(content_id: str, user_instructions: str):\n",
+ " \"\"\"Run code explanation task and log the results.\n",
+ " By default, the code explainer includes sections for:\n",
+ " - Main Purpose and Overall Functionality\n",
+ " - Breakdown of Key Functions or Components\n",
+ " - Potential Risks or Failure Points \n",
+ " - Assumptions or Limitations\n",
+ " If you want default sections, specify user_instructions as an empty string.\n",
+ " \n",
+ " Args:\n",
+ " user_instructions (str): Instructions for how to analyze the code\n",
+ " content_id (str): ID to use when logging the results\n",
+ " \n",
+ " Returns:\n",
+ " The result object from running the code explanation task\n",
+ " \"\"\"\n",
+ " result = vm.experimental.agents.run_task(\n",
+ " task=\"code_explainer\",\n",
+ " input={\n",
+ " \"source_code\": source_code,\n",
+ " \"user_instructions\": user_instructions\n",
+ " }\n",
+ " )\n",
+ " result.log(content_id=content_id)\n",
+ " return result"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='defaultBehavior'></a>\n",
+ "\n",
+ "## 0. Default Behavior"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "By default, the code explainer includes sections for:\n",
+ "- Main Purpose and Overall Functionality\n",
+ "- Breakdown of Key Functions or Components\n",
+ "- Potential Risks or Failure Points \n",
+ "- Assumptions or Limitations\n",
+ "\n",
+ "If you want default sections, specify `user_instructions` as an empty string. For example:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = vm.experimental.agents.run_task(\n",
+ " task=\"code_explainer\",\n",
+ " input={\n",
+ " \"source_code\": source_code,\n",
+ " \"user_instructions\": \"\"\n",
+ " }\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='overview'></a>\n",
+ "\n",
+ "## 1. Codebase Overview\n",
+ "\n",
+ "Let's analyze your codebase structure to understand the main modules, components, entry points and their relationships. We'll also examine the technology stack and frameworks that are being utilized in the implementation."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Describe the overall structure of the source code repository.\n",
+ " - Identify main modules, folders, and scripts.\n",
+ " - Highlight entry points for training, inference, and evaluation.\n",
+ " - State the main programming languages and frameworks used.\n",
+ " \"\"\",\n",
+ " content_id=\"code_structure_summary\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\",\n",
+ " content_id=\"code_structure_summary\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='environment'></a>\n",
+ "## 2. Environment and Dependencies ('environment_setup')\n",
+ "Let's document the technical requirements and setup needed to run your code, including Python packages, system dependencies, and environment configuration files. Understanding these requirements is essential for proper development environment setup and consistent deployments across different environments."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - List Python packages and system dependencies (OS, compilers, etc.).\n",
+ " - Reference environment files (requirements.txt, environment.yml, Dockerfile).\n",
+ " - Include setup instructions using Conda, virtualenv, or containers.\n",
+ " Please remove Potential Risks or Failure Points and Assumptions or Limitations sections. Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"setup_instructions\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='data'></a>\n",
+ "## 3. Data Ingestion and Preprocessing\n",
+ "Let's document how your code handles data, including data sources, validation procedures, and preprocessing steps. We'll examine the data pipeline architecture, covering everything from initial data loading through feature engineering and quality checks."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Specify data input formats and sources.\n",
+ " - Document ingestion, validation, and transformation logic.\n",
+ " - Explain how raw data is preprocessed and features are generated.\n",
+ " Please remove Potential Risks or Failure Points and Assumptions or Limitations sections. Please don't add any other sections. \"\"\",\n",
+ " content_id=\"data_handling_notes\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='model'> </a>\n",
+ "## 4. Model Implementation Details\n",
+ "Let's document the core implementation details of your model, including its architecture, components, and key algorithms. Understanding the technical implementation is crucial for maintenance, debugging, and future improvements to the codebase. We'll examine how theoretical concepts are translated into working code."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Describe the core model code structure (classes, functions).\n",
+ " - Link code to theoretical models or equations when applicable.\n",
+ " - Note custom components like loss functions or feature selectors.\n",
+ " Please remove Potential Risks or Failure Points and Assumptions or Limitations sections. Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"model_code_description\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='training'></a>\n",
+ "## 5. Model Training Pipeline\n",
+ "\n",
+ "Let's document the training pipeline implementation, including how models are trained, optimized and evaluated. We'll examine the training process workflow, hyperparameter tuning approach, and model checkpointing mechanisms. This section provides insights into how the model learns from data and achieves optimal performance.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Explain the training process, optimization strategy, and hyperparameters.\n",
+ " - Describe logging, checkpointing, and early stopping mechanisms.\n",
+ " - Include references to training config files or tuning logic.\n",
+ " Please remove Potential Risks or Failure Points and Assumptions or Limitations sections. Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"training_logic_details\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='evaluation'></a>\n",
+ "\n",
+ "## 6. Evaluation and Validation Code\n",
+ "Let's examine how the model's validation and evaluation code is implemented, including the metrics calculation and validation processes. We'll explore the diagnostic tools and visualization methods used to assess model performance. This section will also cover how validation results are logged and stored for future reference.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Describe how validation is implemented and metrics are calculated.\n",
+ " - Include plots and diagnostic tools (e.g., ROC, SHAP, confusion matrix).\n",
+ " - State how outputs are logged and persisted.\n",
+ " Please remove Potential Risks or Failure Points and Assumptions or Limitations sections. Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"evaluation_logic_notes\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='inference'></a>\n",
+ "\n",
+ "## 7. Inference and Scoring Logic\n",
+ "Let's examine how the model performs inference and scoring on new data. This section will cover the implementation details of loading trained models, making predictions, and any required pre/post-processing steps. We'll also look at the APIs and interfaces available for both real-time serving and batch scoring scenarios.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Detail how the trained model is loaded and used for predictions.\n",
+ " - Explain I/O formats and APIs for serving or batch scoring.\n",
+ " - Include any preprocessing/postprocessing logic required.\n",
+ " Please remove Potential Risks or Failure Points and Assumptions or Limitations sections. Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"inference_mechanism\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='config'></a>\n",
+ "\n",
+ "\n",
+ "## 8. Configuration and Parameters\n",
+ "Let's explore how configuration and parameters are managed in the codebase. We'll examine the configuration files, command-line arguments, environment variables, and other mechanisms used to control model behavior. This section will also cover parameter versioning and how different configurations are tracked across model iterations.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Describe configuration management (files, CLI args, env vars).\n",
+ " - Highlight default parameters and override mechanisms.\n",
+ " - Reference versioning practices for config files.\n",
+ " Please remove Potential Risks or Failure Points and Assumptions or Limitations sections. Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"config_control_notes\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='testing'></a>\n",
+ "\n",
+ "## 9. Unit and Integration Testing\n",
+ "Let's examine the testing strategy and implementation in the codebase. We'll analyze the unit tests, integration tests, and testing frameworks used to ensure code quality and reliability. This section will also cover test coverage metrics and continuous integration practices.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - List unit and integration tests and what they cover.\n",
+ " - Mention testing frameworks and coverage tools used.\n",
+ " - Explain testing strategy for production-readiness.\n",
+ " Please remove Potential Risks or Failure Points and Assumptions or Limitations sections. Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"test_strategy_overview\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='logging'></a>\n",
+ "\n",
+ "## 10. Logging and Monitoring Hooks\n",
+ "Let's analyze how logging and monitoring are implemented in the codebase. We'll examine the logging configuration, monitoring hooks, and key metrics being tracked. This section will also cover any real-time observability integrations and alerting mechanisms in place.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Describe logging configuration and structure.\n",
+ " - Highlight real-time monitoring or observability integrations.\n",
+ " - List key events, metrics, or alerts tracked.\n",
+ " Please remove Potential Risks or Failure Points and Assumptions or Limitations sections. Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"logging_monitoring_notes\"\n",
+ ")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='versioning'></a>\n",
+ "\n",
+ "## 11. Code and Model Versioning\n",
+ "Let's examine how code and model versioning is managed in the codebase. This section will cover version control practices, including Git workflows and model artifact versioning tools like DVC or MLflow. We'll also look at how versioning integrates with the CI/CD pipeline.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Describe Git usage, branching, tagging, and commit standards.\n",
+ " - Include model artifact versioning practices (e.g., DVC, MLflow).\n",
+ " - Reference any automation in CI/CD.\n",
+ " Please remove the following sections: \n",
+ " - Potential Risks or Failure Points\n",
+ " - Assumptions or Limitations\n",
+ " - Breakdown of Key Functions or Components\n",
+ " Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"version_tracking_description\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='security'></a>\n",
+ "\n",
+ "## 12. Security and Access Control\n",
+ "Let's analyze the security and access control measures implemented in the codebase. We'll examine how sensitive data and code are protected through access controls, encryption, and compliance measures. Additionally, we'll review secure deployment practices and any specific handling of PII data.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Document access controls for source code and data.\n",
+ " - Include any encryption, PII handling, or compliance measures.\n",
+ " - Mention secure deployment practices.\n",
+ " Please remove the following sections: \n",
+ " - Potential Risks or Failure Points\n",
+ " - Assumptions or Limitations\n",
+ " - Breakdown of Key Functions or Components\n",
+ " Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"security_policies_notes\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='examples'></a>\n",
+ "\n",
+ "## 13. Example Runs and Scripts\n",
+ "Let's explore example runs and scripts that demonstrate how to use this codebase in practice. We'll look at working examples, command-line usage, and sample notebooks that showcase the core functionality. This section will also point to demo datasets and test scenarios that can help new users get started quickly.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - Provide working script examples.\n",
+ " - Include CLI usage instructions or sample notebooks.\n",
+ " - Link to demo datasets or test scenarios.\n",
+ " Please remove the following sections: \n",
+ " - Potential Risks or Failure Points\n",
+ " - Assumptions or Limitations\n",
+ " - Breakdown of Key Functions or Components\n",
+ " Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"runnable_examples\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='issues'></a>\n",
+ "\n",
+ "## 14. Known Issues and Future Improvements\n",
+ "Let's examine the current limitations and areas for improvement in the codebase. This section will document known technical debt, bugs, and feature gaps that need to be addressed. We'll also outline proposed enhancements and reference any existing tickets or GitHub issues tracking these improvements.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "result = explain_code(\n",
+ " user_instructions=\"\"\"\n",
+ " Please provide a summary of the following bullet points only.\n",
+ " - List current limitations or technical debt.\n",
+ " - Outline proposed enhancements or refactors.\n",
+ " - Reference relevant tickets, GitHub issues, or roadmap items.\n",
+ " Please remove Potential Risks or Failure Points and Assumptions or Limitations sections. Please don't add any other sections.\n",
+ " \"\"\",\n",
+ " content_id=\"issues_and_improvements_log\"\n",
+ ")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "validmind-1QuffXMV-py3.11",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.9"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/site/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb b/site/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb
index 963179129f..fdeb2d5661 100644
--- a/site/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb
+++ b/site/notebooks/code_samples/custom_tests/implement_custom_tests.ipynb
@@ -426,7 +426,7 @@
"source": [
"Just so we can run all of the tests in the template, let's initialize the train and raw dataset.\n",
"\n",
- "(see the `quickstart_customer_churn_full_suite.ipynb` notebook and the ValidMind docs for more information on what we are doing here)"
+ "(Refer to [**Quickstart for model documentation**](../../quickstart/quickstart_model_documentation.ipynb) and the ValidMind docs for more information on what we are doing here)"
]
},
{
diff --git a/site/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb b/site/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb
index 2ac83e3690..6dd0252946 100644
--- a/site/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb
+++ b/site/notebooks/code_samples/model_validation/validate_application_scorecard.ipynb
@@ -202,13 +202,19 @@
"\n",
"In order to log tests as a validator instead of as a developer, on the model details page that appears after you've successfully registered your sample model:\n",
"\n",
- "1. Remove yourself as a developer: \n",
+ "1. Remove yourself as a model owner: \n",
+ "\n",
+ " - Click on the **OWNERS** tile.\n",
+ " - Click the **x** next to your name to remove yourself from that model's role.\n",
+ " - Click **Save** to apply your changes to that role.\n",
+ "\n",
+ "2. Remove yourself as a developer: \n",
"\n",
" - Click on the **DEVELOPERS** tile.\n",
" - Click the **x** next to your name to remove yourself from that model's role.\n",
" - Click **Save** to apply your changes to that role.\n",
"\n",
- "2. Add yourself as a validator: \n",
+ "3. Add yourself as a validator: \n",
"\n",
" - Click on the **VALIDATORS** tile.\n",
" - Select your name from the drop-down menu.\n",
@@ -1358,9 +1364,9 @@
"\n",
"## Run diagnostic tests\n",
"\n",
- "Next we want to inspect the robustness and stability testing comparison between our champion and challenger model.\n",
+ "Next, we want to inspect the robustness and stability testing comparison between our champion and challenger model.\n",
"\n",
- "Use `list_tests()` to identify all the model diagnosis tests for classification:"
+ "Use `list_tests()` to list all available diagnosis tests applicable to classification tasks:"
]
},
{
diff --git a/site/notebooks/how_to/log_metrics_over_time.ipynb b/site/notebooks/how_to/log_metrics_over_time.ipynb
index bbf4b94cbc..a17d1658e1 100644
--- a/site/notebooks/how_to/log_metrics_over_time.ipynb
+++ b/site/notebooks/how_to/log_metrics_over_time.ipynb
@@ -39,7 +39,9 @@
" - [Log unit metrics over time](#toc6_2_) \n",
" - [Pass thresholds](#toc6_3_) \n",
" - [Log multiple metrics with custom thresholds](#toc6_4_) \n",
- "\n",
+ " - [Add acceptable performance flag](#toc6_5_)\n",
+ "- [Next steps](#toc7_)\n",
+ "- [Upgrade ValidMind](#toc8_)\n",
":::\n",
"<!-- jn-toc-notebook-config\n",
"\tnumbering=false\n",
@@ -197,7 +199,6 @@
" # api_key=\"...\",\n",
" # api_secret=\"...\",\n",
" # model=\"...\",\n",
- " monitoring = True\n",
")"
]
},
@@ -216,7 +217,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -300,7 +301,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
@@ -494,7 +495,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -542,7 +543,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -565,7 +566,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -694,13 +695,204 @@
"\n",
""
]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc6_5_'></a>\n",
+ "\n",
+ "### Add acceptable performance flag\n",
+ "\n",
+ "The `passed` parameter in the `log_metric()` function allows you to explicitly mark whether a specific metric value should be considered \"Satisfactory\" or \"Requires Attention\":\n",
+ " - When `passed=True`: A green \"Satisfactory\" badge appears on the chart, indicating the metric value meets your acceptance criteria.\n",
+ " - When `passed=False`: A yellow \"Requires Attention\" badge appears, highlighting potential concerns that may require investigation."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "In the example below, the `passed=True` parameter adds a green \"Satisfactory\" badge to the GINI Score metric visualization, instantly indicating that the 0.75 value meets acceptable performance standards by being above the `medium_risk` threshold of 0.6:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "log_metric(\n",
+ " key=\"GINI Score\",\n",
+ " value=0.75,\n",
+ " recorded_at=datetime(2025, 6, 7),\n",
+ " thresholds = {\n",
+ " \"high_risk\": 0.5,\n",
+ " \"medium_risk\": 0.6,\n",
+ " \"low_risk\": 0.8,\n",
+ " },\n",
+ " passed=True\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "In this example, the `passed=False` parameter adds a yellow \"Requires Attention\" badge to the GINI Score metric visualization, immediately highlighting that the value of 0.5 fails to meet acceptable performance standards by not exceeding the `medium_risk` threshold of 0.6:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "log_metric(\n",
+ " key=\"GINI Score\",\n",
+ " value=0.5,\n",
+ " recorded_at=datetime(2025, 6, 9),\n",
+ " thresholds = {\n",
+ " \"high_risk\": 0.5,\n",
+ " \"medium_risk\": 0.6,\n",
+ " \"low_risk\": 0.8,\n",
+ " },\n",
+ " passed=False\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Here, a custom function `passed_fn` determines the badge status automatically, displaying a green \"Satisfactory\" badge for the 0.65 GINI Score because it exceeds the `medium_risk` threshold of 0.6, enabling programmatic evaluation of metric performance based on predefined business rules:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "gini = 0.65\n",
+ "\n",
+ "thresholds = {\n",
+ " \"high_risk\": 0.5,\n",
+ " \"medium_risk\": 0.6,\n",
+ " \"low_risk\": 0.8,\n",
+ "}\n",
+ "\n",
+ "def passed_fn(value):\n",
+ " return value > thresholds[\"medium_risk\"]\n",
+ "\n",
+ "log_metric(\n",
+ " key=\"GINI Score\",\n",
+ " value=gini, \n",
+ " recorded_at=datetime(2025, 6, 10),\n",
+ " thresholds=thresholds,\n",
+ " passed=passed_fn(gini)\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc7_'></a>\n",
+ "\n",
+ "## Next steps\n",
+ "\n",
+ "You can look at the results of this test suite right in the notebook where you ran the code, as you would expect. But there is a better way — use the ValidMind Platform to work with your model documentation.\n",
+ "\n",
+ "<a id='toc9_1_'></a>\n",
+ "\n",
+ "### Work with your model documentation\n",
+ "\n",
+ "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n",
+ "\n",
+ "2. Click and expand the **Model Development** section.\n",
+ "\n",
+ "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n",
+ "\n",
+ "<a id='toc9_2_'></a>\n",
+ "\n",
+ "### Discover more learning resources\n",
+ "\n",
+ "We offer many interactive notebooks to help you document models:\n",
+ "\n",
+ "- [Run tests & test suites](https://docs.validmind.ai/developer/model-testing/testing-overview.html)\n",
+ "- [Code samples](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)\n",
+ "\n",
+ "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc8_'></a>\n",
+ "\n",
+ "## Upgrade ValidMind\n",
+ "\n",
+ "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\">After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.</div>\n",
+ "\n",
+ "Retrieve the information for the currently installed version of ValidMind:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip show validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n",
+ "\n",
+ "```bash\n",
+ "%pip install --upgrade validmind\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "You may need to restart your kernel after running the upgrade package for changes to be applied."
+ ]
}
],
"metadata": {
"kernelspec": {
- "display_name": "ValidMind Library",
+ "display_name": "validmind-eEL8LtKG-py3.10",
"language": "python",
- "name": "validmind"
+ "name": "python3"
},
"language_info": {
"codemirror_mode": {
@@ -712,7 +904,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.10.13"
+ "version": "3.10.15"
}
},
"nbformat": 4,
diff --git a/site/notebooks/images/log_metric_attention.png b/site/notebooks/images/log_metric_attention.png
new file mode 100644
index 0000000000..f4a87ebd35
Binary files /dev/null and b/site/notebooks/images/log_metric_attention.png differ
diff --git a/site/notebooks/images/log_metric_satisfactory.png b/site/notebooks/images/log_metric_satisfactory.png
new file mode 100644
index 0000000000..5340735c43
Binary files /dev/null and b/site/notebooks/images/log_metric_satisfactory.png differ
diff --git a/site/notebooks/images/log_metric_satisfactory_2.png b/site/notebooks/images/log_metric_satisfactory_2.png
new file mode 100644
index 0000000000..3990689f89
Binary files /dev/null and b/site/notebooks/images/log_metric_satisfactory_2.png differ
diff --git a/site/notebooks/quickstart/quickstart_model_documentation.ipynb b/site/notebooks/quickstart/quickstart_model_documentation.ipynb
new file mode 100644
index 0000000000..67b3f1816d
--- /dev/null
+++ b/site/notebooks/quickstart/quickstart_model_documentation.ipynb
@@ -0,0 +1,840 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "f2c17b2d",
+ "metadata": {},
+ "source": [
+ "# Quickstart for model documentation\n",
+ "\n",
+ "Learn the basics of using ValidMind to document models as part of a model development workflow. Set up the ValidMind Library in your environment, and generate a draft of documentation using ValidMind tests for a binary classification model.\n",
+ "\n",
+ "To document a model with the ValidMind Library, we'll:\n",
+ "\n",
+ "1. Import a sample dataset and preprocess it\n",
+ "2. Split the datasets and initialize them for use with ValidMind\n",
+ "3. Initialize a model object for use with testing\n",
+ "4. Run a full suite of tests as defined by our documentation template, which will send the results of those tests to the ValidMind Platform"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "05dce32a",
+ "metadata": {},
+ "source": [
+ "::: {.content-hidden when-format=\"html\"}\n",
+ "## Contents \n",
+ "- [Introduction](#toc1_) \n",
+ "- [About ValidMind](#toc2_) \n",
+ " - [Before you begin](#toc2_1_) \n",
+ " - [New to ValidMind?](#toc2_2_) \n",
+ " - [Key concepts](#toc2_3_) \n",
+ "- [Setting up](#toc3_) \n",
+ " - [Install the ValidMind Library](#toc3_1_) \n",
+ " - [Initialize the ValidMind Library](#toc3_2_) \n",
+ " - [Get your code snippet](#toc3_2_1_) \n",
+ " - [Initialize the Python environment](#toc3_3_) \n",
+ "- [Getting to know ValidMind](#toc4_) \n",
+ " - [Preview the documentation template](#toc4_1_) \n",
+ " - [View model documentation in the ValidMind Platform](#toc4_2_) \n",
+ "- [Import the sample dataset](#toc5_) \n",
+ "- [Preprocessing the raw dataset](#toc6_) \n",
+ " - [Split the dataset](#toc6_1_) \n",
+ " - [Separate features and targets](#toc6_2_) \n",
+ "- [Training an XGBoost classifier model](#toc7_) \n",
+ " - [Set evaluation metrics](#toc7_1_) \n",
+ " - [Fit the model](#toc7_2_) \n",
+ "- [Initialize the ValidMind datasets](#toc8_) \n",
+ "- [Initialize a model object](#toc9_) \n",
+ " - [Assign predictions](#toc9_1_) \n",
+ "- [Run the full suite of tests](#toc10_) \n",
+ "- [In summary](#toc11_) \n",
+ "- [Next steps](#toc12_) \n",
+ " - [Work with your model documentation](#toc12_1_) \n",
+ " - [Discover more learning resources](#toc12_2_) \n",
+ "- [Upgrade ValidMind](#toc13_) \n",
+ "\n",
+ ":::\n",
+ "<!-- jn-toc-notebook-config\n",
+ "\tnumbering=false\n",
+ "\tanchor=true\n",
+ "\tflat=false\n",
+ "\tminLevel=2\n",
+ "\tmaxLevel=4\n",
+ "\t/jn-toc-notebook-config -->\n",
+ "<!-- THIS CELL WILL BE REPLACED ON TOC UPDATE. DO NOT WRITE YOUR TEXT IN THIS CELL -->"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b1919918",
+ "metadata": {},
+ "source": [
+ "<a id='toc1_'></a>\n",
+ "\n",
+ "## Introduction\n",
+ "\n",
+ "Model development aims to produce a fit-for-purpose *champion model* by conducting thorough testing and analysis, supporting the capabilities of the model with evidence in the form of documentation and test results. Model documentation should be clear and comprehensive, ideally following a structure or template covering all aspects of compliance with model risk regulation.\n",
+ "\n",
+ "A *binary classification model* is a type of predictive model used in churn analysis to identify customers who are likely to leave a service or subscription by analyzing various behavioral, transactional, and demographic factors.\n",
+ "\n",
+ "- This model helps businesses take proactive measures to retain at-risk customers by offering personalized incentives, improving customer service, or adjusting pricing strategies.\n",
+ "- Effective validation of a churn prediction model ensures that businesses can accurately identify potential churners, optimize retention efforts, and enhance overall customer satisfaction while minimizing revenue loss."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c8f85783",
+ "metadata": {},
+ "source": [
+ "<a id='toc2_'></a>\n",
+ "\n",
+ "## About ValidMind\n",
+ "\n",
+ "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models. \n",
+ "\n",
+ "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "87fbc9d8",
+ "metadata": {},
+ "source": [
+ "<a id='toc2_1_'></a>\n",
+ "\n",
+ "### Before you begin\n",
+ "\n",
+ "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n",
+ "\n",
+ "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2f368277",
+ "metadata": {},
+ "source": [
+ "<a id='toc2_2_'></a>\n",
+ "\n",
+ "### New to ValidMind?\n",
+ "\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
+ "\n",
+ "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\"><span style=\"color: #083E44;\"><b>For access to all features available in this notebook, create a free ValidMind account.</b></span>\n",
+ "<br></br>\n",
+ "Signing up is FREE — <a href=\"https://docs.validmind.ai/guide/configuration/register-with-validmind.html\" style=\"color: #DE257E;\"><b>Register with ValidMind</b></a></div>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ec08fd19",
+ "metadata": {},
+ "source": [
+ "<a id='toc2_3_'></a>\n",
+ "\n",
+ "### Key concepts\n",
+ "\n",
+ "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n",
+ "\n",
+ "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n",
+ "\n",
+ "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n",
+ "\n",
+ "**Metrics**: A subset of tests that do not have thresholds. In the context of this notebook, metrics and tests can be thought of as interchangeable concepts.\n",
+ "\n",
+ "**Custom metrics**: Custom metrics are functions that you define to evaluate your model or dataset. These functions can be registered with the ValidMind Library to be used in the ValidMind Platform.\n",
+ "\n",
+ "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n",
+ "\n",
+ " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n",
+ " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n",
+ " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom metric.\n",
+ " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom metric. (Learn more: [Run tests with multiple datasets](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html))\n",
+ "\n",
+ "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a metric, customize its behavior, or provide additional context.\n",
+ "\n",
+ "**Outputs**: Custom metrics can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n",
+ "\n",
+ "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n",
+ "\n",
+ "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c8883927",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_'></a>\n",
+ "\n",
+ "## Setting up"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_1_'></a>\n",
+ "\n",
+ "### Install the ValidMind Library\n",
+ "\n",
+ "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\"><span style=\"color: #083E44;\"><b>Recommended Python versions</b></span>\n",
+ "<br></br>\n",
+ "Python 3.8 <= x <= 3.11</div>\n",
+ "\n",
+ "To install the library:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d1f6dbed",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip install -q validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "797eb7f0",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_2_'></a>\n",
+ "\n",
+ "### Initialize the ValidMind Library\n",
+ "\n",
+ "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_2_1_'></a>\n",
+ "\n",
+ "#### Get your code snippet\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n",
+ "\n",
+ "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n",
+ "\n",
+ " For example, to register a model for use with this notebook, select:"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "74d329e8",
+ "metadata": {},
+ "source": [
+ " - Documentation template: `Binary classification`\n",
+ " - Use case: `Marketing/Sales - Attrition/Churn Management`"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "install-credentials-4c404843-3474-4618-8898-f3bcce33fadd",
+ "metadata": {},
+ "source": [
+ " You can fill in other options according to your preference.\n",
+ " \n",
+ "4. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
+ "\n",
+ "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e2c1dd22",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load your model identifier credentials from an `.env` file\n",
+ "\n",
+ "%load_ext dotenv\n",
+ "%dotenv .env\n",
+ "\n",
+ "# Or replace with your code snippet\n",
+ "\n",
+ "import validmind as vm\n",
+ "\n",
+ "vm.init(\n",
+ " # api_host=\"...\",\n",
+ " # api_key=\"...\",\n",
+ " # api_secret=\"...\",\n",
+ " # model=\"...\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "44bf926b",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_3_'></a>\n",
+ "\n",
+ "### Initialize the Python environment\n",
+ "\n",
+ "Then, let's import the necessary libraries and set up your Python environment for data analysis:\n",
+ "\n",
+ "- Import **Extreme Gradient Boosting** (XGBoost) with an alias so that we can reference its functions in later calls. XGBoost is a powerful machine learning library designed for speed and performance, especially in handling structured or tabular data.\n",
+ "- Enable **`matplotlib`**, a plotting library used for visualizing data. Ensures that any plots you generate will render inline in our notebook output rather than opening in a separate window."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "62d7c2c1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import xgboost as xgb\n",
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "41474c53",
+ "metadata": {},
+ "source": [
+ "<a id='toc4_'></a>\n",
+ "\n",
+ "## Getting to know ValidMind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5d87ae6f",
+ "metadata": {},
+ "source": [
+ "<a id='toc4_1_'></a>\n",
+ "\n",
+ "### Preview the documentation template\n",
+ "\n",
+ "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for your model. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n",
+ "\n",
+ "You will upload documentation and test results unique to your model based on this template later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library and note the empty sections:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b2bce375",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.preview_template()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "aa3b815d",
+ "metadata": {},
+ "source": [
+ "<a id='toc4_2_'></a>\n",
+ "\n",
+ "### View model documentation in the ValidMind Platform\n",
+ "\n",
+ "Next, let's head to the ValidMind Platform to see the template in action:\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this notebook.\n",
+ "\n",
+ "3. Click on the **Documentation** for your model and note how the structure of the documentation matches our preview above."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bde500ce",
+ "metadata": {},
+ "source": [
+ "<a id='toc5_'></a>\n",
+ "\n",
+ "## Import the sample dataset\n",
+ "\n",
+ "First, let's import the public [Bank Customer Churn Prediction](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction) dataset from Kaggle so that we have something to work with.\n",
+ "\n",
+ "In our below example, note that: \n",
+ "\n",
+ "- The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n",
+ "- The ValidMind Library provides a wrapper to automatically load the dataset as a [Pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object. A Pandas Dataframe is a two-dimensional tabular data structure that makes use of rows and columns."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "58d1c94b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from validmind.datasets.classification import customer_churn\n",
+ "\n",
+ "print(\n",
+ " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{customer_churn.target_column}' \\n\\t• Class labels: {customer_churn.class_labels}\"\n",
+ ")\n",
+ "\n",
+ "raw_df = customer_churn.load_data()\n",
+ "raw_df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "52d0aaa8",
+ "metadata": {},
+ "source": [
+ "<a id='toc6_'></a>\n",
+ "\n",
+ "## Preprocessing the raw dataset\n",
+ "\n",
+ "Before running tests with Validmind, we'll need to preprocess our imported dataset. This involves splitting the data and separating the features (inputs) from the targets (outputs)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "83545763",
+ "metadata": {},
+ "source": [
+ "<a id='toc6_1_'></a>\n",
+ "\n",
+ "### Split the dataset\n",
+ "\n",
+ "Splitting our dataset helps assess how well the model generalizes to unseen data.\n",
+ "\n",
+ "Use [`preprocess()`](https://docs.validmind.ai/validmind/validmind/datasets/classification/customer_churn.html#preprocess) to split our dataset into three subsets:\n",
+ "\n",
+ "1. **train_df** — Used to train the model.\n",
+ "2. **validation_df** — Used to evaluate the model's performance during training.\n",
+ "3. **test_df** — Used later on to asses the model's performance on new, unseen data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "418cb5aa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f0ae3dc5",
+ "metadata": {},
+ "source": [
+ "<a id='toc6_2_'></a>\n",
+ "\n",
+ "### Separate features and targets\n",
+ "\n",
+ "To train the model, we need to provide it with:\n",
+ "\n",
+ "1. **Inputs** — Features such as customer age, usage, etc.\n",
+ "2. **Outputs (Expected answers/labels)** — in our case, we would like to know whether the customer churned or not.\n",
+ "\n",
+ "Here, we'll use `x_train` and `x_val` to hold the input data (features), and `y_train` and `y_val` to hold the answers (the target we want to predict):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6fd365fd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "x_train = train_df.drop(customer_churn.target_column, axis=1)\n",
+ "y_train = train_df[customer_churn.target_column]\n",
+ "x_val = validation_df.drop(customer_churn.target_column, axis=1)\n",
+ "y_val = validation_df[customer_churn.target_column]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc7_'></a>\n",
+ "\n",
+ "## Training an XGBoost classifier model\n",
+ "\n",
+ "Next, let's create an XGBoost classifier model that will automatically stop training if it doesn’t improve after 10 tries.\n",
+ "\n",
+ "Setting a threshold avoids wasting time and helps prevent overfitting by stopping training when further improvement isn’t happening."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3296cac6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = xgb.XGBClassifier(early_stopping_rounds=10)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "826d8adf",
+ "metadata": {},
+ "source": [
+ "<a id='toc7_1_'></a>\n",
+ "\n",
+ "### Set evaluation metrics\n",
+ "\n",
+ "Then, we'll set the evaluation metrics, which tells the model to use three different ways to measure its performance:\n",
+ "\n",
+ "1. **error** — Measures how often the model makes incorrect predictions.\n",
+ "2. **logloss** — Indicates how confident the predictions are.\n",
+ "3. **auc** — Evaluates how well the model distinguishes between churn and not churn.\n",
+ "\n",
+ "Using multiple metrics gives a more complete picture of how good (or bad) the model is."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "32d3c3f4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model.set_params(\n",
+ " eval_metric=[\"error\", \"logloss\", \"auc\"],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc7_2_'></a>\n",
+ "\n",
+ "### Fit the model\n",
+ "\n",
+ "Finally, our actual training step — where the model learns patterns from the data, so it can make predictions later:\n",
+ "\n",
+ "- The model is trained on `x_train` and `y_train`, and evaluates its performance using `x_val` and `y_val` to check if it’s learning well.\n",
+ "- To turn off printed output while training, we'll set `verbose` to `False`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3fb95ce4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model.fit(\n",
+ " x_train,\n",
+ " y_train,\n",
+ " eval_set=[(x_val, y_val)],\n",
+ " verbose=False,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc8_'></a>\n",
+ "\n",
+ "## Initialize the ValidMind datasets\n",
+ "\n",
+ "Before you can run tests with your preprocessed datasets, you must first initialize a ValidMind `Dataset` object using the [`init_dataset`](https://docs.validmind.ai/validmind/validmind.html#init_dataset) function from the ValidMind (`vm`) module. **This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind,** but you only need to do it once per dataset.\n",
+ "\n",
+ "For this example, we'll pass in the following arguments:\n",
+ "\n",
+ "- **`dataset`** — The raw dataset that you want to provide as input to tests.\n",
+ "- **`input_id`** — A unique identifier that allows tracking what inputs are used when running each individual test.\n",
+ "- **`target_column`** — A required argument if tests require access to true values. This is the name of the target column in the dataset.\n",
+ "- **`class_labels`** — An optional value to map predicted classes to class labels."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bb6ad06a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize the raw dataset\n",
+ "vm_raw_dataset = vm.init_dataset(\n",
+ " dataset=raw_df,\n",
+ " input_id=\"raw_dataset\",\n",
+ " target_column=customer_churn.target_column,\n",
+ " class_labels=customer_churn.class_labels,\n",
+ ")\n",
+ "\n",
+ "# Initialize the training dataset\n",
+ "vm_train_ds = vm.init_dataset(\n",
+ " dataset=train_df,\n",
+ " input_id=\"train_dataset\",\n",
+ " target_column=customer_churn.target_column,\n",
+ ")\n",
+ "\n",
+ "# Initialize the testing dataset\n",
+ "vm_test_ds = vm.init_dataset(\n",
+ " dataset=test_df,\n",
+ " input_id=\"test_dataset\",\n",
+ " target_column=customer_churn.target_column\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc9_'></a>\n",
+ "\n",
+ "## Initialize a model object\n",
+ "\n",
+ "You'll also need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data for our model.\n",
+ "\n",
+ "You simply initialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0e44eebd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_model = vm.init_model(\n",
+ " model,\n",
+ " input_id=\"model\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc9_1_'></a>\n",
+ "\n",
+ "### Assign predictions\n",
+ "\n",
+ "Once the model has been registered, you can assign model predictions to the training and testing datasets.\n",
+ "\n",
+ "- The [`assign_predictions()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#assign_predictions) from the `Dataset` object can link existing predictions to any number of models.\n",
+ "- This method links the model's class prediction values and probabilities to our `vm_train_ds` and `vm_test_ds` datasets.\n",
+ "\n",
+ "If no prediction values are passed, the method will compute predictions automatically:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "62bd94fc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_train_ds.assign_predictions(\n",
+ " model=vm_model,\n",
+ ")\n",
+ "\n",
+ "vm_test_ds.assign_predictions(\n",
+ " model=vm_model,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc10_'></a>\n",
+ "\n",
+ "## Run the full suite of tests\n",
+ "\n",
+ "This is where it all comes together — you are now ready to **run the documentation tests for the model as defined by the documentation template** you looked at earlier.\n",
+ "\n",
+ "The [`vm.run_documentation_tests`](https://docs.validmind.ai/validmind/validmind.html#run_documentation_tests) function finds and runs every test specified in the template and then uploads all the documentation and test artifacts that get generated to the ValidMind Platform:\n",
+ "\n",
+ "- The function requires information about the inputs to use on every test. These inputs can be passed as an `inputs` argument if we want to use the same inputs for all tests. \n",
+ "- It's also possible to pass a `config` argument that has information about the `params` and `inputs` that each test requires. The `config` parameter is a dictionary with the following structure:\n",
+ "\n",
+ " ```python\n",
+ " config = {\n",
+ " \"<test-id>\": {\n",
+ " \"params\": {\n",
+ " \"param1\": \"value1\",\n",
+ " \"param2\": \"value2\",\n",
+ " ...\n",
+ " },\n",
+ " \"inputs\": {\n",
+ " \"input1\": \"value1\",\n",
+ " \"input2\": \"value2\",\n",
+ " ...\n",
+ " }\n",
+ " },\n",
+ " ...\n",
+ " }\n",
+ " ```\n",
+ "\n",
+ " Each `<test-id>` above corresponds to the test driven block identifiers shown by `vm.preview_template()`. For this model, we will use the default parameters for all tests, but we'll need to specify the input configuration for each one. The method `get_demo_test_config()` below constructs the default input configuration for our demo."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b3d6741b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from validmind.utils import preview_test_config\n",
+ "\n",
+ "test_config = customer_churn.get_demo_test_config()\n",
+ "preview_test_config(test_config)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "70830a61",
+ "metadata": {},
+ "source": [
+ "Now we can pass the input configuration to `vm.run_documentation_tests()` and run the full suite of tests.\n",
+ "\n",
+ "The variable `full_suite` then holds the result of these tests:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ae3accf7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "full_suite = vm.run_documentation_tests(config=test_config)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "79f1b475",
+ "metadata": {},
+ "source": [
+ "<a id='toc11_'></a>\n",
+ "\n",
+ "## In summary\n",
+ "\n",
+ "In this notebook, you learned how to:\n",
+ "\n",
+ "- [x] Register a model within the ValidMind Platform\n",
+ "- [x] Install and initialize the ValidMind Library\n",
+ "- [x] Preview the documentation template for your model\n",
+ "- [x] Import a sample dataset\n",
+ "- [x] Initialize ValidMind datasets and model objects\n",
+ "- [x] Assign model predictions to your ValidMind model objects\n",
+ "- [x] Run a full suite of documentation tests\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "adcc9956",
+ "metadata": {},
+ "source": [
+ "<a id='toc12_'></a>\n",
+ "\n",
+ "## Next steps\n",
+ "\n",
+ "You can look at the output produced by the ValidMind Library right in the notebook where you ran the code, as you would expect. But there is a better way — use the ValidMind Platform to work with your model documentation."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6b8e72f6",
+ "metadata": {},
+ "source": [
+ "<a id='toc12_1_'></a>\n",
+ "\n",
+ "### Work with your model documentation\n",
+ "\n",
+ "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n",
+ "\n",
+ "2. In the left sidebar that appears for your model, click **Documentation**.\n",
+ "\n",
+ " What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "585b79fd",
+ "metadata": {},
+ "source": [
+ "<a id='toc12_2_'></a>\n",
+ "\n",
+ "### Discover more learning resources\n",
+ "\n",
+ "For a more in-depth introduction to using the ValidMind Library for development, check out our introductory development series and the accompanying interactive training:\n",
+ "\n",
+ "- **[ValidMind for model development](https://docs.validmind.ai/developer/validmind-library.html#for-model-development)**\n",
+ "- **[Developer Fundamentals](https://docs.validmind.ai/training/developer-fundamentals/developer-fundamentals-register.html)**\n",
+ "\n",
+ "We also offer many interactive notebooks to help you document models:\n",
+ "\n",
+ "- [Run tests & test suites](https://docs.validmind.ai/guide/testing-overview.html)\n",
+ "- [Code samples](https://docs.validmind.ai/guide/samples-jupyter-notebooks.html)\n",
+ "\n",
+ "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc13_'></a>\n",
+ "\n",
+ "## Upgrade ValidMind\n",
+ "\n",
+ "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\">After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.</div>\n",
+ "\n",
+ "Retrieve the information for the currently installed version of ValidMind:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "35955b6b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip show validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "upgrade-version-ab5a531d-4334-4c5b-b6aa-754e336f127f",
+ "metadata": {},
+ "source": [
+ "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n",
+ "\n",
+ "```bash\n",
+ "%pip install --upgrade validmind\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "upgrade-restart-faf9c324-3332-4eaf-aef5-2c0ca650d5ca",
+ "metadata": {},
+ "source": [
+ "You may need to restart your kernel after running the upgrade package for changes to be applied."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "ValidMind Library",
+ "language": "python",
+ "name": "validmind"
+ },
+ "language_info": {
+ "name": "python",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/site/notebooks/quickstart/quickstart_model_validation.ipynb b/site/notebooks/quickstart/quickstart_model_validation.ipynb
new file mode 100644
index 0000000000..4e233f9631
--- /dev/null
+++ b/site/notebooks/quickstart/quickstart_model_validation.ipynb
@@ -0,0 +1,1174 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "926aad7c",
+ "metadata": {},
+ "source": [
+ "# Quickstart for model validation\n",
+ "\n",
+ "Learn the basics of using ValidMind to validate models as part of a model validation workflow. Set up the ValidMind Library in your environment, and generate a draft of a validation report using ValidMind tests for a binary classification model.\n",
+ "\n",
+ "To validate a model with the ValidMind Library, we'll:\n",
+ "\n",
+ "1. Import a sample dataset and preprocess it, then split the datasets and initialize them for use with ValidMind\n",
+ "2. Independently verify data quality tests performed on datasets by model development\n",
+ "3. Import a champion model for evaluation\n",
+ "4. Run model evaluation tests with the ValidMind Library, which will send the results of those tests to the ValidMind Platform"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dc56a98f",
+ "metadata": {},
+ "source": [
+ "::: {.content-hidden when-format=\"html\"}\n",
+ "## Contents \n",
+ "- [Introduction](#toc1_) \n",
+ "- [About ValidMind](#toc2_) \n",
+ " - [Before you begin](#toc2_1_) \n",
+ " - [New to ValidMind?](#toc2_2_) \n",
+ " - [Key concepts](#toc2_3_) \n",
+ "- [Setting up](#toc3_) \n",
+ " - [Register a sample model](#toc3_1_) \n",
+ " - [Assign validator credentials](#toc3_1_1_) \n",
+ " - [Install the ValidMind Library](#toc3_2_) \n",
+ " - [Initialize the ValidMind Library](#toc3_3_) \n",
+ " - [Get your code snippet](#toc3_3_1_) \n",
+ " - [Initialize the Python environment](#toc3_4_) \n",
+ "- [Getting to know ValidMind](#toc4_) \n",
+ " - [Preview the validation report template](#toc4_1_) \n",
+ " - [View validation report in the ValidMind Platform](#toc4_2_) \n",
+ "- [Importing the sample dataset](#toc5_) \n",
+ " - [Load the sample dataset](#toc5_1_) \n",
+ " - [Preprocess the raw dataset](#toc5_2_) \n",
+ " - [Split the dataset](#toc5_2_1_) \n",
+ " - [Separate features and targets](#toc5_2_2_) \n",
+ "- [Running data quality tests](#toc6_) \n",
+ " - [Identify qualitative tests](#toc6_1_) \n",
+ " - [Initialize the ValidMind datasets](#toc6_2_) \n",
+ " - [Run an individual data quality test](#toc6_3_) \n",
+ " - [Run data comparison tests](#toc6_4_) \n",
+ "- [Importing the champion model](#toc7_) \n",
+ " - [Initialize a model object](#toc7_1_) \n",
+ " - [Assign predictions](#toc7_2_) \n",
+ "- [Running model evaluation tests](#toc8_) \n",
+ " - [Run model performance tests](#toc8_1_) \n",
+ " - [Run diagnostic tests](#toc8_2_) \n",
+ " - [Run feature importance tests](#toc8_3_) \n",
+ "- [In summary](#toc9_) \n",
+ "- [Next steps](#toc10_) \n",
+ " - [Work with your validation report](#toc10_1_) \n",
+ " - [Discover more learning resources](#toc10_2_) \n",
+ "- [Upgrade ValidMind](#toc11_) \n",
+ "\n",
+ ":::\n",
+ "<!-- jn-toc-notebook-config\n",
+ "\tnumbering=false\n",
+ "\tanchor=true\n",
+ "\tflat=false\n",
+ "\tminLevel=2\n",
+ "\tmaxLevel=4\n",
+ "\t/jn-toc-notebook-config -->\n",
+ "<!-- THIS CELL WILL BE REPLACED ON TOC UPDATE. DO NOT WRITE YOUR TEXT IN THIS CELL -->"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e3f7cf22",
+ "metadata": {},
+ "source": [
+ "<a id='toc1_'></a>\n",
+ "\n",
+ "## Introduction\n",
+ "\n",
+ "Model validation aims to independently assess the compliance of *champion models* created by model developers with regulatory guidance by conducting thorough testing and analysis, potentially including the use of challenger models to benchmark performance. Assessments, presented in the form of a validation report, typically include *model findings* and recommendations to address those issues.\n",
+ "\n",
+ "A *binary classification model* is a type of predictive model used in churn analysis to identify customers who are likely to leave a service or subscription by analyzing various behavioral, transactional, and demographic factors.\n",
+ "\n",
+ "- This model helps businesses take proactive measures to retain at-risk customers by offering personalized incentives, improving customer service, or adjusting pricing strategies.\n",
+ "- Effective validation of a churn prediction model ensures that businesses can accurately identify potential churners, optimize retention efforts, and enhance overall customer satisfaction while minimizing revenue loss."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e5065e3c",
+ "metadata": {},
+ "source": [
+ "<a id='toc2_'></a>\n",
+ "\n",
+ "## About ValidMind\n",
+ "\n",
+ "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n",
+ "\n",
+ "You use the ValidMind Library to automate comparison and other validation tests, and then use the ValidMind Platform to submit compliance assessments of champion models via comprehensive validation reports. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model developers."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "541fadfb",
+ "metadata": {},
+ "source": [
+ "<a id='toc2_1_'></a>\n",
+ "\n",
+ "### Before you begin\n",
+ "\n",
+ "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n",
+ "\n",
+ "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a196a66c",
+ "metadata": {},
+ "source": [
+ "<a id='toc2_2_'></a>\n",
+ "\n",
+ "### New to ValidMind?\n",
+ "\n",
+ "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
+ "\n",
+ "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\"><span style=\"color: #083E44;\"><b>For access to all features available in this notebook, create a free ValidMind account.</b></span>\n",
+ "<br></br>\n",
+ "Signing up is FREE — <a href=\"https://docs.validmind.ai/guide/configuration/register-with-validmind.html\" style=\"color: #DE257E;\"><b>Register with ValidMind</b></a></div>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "29878854",
+ "metadata": {},
+ "source": [
+ "<a id='toc2_3_'></a>\n",
+ "\n",
+ "### Key concepts\n",
+ "\n",
+ "**Validation report**: A comprehensive and structured assessment of a model’s development and performance, focusing on verifying its integrity, appropriateness, and alignment with its intended use. It includes analyses of model assumptions, data quality, performance metrics, outcomes of testing procedures, and risk considerations. The validation report supports transparency, regulatory compliance, and informed decision-making by documenting the validator’s independent review and conclusions.\n",
+ "\n",
+ "**Validation report template**: Serves as a standardized framework for conducting and documenting model validation activities. It outlines the required sections, recommended analyses, and expected validation tests, ensuring consistency and completeness across validation reports. The template helps guide validators through a systematic review process while promoting comparability and traceability of validation outcomes.\n",
+ "\n",
+ "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets.\n",
+ "\n",
+ "**Metrics**: A subset of tests that do not have thresholds. In the context of this notebook, metrics and tests can be thought of as interchangeable concepts.\n",
+ "\n",
+ "**Custom metrics**: Custom metrics are functions that you define to evaluate your model or dataset. These functions can be registered with the ValidMind Library to be used in the ValidMind Platform.\n",
+ "\n",
+ "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n",
+ "\n",
+ " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n",
+ " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n",
+ " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom metric.\n",
+ " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom metric. (Learn more: [Run tests with multiple datasets](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html))\n",
+ "\n",
+ "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a metric, customize its behavior, or provide additional context.\n",
+ "\n",
+ "**Outputs**: Custom metrics can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "58b32cff",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_'></a>\n",
+ "\n",
+ "## Setting up"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e9be1dd1",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_1_'></a>\n",
+ "\n",
+ "### Register a sample model\n",
+ "\n",
+ "In a usual model lifecycle, a champion model will have been independently registered in your model inventory and submitted to you for validation by your model development team as part of the effective challenge process. (**Learn more:** [Submit for approval](https://docs.validmind.ai/guide/model-documentation/submit-for-approval.html))\n",
+ "\n",
+ "For this notebook, we'll have you register a dummy model in the ValidMind Platform inventory and assign yourself as the validator to familiarize you with the ValidMind interface and circumvent the need for an existing model:\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n",
+ "\n",
+ "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n",
+ "\n",
+ " For example, to register a model for use with this notebook, select:\n",
+ "\n",
+ " - Documentation template: `Binary classification`\n",
+ " - Use case: `Marketing/Sales - Attrition/Churn Management`\n",
+ "\n",
+ " You can fill in other options according to your preference."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f7815e5f",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_1_1_'></a>\n",
+ "\n",
+ "#### Assign validator credentials\n",
+ "\n",
+ "In order to log tests as a validator instead of as a developer, on the model details page that appears after you've successfully registered your sample model:\n",
+ "\n",
+ "1. Remove yourself as a model owner: \n",
+ "\n",
+ " - Click on the **OWNERS** tile.\n",
+ " - Click the **x** next to your name to remove yourself from that model's role.\n",
+ " - Click **Save** to apply your changes to that role.\n",
+ "\n",
+ "2. Remove yourself as a developer: \n",
+ "\n",
+ " - Click on the **DEVELOPERS** tile.\n",
+ " - Click the **x** next to your name to remove yourself from that model's role.\n",
+ " - Click **Save** to apply your changes to that role.\n",
+ "\n",
+ "3. Add yourself as a validator: \n",
+ "\n",
+ " - Click on the **VALIDATORS** tile.\n",
+ " - Select your name from the drop-down menu.\n",
+ " - Click **Save** to apply your changes to that role."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_2_'></a>\n",
+ "\n",
+ "### Install the ValidMind Library\n",
+ "\n",
+ "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\"><span style=\"color: #083E44;\"><b>Recommended Python versions</b></span>\n",
+ "<br></br>\n",
+ "Python 3.8 <= x <= 3.11</div>\n",
+ "\n",
+ "To install the library:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "64eb485c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip install -q validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "330c8f40",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_3_'></a>\n",
+ "\n",
+ "### Initialize the ValidMind Library\n",
+ "\n",
+ "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3f04449a",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_3_1_'></a>\n",
+ "\n",
+ "#### Get your code snippet\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this notebook.\n",
+ "\n",
+ "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
+ "\n",
+ "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9c6ce354",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load your model identifier credentials from an `.env` file\n",
+ "\n",
+ "%load_ext dotenv\n",
+ "%dotenv .env\n",
+ "\n",
+ "# Or replace with your code snippet\n",
+ "\n",
+ "import validmind as vm\n",
+ "\n",
+ "vm.init(\n",
+ " # api_host=\"...\",\n",
+ " # api_key=\"...\",\n",
+ " # api_secret=\"...\",\n",
+ " # model=\"...\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b07758a3",
+ "metadata": {},
+ "source": [
+ "<a id='toc3_4_'></a>\n",
+ "\n",
+ "### Initialize the Python environment\n",
+ "\n",
+ "Then, let's import the necessary libraries and set up your Python environment for data analysis by enabling **`matplotlib`**, a plotting library used for visualizing data.\n",
+ "\n",
+ "This ensures that any plots you generate will render inline in our notebook output rather than opening in a separate window:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1e53065d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ecc3d3a1",
+ "metadata": {},
+ "source": [
+ "<a id='toc4_'></a>\n",
+ "\n",
+ "## Getting to know ValidMind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ce709365",
+ "metadata": {},
+ "source": [
+ "<a id='toc4_1_'></a>\n",
+ "\n",
+ "### Preview the validation report template\n",
+ "\n",
+ "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for model validation. A template predefines sections for your validation report and provides a general outline to follow, making the validation process much easier.\n",
+ "\n",
+ "You will attach evidence to this template in the form of risk assessment notes, findings, and test results later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "be445598",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.preview_template()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "43966e46",
+ "metadata": {},
+ "source": [
+ "<a id='toc4_2_'></a>\n",
+ "\n",
+ "### View validation report in the ValidMind Platform\n",
+ "\n",
+ "Next, let's head to the ValidMind Platform to see the template in action:\n",
+ "\n",
+ "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
+ "\n",
+ "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this notebook.\n",
+ "\n",
+ "3. Click on the **Validation Report** for your model and note:\n",
+ "\n",
+ " - [x] The risk assessment compliance summary at the top of the report (screenshot below)\n",
+ " - [x] How the structure of the validation report reflects the previewed template\n",
+ "\n",
+ " <img src= \"../tutorials/model_validation/compliance-summary.png\" alt=\"Screenshot showing the risk assessment compliance summary\" style=\"border: 2px solid #083E44; border-radius: 8px; border-right-width: 2px; border-bottom-width: 3px;\">\n",
+ " <br><br>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5cd4890e",
+ "metadata": {},
+ "source": [
+ "<a id='toc5_'></a>\n",
+ "\n",
+ "## Importing the sample dataset"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6aeb700e",
+ "metadata": {},
+ "source": [
+ "<a id='toc5_1_'></a>\n",
+ "\n",
+ "### Load the sample dataset\n",
+ "\n",
+ "First, let's import the public [Bank Customer Churn Prediction](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction) dataset from Kaggle, which was used to develop the dummy champion model.\n",
+ "\n",
+ "We'll use this dataset to review steps that should have been conducted during the initial development and documentation of the model to ensure that the model was built correctly. By independently performing steps taken by the model development team, we can confirm whether the model was built using appropriate and properly processed data.\n",
+ "\n",
+ "In our below example, note that:\n",
+ "\n",
+ "- The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n",
+ "- The ValidMind Library provides a wrapper to automatically load the dataset as a [Pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object. A Pandas Dataframe is a two-dimensional tabular data structure that makes use of rows and columns."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "73076ee3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from validmind.datasets.classification import customer_churn\n",
+ "\n",
+ "print(\n",
+ " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{customer_churn.target_column}' \\n\\t• Class labels: {customer_churn.class_labels}\"\n",
+ ")\n",
+ "\n",
+ "raw_df = customer_churn.load_data()\n",
+ "raw_df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6c1e567a",
+ "metadata": {},
+ "source": [
+ "<a id='toc5_2_'></a>\n",
+ "\n",
+ "### Preprocess the raw dataset\n",
+ "\n",
+ "Let's say that thanks to the documentation submitted by the model development team ([Learn more ...](quickstart_model_documentation.ipynb)), we know that the sample dataset was first preprocessed before being used to train the champion model.\n",
+ "\n",
+ "During model validation, we use the same data processing logic and training procedure to confirm that the model's results can be reproduced independently, so let's also start by preprocessing our imported dataset to verify that preprocessing was done correctly. This involves splitting the data and separating the features (inputs) from the targets (outputs)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4b98b04a",
+ "metadata": {},
+ "source": [
+ "<a id='toc5_2_1_'></a>\n",
+ "\n",
+ "#### Split the dataset\n",
+ "\n",
+ "Splitting our dataset helps assess how well the model generalizes to unseen data.\n",
+ "\n",
+ "Use [`preprocess()`](https://docs.validmind.ai/validmind/validmind/datasets/classification/customer_churn.html#preprocess) to split our dataset into three subsets:\n",
+ "\n",
+ "1. **train_df** — Used to train the model.\n",
+ "2. **validation_df** — Used to evaluate the model's performance during training.\n",
+ "3. **test_df** — Used later on to asses the model's performance on new, unseen data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ee8cfaaf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f6549607",
+ "metadata": {},
+ "source": [
+ "<a id='toc5_2_2_'></a>\n",
+ "\n",
+ "#### Separate features and targets\n",
+ "\n",
+ "To train the model, we need to provide it with:\n",
+ "\n",
+ "1. **Inputs** — Features such as customer age, usage, etc.\n",
+ "2. **Outputs (Expected answers/labels)** — in our case, we would like to know whether the customer churned or not.\n",
+ "\n",
+ "Here, we'll use `x_train` to hold the input features, and `y_train` to hold the target variable — the values we want the model to predict:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6fe65be5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "x_train = train_df.drop(customer_churn.target_column, axis=1)\n",
+ "y_train = train_df[customer_churn.target_column]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a467c9ce",
+ "metadata": {},
+ "source": [
+ "<a id='toc6_'></a>\n",
+ "\n",
+ "## Running data quality tests\n",
+ "\n",
+ "With everything ready to go, let's explore some of ValidMind's available tests to help us assess the quality of our datasets. Using ValidMind’s repository of tests streamlines your validation testing, and helps you ensure that your models are being validated appropriately."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "da09a71d",
+ "metadata": {},
+ "source": [
+ "<a id='toc6_1_'></a>\n",
+ "\n",
+ "### Identify qualitative tests\n",
+ "\n",
+ "We want to narrow down the tests we want to run from the selection provided by ValidMind, so we'll use the [`vm.tests.list_tasks_and_tags()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tasks_and_tags) to list which `tags` are associated with each `task` type:\n",
+ "\n",
+ "- **`tasks`** represent the kind of modeling task associated with a test. Here we'll focus on `classification` tasks.\n",
+ "- **`tags`** are free-form descriptions providing more details about the test, for example, what category the test falls into. Here we'll focus on the `data_quality` tag."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "85bc2f85",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.list_tasks_and_tags()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "39f050b1",
+ "metadata": {},
+ "source": [
+ "Then we'll call [the `vm.tests.list_tests()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to list all the data quality tests for classification:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "31b31a51",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.list_tests(\n",
+ " tags=[\"data_quality\"], task=\"classification\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d433c949",
+ "metadata": {},
+ "source": [
+ "<a id='toc6_2_'></a>\n",
+ "\n",
+ "### Initialize the ValidMind datasets\n",
+ "\n",
+ "Before you can run tests with your preprocessed datasets, you must first initialize a ValidMind `Dataset` object using the [`init_dataset`](https://docs.validmind.ai/validmind/validmind.html#init_dataset) function from the ValidMind (`vm`) module. **This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind,** but you only need to do it once per dataset.\n",
+ "\n",
+ "For this example, we'll pass in the following arguments:\n",
+ "\n",
+ "- **`dataset`** — The raw dataset that you want to provide as input to tests.\n",
+ "- **`input_id`** — A unique identifier that allows tracking what inputs are used when running each individual test.\n",
+ "- **`target_column`** — A required argument if tests require access to true values. This is the name of the target column in the dataset.\n",
+ "- **`class_labels`** — An optional value to map predicted classes to class labels."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ba677dd7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize the raw dataset\n",
+ "vm_raw_dataset = vm.init_dataset(\n",
+ " dataset=raw_df,\n",
+ " input_id=\"raw_dataset\",\n",
+ " target_column=customer_churn.target_column,\n",
+ " class_labels=customer_churn.class_labels,\n",
+ ")\n",
+ "\n",
+ "# Initialize the training dataset\n",
+ "vm_train_ds = vm.init_dataset(\n",
+ " dataset=train_df,\n",
+ " input_id=\"train_dataset\",\n",
+ " target_column=customer_churn.target_column,\n",
+ ")\n",
+ "\n",
+ "# Initialize the validation dataset\n",
+ "vm_validation_ds = vm.init_dataset(\n",
+ " dataset=validation_df,\n",
+ " input_id=\"validation_dataset\",\n",
+ " target_column=customer_churn.target_column,\n",
+ ")\n",
+ "\n",
+ "# Initialize the testing dataset\n",
+ "vm_test_ds = vm.init_dataset(\n",
+ " dataset=test_df,\n",
+ " input_id=\"test_dataset\",\n",
+ " target_column=customer_churn.target_column\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "45ff3201",
+ "metadata": {},
+ "source": [
+ "<a id='toc6_3_'></a>\n",
+ "\n",
+ "### Run an individual data quality test\n",
+ "\n",
+ "Next, we'll use our previously initialized raw dataset (`vm_raw_dataset`) as input to run an individual test, then log the result to the ValidMind Platform.\n",
+ "\n",
+ "- You run validation tests by calling [the `run_test` function](https://docs.validmind.ai/validmind/validmind/tests.html#run_test) provided by the `validmind.tests` module.\n",
+ "- Every test result returned by the `run_test()` function has a [`.log()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#TestResult.log) that can be used to send the test results to the ValidMind Platform.\n",
+ "\n",
+ "Here, we'll use the [`ClassImbalance` test](https://docs.validmind.ai/tests/data_validation/ClassImbalance.html) as an example:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dcb9b017",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.run_test(\n",
+ " test_id=\"validmind.data_validation.ClassImbalance\",\n",
+ " inputs={\n",
+ " \"dataset\": vm_raw_dataset\n",
+ " }\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cc4829be",
+ "metadata": {},
+ "source": [
+ "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\"><span style=\"color: #083E44;\"><b>Note the output returned indicating that a test-driven block doesn't currently exist in your model's documentation for some test IDs. </b></span>\n",
+ "<br></br>\n",
+ "That's expected, as when we run validations tests the results logged need to be manually added to your report as part of your compliance assessment process within the ValidMind Platform. You'll continue to see this message throughout this notebook as we run and log more tests.</div>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "43411ece",
+ "metadata": {},
+ "source": [
+ "<a id='toc6_4_'></a>\n",
+ "\n",
+ "### Run data comparison tests\n",
+ "<span id=\"data-comparison\">\n",
+ "\n",
+ "We can also use ValidMind to perform comparison tests between our datasets, again logging the results to the ValidMind Platform. Below, we'll perform two sets of comparison tests with a mix of our datasets and the same class imbalance test:\n",
+ "\n",
+ "- When running individual tests, you can use a custom **`result_id`** to tag the individual result with a unique identifier, appended to the `test_id` with a `:` separator.\n",
+ "- We can specify all the tests we'd ike to run in a dictionary called `test_config`, and we'll pass in an **`input_grid`** of individual test inputs to compare. In this case, we'll input our two datasets for comparison. Note here that the `input_grid` expects the `input_id` of the dataset as the value rather than the variable name we specified."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d53edde7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Individual test config with inputs specified\n",
+ "test_config = {\n",
+ " # Comparison between training and testing datasets to check if class balance is the same in both sets\n",
+ " \"validmind.data_validation.ClassImbalance:train_vs_validation\": {\n",
+ " \"input_grid\": {\"dataset\": [\"train_dataset\", \"validation_dataset\"]}\n",
+ " },\n",
+ " # Comparison between training and testing datasets to confirm that both sets have similar class distributions\n",
+ " \"validmind.data_validation.ClassImbalance:train_vs_test\": {\n",
+ " \"input_grid\": {\"dataset\": [\"train_dataset\", \"test_dataset\"]},\n",
+ " },\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "16132a57",
+ "metadata": {},
+ "source": [
+ "Then batch run and log our tests in `test_config`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1b97e404",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for t in test_config:\n",
+ " print(t)\n",
+ " try:\n",
+ " # Check if test has input_grid\n",
+ " if 'input_grid' in test_config[t]:\n",
+ " # For tests with input_grid, pass the input_grid configuration\n",
+ " if 'params' in test_config[t]:\n",
+ " vm.tests.run_test(t, input_grid=test_config[t]['input_grid'], params=test_config[t]['params']).log()\n",
+ " else:\n",
+ " vm.tests.run_test(t, input_grid=test_config[t]['input_grid']).log()\n",
+ " else:\n",
+ " # Original logic for regular inputs\n",
+ " if 'params' in test_config[t]:\n",
+ " vm.tests.run_test(t, inputs=test_config[t]['inputs'], params=test_config[t]['params']).log()\n",
+ " else:\n",
+ " vm.tests.run_test(t, inputs=test_config[t]['inputs']).log()\n",
+ " except Exception as e:\n",
+ " print(f\"Error running test {t}: {str(e)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bd8d4805",
+ "metadata": {},
+ "source": [
+ "<a id='toc7_'></a>\n",
+ "\n",
+ "## Importing the champion model\n",
+ "\n",
+ "With our raw dataset preprocessed, let's go ahead and import the champion model submitted by the model development team in the format of a `.pkl` file: **[xgboost_model_champion.pkl](xgboost_model_champion.pkl)**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7f18188e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Import the champion model\n",
+ "import joblib\n",
+ "\n",
+ "xgboost = joblib.load(\"xgboost_model_champion.pkl\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9f858689",
+ "metadata": {},
+ "source": [
+ "<a id='toc7_1_'></a>\n",
+ "\n",
+ "### Initialize a model object\n",
+ "\n",
+ "In addition to the initialized datasets, you'll also need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data for our champion model.\n",
+ "\n",
+ "You simply initialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0a799cf2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize the champion XGBoost model\n",
+ "vm_xgboost = vm.init_model(\n",
+ " xgboost,\n",
+ " input_id=\"xgboost_champion\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "20ef72d4",
+ "metadata": {},
+ "source": [
+ "<a id='toc7_2_'></a>\n",
+ "\n",
+ "### Assign predictions\n",
+ "\n",
+ "Once the model has been registered, you can assign model predictions to the training and testing datasets.\n",
+ "\n",
+ "- The [`assign_predictions()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#assign_predictions) from the `Dataset` object can link existing predictions to any number of models.\n",
+ "- This method links the model's class prediction values and probabilities to our `vm_train_ds` and `vm_test_ds` datasets.\n",
+ "\n",
+ "If no prediction values are passed, the method will compute predictions automatically:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "71dd8e7b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm_train_ds.assign_predictions(\n",
+ " model=vm_xgboost,\n",
+ ")\n",
+ "\n",
+ "vm_test_ds.assign_predictions(\n",
+ " model=vm_xgboost,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "629964ef",
+ "metadata": {},
+ "source": [
+ "<a id='toc8_'></a>\n",
+ "\n",
+ "## Running model evaluation tests\n",
+ "\n",
+ "With our setup complete, let's run the rest of our validation tests. Since we have already verified the data quality of the dataset used to train our champion model, we will now focus on evaluating the model's performance."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b3b40873",
+ "metadata": {},
+ "source": [
+ "<a id='toc8_1_'></a>\n",
+ "\n",
+ "### Run model performance tests\n",
+ "\n",
+ "First, let's run some performance tests. Use [`vm.tests.list_tests()`](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to identify all the model performance tests for classification:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "202792e8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.list_tests(tags=[\"model_performance\"], task=\"classification\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0b49a782",
+ "metadata": {},
+ "source": [
+ "We'll isolate the specific tests we want to run in `mpt`, and append an identifier for our champion model here to the `result_id` with a `:` separator like we did above in another test:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9fc18843",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mpt = [\n",
+ " \"validmind.model_validation.sklearn.ClassifierPerformance:xgboost_champion\",\n",
+ " \"validmind.model_validation.sklearn.ConfusionMatrix:xgboost_champion\",\n",
+ " \"validmind.model_validation.sklearn.ROCCurve:xgboost_champion\"\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "78becfa1",
+ "metadata": {},
+ "source": [
+ "Now, let's run and log our batch of model performance tests using our testing dataset (`vm_test_ds`) for our champion model:\n",
+ "\n",
+ "- The test set serves as a proxy for real-world data, providing an unbiased estimate of model performance since it was not used during training or tuning.\n",
+ "- The test set also acts as protection against selection bias and model tweaking, giving a final, more unbiased checkpoint."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6866b21c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for test in mpt:\n",
+ " vm.tests.run_test(\n",
+ " test,\n",
+ " inputs={\n",
+ " \"dataset\": vm_test_ds, \"model\" : vm_xgboost,\n",
+ " },\n",
+ " ).log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2aae3040",
+ "metadata": {},
+ "source": [
+ "<a id='toc8_2_'></a>\n",
+ "\n",
+ "### Run diagnostic tests\n",
+ "\n",
+ "Next, we want to inspect the robustness and stability of our champion model. Use `list_tests()` to list all available diagnosis tests applicable to classification tasks:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c9b3caa4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.list_tests(tags=[\"model_diagnosis\"], task=\"classification\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ecefecac",
+ "metadata": {},
+ "source": [
+ "Let’s now assess the model for potential signs of *overfitting* and identify any sub-segments where performance may inconsistent.\n",
+ "\n",
+ "Overfitting occurs when a model learns the training data too well, capturing not only the true pattern but noise and random fluctuations resulting in excellent performance on the training dataset but poor generalization to new, unseen data:\n",
+ "\n",
+ "- Since the training dataset (`vm_train_ds`) was used to fit the model, we use this set to establish a baseline performance for how well the model performs on data it has already seen.\n",
+ "- The testing dataset (`vm_test_ds`) was never seen during training, and here simulates real-world generalization, or how well the model performs on new, unseen data. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "82f824f2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.run_test(\n",
+ " test_id=\"validmind.model_validation.sklearn.OverfitDiagnosis:xgboost_champion\",\n",
+ " input_grid={\n",
+ " \"datasets\": [[vm_train_ds,vm_test_ds]],\n",
+ " \"model\" : [vm_xgboost]\n",
+ " }\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c7f7988e",
+ "metadata": {},
+ "source": [
+ "Let's also conduct *robustness* and *stability* tests.\n",
+ "\n",
+ "- Robustness evaluates the model’s ability to maintain consistent performance under varying input conditions.\n",
+ "- Stability assesses whether the model produces consistent outputs across different data subsets or over time.\n",
+ "\n",
+ "Again, we'll use both the training and testing datasets to establish baseline performance and to simulate real-world generalization:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b2676197",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vm.tests.run_test(\n",
+ " test_id=\"validmind.model_validation.sklearn.RobustnessDiagnosis:xgboost_champion\",\n",
+ " input_grid={\n",
+ " \"datasets\": [[vm_train_ds,vm_test_ds]],\n",
+ " \"model\" : [vm_xgboost]\n",
+ " },\n",
+ ").log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0be88c10",
+ "metadata": {},
+ "source": [
+ "<a id='toc8_3_'></a>\n",
+ "\n",
+ "### Run feature importance tests\n",
+ "\n",
+ "We also want to verify the relative influence of different input features on our model's predictions. Use `list_tests()` to identify all the feature importance tests for classification and store them in `FI`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9c8c26e6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Store the feature importance tests\n",
+ "FI = vm.tests.list_tests(tags=[\"feature_importance\"], task=\"classification\",pretty=False)\n",
+ "FI"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "31326bc8",
+ "metadata": {},
+ "source": [
+ "We'll only use our testing dataset (`vm_test_ds`) here, to provide a realistic, unseen sample that mimic future or production data, as the training dataset has already influenced our model during learning:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5a49f550",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Run and log our feature importance tests with the testing dataset\n",
+ "for test in FI:\n",
+ " vm.tests.run_test(\n",
+ " \"\".join((test,':xgboost_champion')),\n",
+ " inputs={\n",
+ " \"dataset\": vm_test_ds, \"model\": vm_xgboost\n",
+ " },\n",
+ " ).log()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "19a0caa4",
+ "metadata": {},
+ "source": [
+ "<a id='toc9_'></a>\n",
+ "\n",
+ "## In summary\n",
+ "\n",
+ "In this notebook, you learned how to:\n",
+ "\n",
+ "- [x] Register a model within the ValidMind Platform\n",
+ "- [x] Install and initialize the ValidMind Library\n",
+ "- [x] Preview the validation report template for your model\n",
+ "- [x] Import a sample dataset and champion model\n",
+ "- [x] Initialize ValidMind datasets and model objects\n",
+ "- [x] Assign model predictions to your ValidMind model objects\n",
+ "- [x] Identify and run various validation tests\n",
+ "\n",
+ "In a usual model validation workflow, you would wrap up your validation testing by verifying that all the tests provided by the model development team were run and reported accurately, and perhaps even propose a challenger model, comparing the performance of the challenger with the running champion.\n",
+ "\n",
+ "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\"><span style=\"color: #083E44;\"><b>With ValidMind, you can easily:</b></span>\n",
+ "<ul>\n",
+ " <li>Specify all the tests you'd like to independently rerun, just like you did in the step <a href=\"#run-data-comparison-tests\" style=\"color: #DE257E;\">Run data comparision tests</a></li>\n",
+ " <li>Evaluate the performance of a challenger model against the champion, just like you did in the steps under <a href=\"#running-model-evaluation-tests\" style=\"color: #DE257E;\">Running model evaluation tests</a></li>\n",
+ "</ul>\n",
+ "</div>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ed4bc468",
+ "metadata": {},
+ "source": [
+ "<a id='toc10_'></a>\n",
+ "\n",
+ "## Next steps\n",
+ "\n",
+ "You can look at the output produced by the ValidMind Library right in the notebook where you ran the code, as you would expect. But there is a better way — use the ValidMind Platform to work with your validation report."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3836cbc2",
+ "metadata": {},
+ "source": [
+ "<a id='toc10_1_'></a>\n",
+ "\n",
+ "### Work with your validation report\n",
+ "\n",
+ "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n",
+ "\n",
+ "2. In the left sidebar that appears for your model, click **Validation Report**.\n",
+ "\n",
+ " From here, you can link test results as evidence, add model findings, assess compliance, and submit your validation report for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-validation/preparing-validation-reports.html)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8bdf85fe",
+ "metadata": {},
+ "source": [
+ "<a id='toc10_2_'></a>\n",
+ "\n",
+ "### Discover more learning resources\n",
+ "\n",
+ "For a more in-depth introduction to using the ValidMind Library for validation, check out our introductory validation series and the accompanying interactive training:\n",
+ "\n",
+ "- **[ValidMind for model validation](https://docs.validmind.ai/developer/validmind-library.html#for-model-validation)**\n",
+ "- **[Validator Fundamentals](https://docs.validmind.ai/training/validator-fundamentals/validator-fundamentals-register.html)**\n",
+ "\n",
+ "We offer many interactive notebooks to help you validate models:\n",
+ "\n",
+ "- [Run tests & test suites](https://docs.validmind.ai/guide/testing-overview.html)\n",
+ "- [Code samples](https://docs.validmind.ai/guide/samples-jupyter-notebooks.html)\n",
+ "\n",
+ "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "383cd893",
+ "metadata": {},
+ "source": [
+ "<a id='toc11_'></a>\n",
+ "\n",
+ "## Upgrade ValidMind\n",
+ "\n",
+ "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\">After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.</div>\n",
+ "\n",
+ "Retrieve the information for the currently installed version of ValidMind:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "upgrade-show-c0a446ff-f26f-4ad0-839a-e92927711798",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip show validmind"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "upgrade-version-098b75d8-4380-4cc0-ac06-da363a3cf5a0",
+ "metadata": {},
+ "source": [
+ "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n",
+ "\n",
+ "```bash\n",
+ "%pip install --upgrade validmind\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "upgrade-restart-aebde628-1c17-4f70-88c8-d4561e803abe",
+ "metadata": {},
+ "source": [
+ "You may need to restart your kernel after running the upgrade package for changes to be applied."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "ValidMind Library",
+ "language": "python",
+ "name": "validmind"
+ },
+ "language_info": {
+ "name": "python",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/site/notebooks/quickstart/xgboost_model_champion.pkl b/site/notebooks/quickstart/xgboost_model_champion.pkl
new file mode 100644
index 0000000000..e6fdef9b04
Binary files /dev/null and b/site/notebooks/quickstart/xgboost_model_champion.pkl differ
diff --git a/site/notebooks/quickstart_customer_churn_full_suite.ipynb b/site/notebooks/quickstart_customer_churn_full_suite.ipynb
deleted file mode 100644
index d7a9c6b878..0000000000
--- a/site/notebooks/quickstart_customer_churn_full_suite.ipynb
+++ /dev/null
@@ -1,567 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Quickstart for model documentation\n",
- "\n",
- "Welcome! Let's get you started with the basic process of documenting models with ValidMind.\n",
- "\n",
- "You will learn how to initialize the ValidMind Library, load a sample dataset to train a simple classification model, and then run a ValidMind test suite to quickly generate documentation about the data and model.\n",
- "\n",
- "This notebook uses the [Bank Customer Churn Prediction](https://www.kaggle.com/code/kmalit/bank-customer-churn-prediction/data) sample dataset from Kaggle to train the classification model."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc0_'></a>\n",
- "\n",
- "## Contents \n",
- "- [About ValidMind](#toc1_) \n",
- " - [Before you begin](#toc1_1_) \n",
- " - [New to ValidMind?](#toc1_2_) \n",
- " - [Key concepts](#toc1_3_) \n",
- "- [Install the ValidMind Library](#toc2_)\n",
- "- [Initialize the client library](#toc3_) \n",
- " - [Get your code snippet](#toc3_1_)\n",
- "- [Initialize the Python environment](#toc4_) \n",
- " - [Preview the documentation template](#toc4_1_) \n",
- "- [Load the sample dataset](#toc5_) \n",
- "- [Document the model](#toc6_) \n",
- " - [Prepocess the raw dataset](#toc6_1_) \n",
- " - [Initialize the ValidMind datasets](#toc6_2_) \n",
- " - [Initialize a model object](#toc6_3_) \n",
- " - [Assign predictions to the datasets](#toc6_4_) \n",
- " - [Run the full suite of tests](#toc6_5_) \n",
- "- [Next steps](#toc7_) \n",
- " - [Work with your model documentation](#toc7_1_) \n",
- " - [Discover more learning resources](#toc7_2_) \n",
- "- [Upgrade ValidMind](#toc8_) \n",
- "\n",
- "<!-- vscode-jupyter-toc-config\n",
- "\tnumbering=false\n",
- "\tanchor=true\n",
- "\tflat=false\n",
- "\tminLevel=2\n",
- "\tmaxLevel=4\n",
- "\t/vscode-jupyter-toc-config -->\n",
- "<!-- THIS CELL WILL BE REPLACED ON TOC UPDATE. DO NOT WRITE YOUR TEXT IN THIS CELL -->"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc1_'></a>\n",
- "\n",
- "## About ValidMind\n",
- "\n",
- "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n",
- "\n",
- "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n",
- "\n",
- "<a id='toc1_1_'></a>\n",
- "\n",
- "### Before you begin\n",
- "\n",
- "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n",
- "\n",
- "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n",
- "\n",
- "<a id='toc1_2_'></a>\n",
- "\n",
- "### New to ValidMind?\n",
- "\n",
- "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n",
- "\n",
- "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\"><span style=\"color: #083E44;\"><b>For access to all features available in this notebook, create a free ValidMind account.</b></span>\n",
- "<br></br>\n",
- "Signing up is FREE — <a href=\"https://docs.validmind.ai/guide/configuration/register-with-validmind.html\" style=\"color: #DE257E;\"><b>Register with ValidMind</b></a></div>\n",
- "\n",
- "<a id='toc1_3_'></a>\n",
- "\n",
- "### Key concepts\n",
- "\n",
- "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n",
- "\n",
- "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n",
- "\n",
- "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n",
- "\n",
- "**Custom tests**: Custom tests are functions that you define to evaluate your model or dataset. These functions can be registered via the ValidMind Library to be used with the ValidMind Platform.\n",
- "\n",
- "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n",
- "\n",
- " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n",
- " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n",
- " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom test.\n",
- " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom test. See this [example](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html) for more information.\n",
- "\n",
- "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a test, customize its behavior, or provide additional context.\n",
- "\n",
- "**Outputs**: Custom tests can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n",
- "\n",
- "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n",
- "\n",
- "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases.\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc2_'></a>\n",
- "\n",
- "## Install the ValidMind Library\n",
- "\n",
- "To install the library:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "%pip install -q validmind"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc3_'></a>\n",
- "\n",
- "## Initialize the ValidMind Library\n",
- "\n",
- "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n",
- "\n",
- "<a id='toc3_1_'></a>\n",
- "\n",
- "### Get your code snippet\n",
- "\n",
- "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n",
- "\n",
- "2. In the left sidebar, navigate to **Model Inventory** and click **+ Register Model**.\n",
- "\n",
- "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n",
- "\n",
- " For example, to register a model for use with this notebook, select:\n",
- "\n",
- " - Documentation template: `Binary classification`\n",
- " - Use case: `Marketing/Sales - Attrition/Churn Management`\n",
- "\n",
- " You can fill in other options according to your preference.\n",
- "\n",
- "4. Go to **Getting Started** and click **Copy snippet to clipboard**.\n",
- "\n",
- "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Load your model identifier credentials from an `.env` file\n",
- "\n",
- "%load_ext dotenv\n",
- "%dotenv .env\n",
- "\n",
- "# Or replace with your code snippet\n",
- "\n",
- "import validmind as vm\n",
- "\n",
- "vm.init(\n",
- " # api_host=\"...\",\n",
- " # api_key=\"...\",\n",
- " # api_secret=\"...\",\n",
- " # model=\"...\",\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc4_'></a>\n",
- "\n",
- "## Initialize the Python environment\n",
- "\n",
- "Next, let's import the necessary libraries and set up your Python environment for data analysis:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import xgboost as xgb\n",
- "\n",
- "%matplotlib inline"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc4_1_'></a>\n",
- "\n",
- "### Preview the documentation template\n",
- "\n",
- "A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n",
- "\n",
- "You will upload documentation and test results into this template later on. For now, take a look at the structure that the template provides with the `vm.preview_template()` function from the ValidMind library and note the empty sections:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "vm.preview_template()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc5_'></a>\n",
- "\n",
- "## Load the sample dataset\n",
- "\n",
- "The sample dataset used here is provided by the ValidMind library. To be able to use it, you need to import the dataset and load it into a pandas [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html), a two-dimensional tabular data structure that makes use of rows and columns:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Import the sample dataset from the library\n",
- "\n",
- "from validmind.datasets.classification import customer_churn\n",
- "\n",
- "print(\n",
- " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{customer_churn.target_column}' \\n\\t• Class labels: {customer_churn.class_labels}\"\n",
- ")\n",
- "\n",
- "raw_df = customer_churn.load_data()\n",
- "raw_df.head()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc6_'></a>\n",
- "\n",
- "## Document the model\n",
- "\n",
- "As part of documenting the model with the ValidMind Library, you need to preprocess the raw dataset, initialize some training and test datasets, initialize a model object you can use for testing, and then run the full suite of tests.\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc6_1_'></a>\n",
- "\n",
- "### Prepocess the raw dataset\n",
- "\n",
- "Preprocessing performs a number of operations to get ready for the subsequent steps:\n",
- "\n",
- "- Preprocess the data: Splits the DataFrame (`df`) into multiple datasets (`train_df`, `validation_df`, and `test_df`) using `demo_dataset.preprocess` to simplify preprocessing.\n",
- "- Separate features and targets: Drops the target column to create feature sets (`x_train`, `x_val`) and target sets (`y_train`, `y_val`).\n",
- "- Initialize XGBoost classifier: Creates an `XGBClassifier` object with early stopping rounds set to 10.\n",
- "- Set evaluation metrics: Specifies metrics for model evaluation as \"error,\" \"logloss,\" and \"auc.\"\n",
- "- Fit the model: Trains the model on `x_train` and `y_train` using the validation set `(x_val, y_val)`. Verbose output is disabled.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)\n",
- "\n",
- "x_train = train_df.drop(customer_churn.target_column, axis=1)\n",
- "y_train = train_df[customer_churn.target_column]\n",
- "x_val = validation_df.drop(customer_churn.target_column, axis=1)\n",
- "y_val = validation_df[customer_churn.target_column]\n",
- "\n",
- "model = xgb.XGBClassifier(early_stopping_rounds=10)\n",
- "model.set_params(\n",
- " eval_metric=[\"error\", \"logloss\", \"auc\"],\n",
- ")\n",
- "model.fit(\n",
- " x_train,\n",
- " y_train,\n",
- " eval_set=[(x_val, y_val)],\n",
- " verbose=False,\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc6_2_'></a>\n",
- "\n",
- "### Initialize the ValidMind datasets\n",
- "\n",
- "Before you can run tests, you must first initialize a ValidMind dataset object using the [`init_dataset`](https://docs.validmind.ai/validmind/validmind.html#init_dataset) function from the ValidMind (`vm`) module.\n",
- "\n",
- "This function takes a number of arguments:\n",
- "\n",
- "- `dataset` — the raw dataset that you want to provide as input to tests\n",
- "- `input_id` - a unique identifier that allows tracking what inputs are used when running each individual test\n",
- "- `target_column` — a required argument if tests require access to true values. This is the name of the target column in the dataset\n",
- "- `class_labels` — an optional value to map predicted classes to class labels\n",
- "\n",
- "With all datasets ready, you can now initialize the raw, training and test datasets (`raw_df`, `train_df` and `test_df`) created earlier into their own dataset objects using [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset):\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "vm_raw_dataset = vm.init_dataset(\n",
- " dataset=raw_df,\n",
- " input_id=\"raw_dataset\",\n",
- " target_column=customer_churn.target_column,\n",
- " class_labels=customer_churn.class_labels,\n",
- ")\n",
- "\n",
- "vm_train_ds = vm.init_dataset(\n",
- " dataset=train_df,\n",
- " input_id=\"train_dataset\",\n",
- " target_column=customer_churn.target_column,\n",
- ")\n",
- "\n",
- "vm_test_ds = vm.init_dataset(\n",
- " dataset=test_df, input_id=\"test_dataset\", target_column=customer_churn.target_column\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc6_3_'></a>\n",
- "\n",
- "### Initialize a model object\n",
- "\n",
- "Additionally, you need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data. You simply intialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model):\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "vm_model = vm.init_model(\n",
- " model,\n",
- " input_id=\"model\",\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc6_4_'></a>\n",
- "\n",
- "### Assign predictions to the datasets\n",
- "\n",
- "We can now use the assign_predictions() method from the Dataset object to link existing predictions to any model. If no prediction values are passed, the method will compute predictions automatically:\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "vm_train_ds.assign_predictions(\n",
- " model=vm_model,\n",
- ")\n",
- "\n",
- "vm_test_ds.assign_predictions(\n",
- " model=vm_model,\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc6_5_'></a>\n",
- "\n",
- "### Run the full suite of tests\n",
- "\n",
- "This is where it all comes together: you are now ready to run the documentation tests for the model as defined by the documentation template you looked at earlier.\n",
- "\n",
- "The [`vm.run_documentation_tests`](https://docs.validmind.ai/validmind/validmind.html#run_documentation_tests) function finds and runs every test specified in the template and then uploads all the documentation and test artifacts that get generated to the ValidMind Platform.\n",
- "\n",
- "The function requires information about the inputs to use on every test. These inputs can be passed as an `inputs` argument if we want to use the same inputs for all tests. It's also possible to pass a `config` argument that has information about the `params` and `inputs` that each test requires. The `config` parameter is a dictionary with the following structure:\n",
- "\n",
- "```python\n",
- "config = {\n",
- " \"<test-id>\": {\n",
- " \"params\": {\n",
- " \"param1\": \"value1\",\n",
- " \"param2\": \"value2\",\n",
- " ...\n",
- " },\n",
- " \"inputs\": {\n",
- " \"input1\": \"value1\",\n",
- " \"input2\": \"value2\",\n",
- " ...\n",
- " }\n",
- " },\n",
- " ...\n",
- "}\n",
- "```\n",
- "\n",
- "Each `<test-id>` above corresponds to the test driven block identifiers shown by `vm.preview_template()`. For this model, we will use the default parameters for all tests, but we'll need to specify the input configuration for each one. The method `get_demo_test_config()` below constructs the default input configuration for our demo.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from validmind.utils import preview_test_config\n",
- "\n",
- "test_config = customer_churn.get_demo_test_config()\n",
- "preview_test_config(test_config)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now we can pass the input configuration to `vm.run_documentation_tests()` and run the full suite of tests. The variable `full_suite` then holds the result of these tests.\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "full_suite = vm.run_documentation_tests(config=test_config)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc7_'></a>\n",
- "\n",
- "## Next steps\n",
- "\n",
- "You can look at the results of this test suite right in the notebook where you ran the code, as you would expect. But there is a better way — use the ValidMind Platform to work with your model documentation.\n",
- "\n",
- "<a id='toc7_1_'></a>\n",
- "\n",
- "### Work with your model documentation\n",
- "\n",
- "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n",
- "\n",
- "2. Click and expand the **Model Development** section.\n",
- "\n",
- "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n",
- "\n",
- "<a id='toc7_2_'></a>\n",
- "\n",
- "### Discover more learning resources\n",
- "\n",
- "We offer many interactive notebooks to help you document models:\n",
- "\n",
- "- [Run tests & test suites](https://docs.validmind.ai/developer/model-testing/testing-overview.html)\n",
- "- [Code samples](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)\n",
- "\n",
- "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id='toc8_'></a>\n",
- "\n",
- "## Upgrade ValidMind\n",
- "\n",
- "<div class=\"alert alert-block alert-info\" style=\"background-color: #B5B5B510; color: black; border: 1px solid #083E44; border-left-width: 5px; box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);border-radius: 5px;\">After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.</div>\n",
- "\n",
- "Retrieve the information for the currently installed version of ValidMind:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "%pip show validmind"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n",
- "\n",
- "```bash\n",
- "%pip install --upgrade validmind\n",
- "```"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "You may need to restart your kernel after running the upgrade package for changes to be applied."
- ]
- }
- ],
- "metadata": {
- "colab": {
- "provenance": []
- },
- "gpuClass": "standard",
- "kernelspec": {
- "display_name": "ValidMind Library",
- "language": "python",
- "name": "validmind"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.10.13"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
diff --git a/site/notebooks/templates/next-steps.ipynb b/site/notebooks/templates/next-steps.ipynb
index 8f0a427569..e87363fb69 100644
--- a/site/notebooks/templates/next-steps.ipynb
+++ b/site/notebooks/templates/next-steps.ipynb
@@ -7,7 +7,7 @@
"source": [
"## Next steps\n",
"\n",
- "You can look at the output produced by the ValidMind Library right in the notebook where you ran the code, as you would expect. But there is a better way: use the ValidMind Platform to work with your model documentation."
+ "You can look at the output produced by the ValidMind Library right in the notebook where you ran the code, as you would expect. But there is a better way — use the ValidMind Platform to work with your model documentation."
]
},
{
diff --git a/site/notebooks/tutorials/model_development/2-start_development_process.ipynb b/site/notebooks/tutorials/model_development/2-start_development_process.ipynb
index 939b907661..31bfd062e3 100644
--- a/site/notebooks/tutorials/model_development/2-start_development_process.ipynb
+++ b/site/notebooks/tutorials/model_development/2-start_development_process.ipynb
@@ -142,7 +142,7 @@
"In our below example, note that: \n",
"\n",
"- The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n",
- "- The ValidMind Library provides a wrapper to automatically load the dataset as a Pandas DataFrame object."
+ "- The ValidMind Library provides a wrapper to automatically load the dataset as a [Pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object. A Pandas Dataframe is a two-dimensional tabular data structure that makes use of rows and columns."
]
},
{
@@ -887,7 +887,7 @@
"\n",
"### Assign predictions\n",
"\n",
- "Once the model has been registered you can assign model predictions to the training and test datasets.\n",
+ "Once the model has been registered you can assign model predictions to the training and testing datasets.\n",
"\n",
"- The [`assign_predictions()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#assign_predictions) from the `Dataset` object can link existing predictions to any number of models.\n",
"- This method links the model's class prediction values and probabilities to our `vm_train_ds` and `vm_test_ds` datasets.\n",
@@ -959,7 +959,7 @@
"\n",
"- [x] Import a sample dataset\n",
"- [x] Identify which tests you might want to run with ValidMind\n",
- "- [x] Initialize ValidMind datasets\n",
+ "- [x] Initialize ValidMind datasets and model objects\n",
"- [x] Run individual tests\n",
"- [x] Utilize the output from tests you've run\n",
"- [x] Log test results from sets of or individual tests as evidence to the ValidMind Platform\n",
diff --git a/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb b/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb
index e38978aec3..58e4016929 100644
--- a/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb
+++ b/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb
@@ -896,7 +896,19 @@
"\n",
"### Work with your model documentation\n",
"\n",
- "Now that you've logged all your test results and generated a draft for your model documentation, head to the ValidMind Platform to make qualitative edits, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. **Learn more:** [Working with model documentation](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)"
+ "Now that you've logged all your test results and generated a draft for your model documentation, head to the ValidMind Platform to wrap up your model documentation. Continue to work on your model documentation by:\n",
+ "\n",
+ "- **Run and log more tests:** Use the skills you learned in this series of notebooks to run and log more individual tests, including custom tests, then insert them into your documentation as supplementary evidence. (Learn more: [`validmind.tests`](https://docs.validmind.ai/validmind/validmind/tests.html))\n",
+ "\n",
+ "- **Inserting additional test results:** Add **Test-Driven Blocks** under any relevant section of your model documentation. (Learn more: [Work with test results](https://docs.validmind.ai/guide/model-documentation/work-with-test-results.html))\n",
+ "\n",
+ "- **Making qualitative edits to your test descriptions:** Click on the description of any inserted test results to review and edit the ValidMind-generated test descriptions for quality and accuracy. (Learn more: [Working with model documentation](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html#add-or-edit-documentation))\n",
+ "\n",
+ "- **View guidelines:** In any section of your model documentation, click **ValidMind Insights** in the top right corner to reveal the Documentation Guidelines for each section to help guide the contents of your model documentation. (Learn more: [View documentation guidelines](https://docs.validmind.ai/guide/model-documentation/view-documentation-guidelines.html))\n",
+ "\n",
+ "- **Collaborate with other stakeholders:** Use the ValidMind Platform's real-time collaborative features to work seamlessly together with the rest of your organization, including model validators. Review suggested changes in your content blocks, work with versioned history, and use comments to discuss specific portions of your model documentation. (Learn more: [Collaborate with others](https://docs.validmind.ai/guide/model-documentation/collaborate-with-others.html))\n",
+ "\n",
+ "When your model documentation is complete and ready for review, submit it for approval from the same ValidMind Platform where you made your edits and collaborated with the rest of your organization, ensuring transparency and a thorough model development history. (Learn more: [Submit for approval](https://docs.validmind.ai/guide/model-documentation/submit-for-approval.html))"
]
},
{
diff --git a/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb b/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb
index 51bb9dc97b..747a5dd1e4 100644
--- a/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb
+++ b/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb
@@ -147,7 +147,7 @@
"In our below example, note that:\n",
"\n",
"- The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n",
- "- The ValidMind Library provides a wrapper to automatically load the dataset as a Pandas [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object."
+ "- The ValidMind Library provides a wrapper to automatically load the dataset as a [Pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object. A Pandas Dataframe is a two-dimensional tabular data structure that makes use of rows and columns."
]
},
{
@@ -564,7 +564,7 @@
"\n",
"## Documenting test results\n",
"\n",
- "Now that we've done some analysis on two different datasets, we can use ValidMind to easily document why certain things were done to our raw data with testing to support it. As we learned above, every test result returned by the `run_test()` function has a `.log()` method that can be used to send the test results to the ValidMind Platform.\n",
+ "Now that we've done some analysis on two different datasets, we can use ValidMind to easily document why certain things were done to our raw data with testing to support it. Every test result returned by the `run_test()` function has a `.log()` method that can be used to send the test results to the ValidMind Platform.\n",
"\n",
"When logging validation test results to the platform, you'll need to manually add those results to the desired section of the validation report. To demonstrate how to add test results to your validation report, we'll log our data quality tests and insert the results via the ValidMind Platform."
]
diff --git a/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb b/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb
index ba6958f286..83061a8273 100644
--- a/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb
+++ b/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb
@@ -520,7 +520,7 @@
"\n",
"## Running model evaluation tests\n",
"\n",
- "With everything ready for us, let's run the rest of our validation tests. We'll focus on comprehensive testing around model performance of both the champion and challenger models going forward as we've already verified the data quality of the datasets used to train the champion model."
+ "With our setup complete, let's run the rest of our validation tests. Since we have already verified the data quality of the dataset used to train our champion model, we will now focus on comprehensive performance evaluations of both the champion and challenger models."
]
},
{
@@ -584,7 +584,10 @@
"\n",
"#### Evaluate performance of the champion model\n",
"\n",
- "Now, let's run and log our batch of model performance tests using our testing dataset (`vm_test_ds`) for our champion model:"
+ "Now, let's run and log our batch of model performance tests using our testing dataset (`vm_test_ds`) for our champion model:\n",
+ "\n",
+ "- The test set serves as a proxy for real-world data, providing an unbiased estimate of model performance since it was not used during training or tuning.\n",
+ "- The test set also acts as protection against selection bias and model tweaking, giving a final, more unbiased checkpoint."
]
},
{
@@ -725,9 +728,9 @@
"\n",
"### Run diagnostic tests\n",
"\n",
- "Next we want to inspect the robustness and stability testing comparison between our champion and challenger model.\n",
+ "Next, we want to inspect the robustness and stability testing comparison between our champion and challenger model.\n",
"\n",
- "Use `list_tests()` to identify all the model diagnosis tests for classification:"
+ "Use `list_tests()` to list all available diagnosis tests applicable to classification tasks:"
]
},
{
@@ -743,9 +746,12 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Let's see if models suffer from any *overfit* potentials and also where there are potential sub-segments of issues with the [`OverfitDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/OverfitDiagnosis.html). \n",
+ "Let’s now assess the models for potential signs of *overfitting* and identify any sub-segments where performance may inconsistent with the [`OverfitDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/OverfitDiagnosis.html).\n",
+ "\n",
+ "Overfitting occurs when a model learns the training data too well, capturing not only the true pattern but noise and random fluctuations resulting in excellent performance on the training dataset but poor generalization to new, unseen data:\n",
"\n",
- "Overfitting occurs when a model learns the training data too well, capturing not only the true pattern but noise and random fluctuations resulting in excellent performance on the training dataset but poor generalization to new, unseen data."
+ "- Since the training dataset (`vm_train_ds`) was used to fit the model, we use this set to establish a baseline performance for how well the model performs on data it has already seen.\n",
+ "- The testing dataset (`vm_test_ds`) was never seen during training, and here simulates real-world generalization, or how well the model performs on new, unseen data. "
]
},
{
@@ -767,9 +773,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Let's also conduct *robustness* and *stability* testing of the two models with the [`RobustnessDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/RobustnessDiagnosis.html).\n",
+ "Let's also conduct *robustness* and *stability* testing of the two models with the [`RobustnessDiagnosis` test](https://docs.validmind.ai/tests/model_validation/sklearn/RobustnessDiagnosis.html). Robustness refers to a model's ability to maintain consistent performance, and stability refers to a model's ability to produce consistent outputs over time across different data subsets.\n",
"\n",
- "Robustness refers to a model's ability to maintain consistent performance, and stability refers to a model's ability to produce consistent outputs over time across different data subsets."
+ "Again, we'll use both the training and testing datasets to establish baseline performance and to simulate real-world generalization:"
]
},
{
@@ -811,6 +817,13 @@
"FI"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We'll only use our testing dataset (`vm_test_ds`) here, to provide a realistic, unseen sample that mimic future or production data, as the training dataset has already influenced our model during learning:"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb b/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb
index 886f9e9061..ffdc4522a2 100644
--- a/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb
+++ b/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb
@@ -1144,13 +1144,17 @@
"\n",
"- **Inserting additional test results:** Click **Link Evidence to Report** under any section of 2. Validation in your validation report. (Learn more: [Link evidence to reports](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#link-evidence-to-reports))\n",
"\n",
- "- **Making qualitative edits to your test descriptions:** Expand any linked evidence under Validator Evidence and click **See evidence details** to review and edit the ValidMind-generated test descriptions for quality and accuracy.\n",
+ "- **Making qualitative edits to your test descriptions:** Expand any linked evidence under Validator Evidence and click **See evidence details** to review and edit the ValidMind-generated test descriptions for quality and accuracy. (Learn more: [Preparing validation reports](https://docs.validmind.ai/guide/model-validation/preparing-validation-reports.html#get-started))\n",
"\n",
"- **Adding more findings:** Click **Link Finding to Report** in any validation report section, then click **+ Create New Finding**. (Learn more: [Add and manage model findings](https://docs.validmind.ai/guide/model-validation/add-manage-model-findings.html))\n",
"\n",
- "- **Adding risk assessment notes:** Click under **Risk Assessment Notes** in any validation report section to access the text editor and content editing toolbar, including an option to generate a draft with AI. Edit your ValidMind-generated test descriptions (Learn more: [Work with content blocks](https://docs.validmind.ai/guide/model-documentation/work-with-content-blocks.html#content-editing-toolbar))\n",
+ "- **Adding risk assessment notes:** Click under **Risk Assessment Notes** in any validation report section to access the text editor and content editing toolbar, including an option to generate a draft with AI. Once generated, edit your ValidMind-generated test descriptions to adhere to your organization's requirements. (Learn more: [Work with content blocks](https://docs.validmind.ai/guide/model-documentation/work-with-content-blocks.html#content-editing-toolbar))\n",
"\n",
- "- **Assessing compliance:** Under the Guideline for any validation report section, click **ASSESSMENT** and select the compliance status from the drop-down menu. (Learn more: [Provide compliance assessments](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#provide-compliance-assessments))"
+ "- **Assessing compliance:** Under the Guideline for any validation report section, click **ASSESSMENT** and select the compliance status from the drop-down menu. (Learn more: [Provide compliance assessments](https://docs.validmind.ai/guide/model-validation/assess-compliance.html#provide-compliance-assessments))\n",
+ "\n",
+ "- **Collaborate with other stakeholders:** Use the ValidMind Platform's real-time collaborative features to work seamlessly together with the rest of your organization, including model developers. Propose suggested changes in the model documentation, work with versioned history, and use comments to discuss specific portions of the model documentation. (Learn more: [Collaborate with others](https://docs.validmind.ai/guide/model-documentation/collaborate-with-others.html))\n",
+ "\n",
+ "When your validation report is complete and ready for review, submit it for approval from the same ValidMind Platform where you made your edits and collaborated with the rest of your organization, ensuring transparency and a thorough model validation history. (Learn more: [Submit for approval](https://docs.validmind.ai/guide/model-documentation/submit-for-approval.html))"
]
},
{
diff --git a/site/releases/2023/2023-sep-27/highlights.qmd b/site/releases/2023/2023-sep-27/highlights.qmd
index 0dc0d41631..8f36bd2901 100644
--- a/site/releases/2023/2023-sep-27/highlights.qmd
+++ b/site/releases/2023/2023-sep-27/highlights.qmd
@@ -125,7 +125,7 @@ We also updated the QuickStart notebook to have a consistent experience.
:::
::: {.w-40-ns}
-[Quickstart for model documentation](/notebooks/quickstart_customer_churn_full_suite.ipynb){.button .button-green}
+[Quickstart for model documentation](/notebooks/quickstart/quickstart_model_documentation.ipynb){.button .button-green}
:::
@@ -343,18 +343,18 @@ We enhanced the architecture and content of our external docs site to make the u
:::: {.flex .flex-wrap .justify-around}
-::: {.w-30-ns}
+::: {.w-30-ns .tc}
[Get started](/get-started/get-started.qmd){.button}
:::
-::: {.w-50-ns}
+::: {.w-50-ns .tc}
[{{< var validmind.developer >}}](/developer/validmind-library.qmd){.button}
:::
::::
-
+
### Site enhancements
<!---Docs site improvements by @nrichers in [#132](https://github.com/validmind/documentation/pull/132)--->
diff --git a/site/training/assets/validmind-slideover.css b/site/training/assets/validmind-slideover.css
new file mode 100644
index 0000000000..06b9be80eb
--- /dev/null
+++ b/site/training/assets/validmind-slideover.css
@@ -0,0 +1,80 @@
+/* Global variables */
+:root {
+ --slideover-default-bg: #FAFAFA;
+ --slideover-default-text: #083e44;
+ --slideover-default-border: 1px solid #083e44;
+ --slideover-default-links: #DE257E;
+ --slideover-default-accent: #3E6C69;
+}
+
+/* Global slideover styling */
+.slideover__content {
+ background: var(--slideover-default-bg) !important;
+ color: var(--slideover-default-text) !important;
+ border: var(--slideover-default-border) !important;
+}
+
+/* Right-aligned emphasis border */
+.slideover--r.slideover__content {
+ border-left-width: 5px !important;
+}
+
+/* Left-aligned emphasis border */
+.slideover--l.slideover__content {
+ border-right-width: 5px !important;
+}
+
+/* Top-aligned emphasis border */
+.slideover--t.slideover__content {
+ border-bottom-width: 5px !important;
+}
+
+/* Bottom-aligned emphasis border */
+.slideover--b.slideover__content {
+ border-top-width: 5px !important;
+}
+
+/* Link styling */
+.slideover__content-area a {
+ color: var(--slideover-default-links);
+ text-decoration: none;
+}
+
+.slideover__content-area a:hover {
+ text-decoration: underline 2px solid var(--slideover-default-text);
+}
+
+/* Callout embed styling */
+.slideover__content-area .embed {
+ background-color: #f5fcfd;
+ border: 1px solid rgba(8, 62, 68, 0.2);
+}
+
+/* Emphasis styling */
+.slideover__content-area strong,
+.slideover__content-area b,
+.slideover__content-area i,
+.slideover__content-area em {
+ color: var(--slideover-default-accent) !important;
+}
+
+/* Quotation styling */
+q {
+ color: var(--slideover-default-accent) !important;
+}
+
+.slideover__content-area blockquote {
+ border-left: 4px solid var(--slideover-default-accent) !important;
+ color: var(--slideover-default-accent) !important;
+}
+
+/* Code styling */
+.slideover__content-area div.sourceCode {
+ background-color: var(--slideover-default-bg) !important;
+}
+
+.slideover__content-area code {
+ color: var(--slideover-default-text) !important;
+ background-color: #EAF8FA;
+ border: 1px solid rgba(8, 62, 68, 0.2);
+}
diff --git a/site/training/developer-fundamentals/finalizing-model-documentation.qmd b/site/training/developer-fundamentals/finalizing-model-documentation.qmd
index 58149929bd..0781067f91 100644
--- a/site/training/developer-fundamentals/finalizing-model-documentation.qmd
+++ b/site/training/developer-fundamentals/finalizing-model-documentation.qmd
@@ -16,6 +16,8 @@ format:
view-distance: 2
logo: /validmind.png
footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)"
+ revealjs-plugins:
+ - slideover
html:
# Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml
output-file: _finalizing-model-documentation.html
@@ -150,23 +152,22 @@ From the {{< var validmind.platform >}}:
{{< include /guide/model-documentation/_view-test-result-metadata.qmd >}}
-
-
:::
+
::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Review model documentation**
+:::
-::: {.f6}
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**.
-3. Click into any section of the documentation to review the test results logged via the {{< var validmind.developer >}}. <br>
- For example: **2.3 Correlations and Interactions** / **3.2 Model Evaluation**
-
-:::
+3. Click into any section of the documentation to review the test results logged via the {{< var validmind.developer >}}. <br>For example:
+ - **2.3 Correlations and Interactions**
+ - **3.2 Model Evaluation**
When you're done taking a look around, click [{{< fa chevron-right >}}]() to continue.
@@ -218,18 +219,20 @@ For example:
:::
:::
+
::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Add & edit content blocks**
-
-::: {.f6}
-From your model's **{{< fa book-open >}} Documentation**, click into any section of the documentation to add and edit a content block.
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. On the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**.
+3. Click into any section of the documentation to add and edit a content block.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -278,16 +281,15 @@ To transition through the approval workflow, all required workflow steps must be
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Submit for approval**
+:::
-::: {.f6}
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. Locate the **[model status]{.smallcaps}** section.
3. Open up the status transition panel, enter your **[notes]{.smallcaps}** and any other additional inventory fields, then click **Submit**.
-:::
-
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -325,14 +327,15 @@ Comment threads
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Comment on model documentation**
-
-::: {.f6}
-**In the content block you added earlier** within your model's **{{< fa book-open >}} Documentation**: Post a comment, reply to it, and then resolve the thread.
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. On the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**.
+3. **In the content block you added earlier**: Post a comment, reply to it, and then resolve the thread.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -366,15 +369,14 @@ Track changes & updates
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Review model activity**
+:::
-::: {.f6}
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. In the left sidebar that appears for your model, click **{{< fa wifi >}} Model Activity**.
-3. Filter the following activity: **Comments** | **Status Updates** | **Model Updates** | **Test Results**
-
-:::
+3. Filter the following activity: <br>**Comments** | **Status Updates** | **Model Updates** | **Test Results**
When you're done, click [{{< fa chevron-right >}}]() to continue.
@@ -384,21 +386,20 @@ When you're done, click [{{< fa chevron-right >}}]() to continue.
## {background-iframe="https://app.prod.validmind.ai/analytics" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--l .three-quarters .auto-collapse-10}
**Welcome to {{< fa square-poll-vertical >}} Analytics**
-::: {.f6}
-Here, you can find executive summaries, track information on models, findings, and more. For example:
+Here, you can find executive summaries, track information on models, findings, and more.
+
+For example:
1. Click **Models** to review reports on models.
3. Click into any widget to review the models reported by that widget.
-:::
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
-
# In summary {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
## {.scrollable .center}
diff --git a/site/training/developer-fundamentals/implementing-custom-tests.qmd b/site/training/developer-fundamentals/implementing-custom-tests.qmd
index e575d8d37d..444dcb6a90 100644
--- a/site/training/developer-fundamentals/implementing-custom-tests.qmd
+++ b/site/training/developer-fundamentals/implementing-custom-tests.qmd
@@ -16,6 +16,8 @@ format:
view-distance: 2
logo: /validmind.png
footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)"
+ revealjs-plugins:
+ - slideover
html:
# Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml
output-file: _implementing-custom-tests.html
@@ -183,14 +185,11 @@ Let's continue our journey with **Section 1** on the next page. {{< fa hand-poin
## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html" background-interactive="yes" data-preload="yes"}
-:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay}
+:::: {.slideover--r .three-quarters}
**3 — Integrate custom tests**
-::: {.f6}
This is the third notebook in our introductory series, which will walk you through how to implement different types of custom tests with {{< var vm.product >}}.
-:::
-
**Scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -222,7 +221,7 @@ This is the third notebook in our introductory series, which will walk you throu
::: {.f5}
{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your developer environment:
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **Getting Started**.
3. Locate the code snippet and click **Copy snippet to clipboard**.
@@ -243,17 +242,14 @@ Make sure you're logged in and have refreshed the page in a Chromium-based web b
## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#initialize-the-validmind-library" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Connect to your model**
-::: {.f6}
With your code snippet copied to your clipboard:
1. Open **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"}
2. Run all the cells under the **Setting up** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -281,33 +277,29 @@ Let's implement a custom *inline test* that calculates the confusion matrix for
## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#create-a-confusion-matrix-plot" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Create a custom test**
-::: {.f6}
The [`@vm.test` wrapper](/validmind/validmind.qmd#test){target="_blank"} allows you to create a reusable test:
1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"}
2. Run all the cells in the following section under Implementing a custom inline test: **Create a confusion matrix plot**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#add-parameters-to-custom-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Adjust your custom test**
-::: {.f6}
Custom tests can take parameters just like any other function:
1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"}
-2. Run all the cells in the following sections under Implementing a custom inline test: **Add parameters to custom tests** / **Pass parameters to custom tests**
-
-:::
+2. Run all the cells in the following sections under Implementing a custom inline test:
+ - **Add parameters to custom tests**
+ - **Pass parameters to custom tests**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -315,22 +307,18 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#log-the-confusion-matrix-results" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Log your custom test**
-::: {.f6}
Use the [`.log()` method](/validmind/validmind/vm_models.qmd#log){target="_blank"} to send the results of your custom test to the {{< var validmind.platform >}}:
1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"}
2. Run the cell in the following section under Implementing a custom inline test: **Log the confusion matrix results**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
-
# Use external test providers {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
## {.scrollable .center}
@@ -356,51 +344,42 @@ Creating inline custom tests with a function is a great way to customize your mo
## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#create-custom-tests-folder" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Create custom tests folder**
-::: {.f6}
Create a new folder that will contain reusable custom tests from your existing inline tests:
1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"}
2. Run the cell under the following Using external test providers section: **Create custom tests folder**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#save-an-inline-test" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Save inline test**
-::: {.f6}
The `@vm.test` decorator also includes a convenience method that allows you to save the test to a Python file at a specified path:
1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"}
2. Run all the cells under the Using external test providers section: **Save an inline test**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_development/3-integrate_custom_tests.html#register-a-local-test-provider" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Register local test provider**
-::: {.f6}
Next, let's initialize a test provider that will tell the {{< var validmind.developer >}} where to find your saved custom tests:
1. Continue with **3 — Integrate custom tests**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/3-integrate_custom_tests.ipynb){target="_blank"}
2. Run all the cells under the Using external test providers section: **Register a local test provider**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -440,7 +419,7 @@ With the custom test results logged, let's head to the model we connected to at
5. Click **{{< fa plus >}}** and then select **Test-Driven Block** under [from library]{.smallcaps}:
- Click on **Custom** under [test-driven]{.smallcaps} in the left sidebar.
- - Select the two custom `ConfusionMatrix` tests you logged above:
+ - Select the two custom `ConfusionMatrix` tests you logged previously:
{fig-alt="Screenshot showing the ConfusionMatrix tests selected" .screenshot}
@@ -449,35 +428,37 @@ With the custom test results logged, let's head to the model we connected to at
Confirm that the two individual results for the confusion matrix tests have been correctly inserted into section **3.2 Model Evaluation** of the documentation.
:::
-::::
+::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory/" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Insert custom test-driven blocks**
-
-::: {.f6}
-3.2 Model Evaluation — `my_custom_tests.ConfusionMatrix:test_dataset_normalized` / `my_test_provider.ConfusionMatrix`
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. In the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**.
+3. Locate the Data Preparation section and click on **3.2 Model Evaluation** to expand that section.
+4. Hover under the Pearson Correlation Matrix content block until a horizontal dashed line with a **{{< fa plus >}}** button appears, indicating that you can insert a new block.
+5. Click **{{< fa plus >}}** and then select **Test-Driven Block** under [from library]{.smallcaps}:
+ - Click on **Custom** under [test-driven]{.smallcaps} in the left sidebar.
+ - Select the two custom `ConfusionMatrix` tests you logged previously.
+6. Finally, click **Insert 2 Test Results to Document** to add the test results to the documentation.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
-
-
# Section 2 {background-color="#083E44" background-image="/assets/img/about-us-esphere.svg"}
## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html" background-interactive="yes" data-preload="yes"}
-:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay}
+:::: {.slideover--r .three-quarters}
**4 — Finalize testing and documentation**
-::: {.f6}
This is the final notebook in our introductory series, which will walk you through wrapping custom test results into your documentation, as well as how to update the configuration for the entire model documentation template to suit your needs.
-:::
**Scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue.
@@ -510,7 +491,7 @@ This is the final notebook in our introductory series, which will walk you throu
::: {.f4}
As usual, let's connect back up to your model in the {{< var validmind.platform >}}:
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **Getting Started**.
3. Locate the code snippet and click **Copy snippet to clipboard**.
@@ -524,17 +505,14 @@ When you're done, click [{{< fa chevron-right >}}]() to continue.
## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html#initialize-the-validmind-library" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Connect to your model**
-::: {.f6}
With your code snippet copied to your clipboard:
1. Open **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"}
2. Run all the cells under the **Setting up** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -543,34 +521,28 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html#reconnect-to-validmind" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Reconnect to {{< var vm.product >}}**
-::: {.f6}
After you insert test-driven blocks, changes should persist and become available every time you call the [`preview_template()` function](/validmind/validmind.qmd#preview_template){target="_blank"}:
1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"}
2. Run all the cells under the **Reconnect to {{< var vm.product >}}** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html#include-custom-test-results" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Log custom test results**
-::: {.f6}
Now that your custom test IDs are part of your template, you can now run tests for an entire section and all additional custom tests will be loaded:
1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"}
2. Run the cell under the **Include custom test results** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -606,32 +578,26 @@ The utility function `vm.get_test_suite().get_default_config()` will return the
## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html#documentation-template-configuration" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Retrieve default configuration**
-::: {.f6}
1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"}
2. Run the first cell under the **Documentation template configuration** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_development/4-finalize_testing_documentation.html#update-the-config" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Update template configuration**
-::: {.f6}
The default configuration does not assign any inputs to a test, but you can assign inputs to individual tests as needed:
1. Continue with **4 — Finalize testing and documentation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/4-finalize_testing_documentation.ipynb){target="_blank"}
2. Run the all the cells under the following Documentation template configuration section: **Update the config**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
diff --git a/site/training/developer-fundamentals/learning-to-run-tests.qmd b/site/training/developer-fundamentals/learning-to-run-tests.qmd
index dfc280afcc..60ea84a981 100644
--- a/site/training/developer-fundamentals/learning-to-run-tests.qmd
+++ b/site/training/developer-fundamentals/learning-to-run-tests.qmd
@@ -16,6 +16,8 @@ format:
view-distance: 2
logo: /validmind.png
footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)"
+ revealjs-plugins:
+ - slideover
html:
# Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml
output-file: _learning-to-run-tests.html
@@ -148,14 +150,11 @@ Let's continue our journey with **2 — Start the model development process** on
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html" background-interactive="yes" data-preload="yes"}
-:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay}
+:::: {.slideover--r .three-quarters}
**2 — Start the model development process**
-::: {.f6}
During this course, we'll run through these notebooks together, and at the end of your learning journey you'll have a fully documented sample model ready for review.
-:::
-
For now, **scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -166,21 +165,16 @@ For now, **scroll through this notebook** to explore. When you are done, click [
## {background-iframe="/developer/model-testing/test-descriptions.html" background-interactive="true" data-preload="yes"}
-::: footer
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f4 .tc .pl4 .overlay}
+:::: {.slideover--l .three-quarters .auto-collapse-5}
**{{< var vm.product >}} test repository**
-::: {.f5}
{{< var vm.product >}} provides a wealth out-of-the-box of tests to help you ensure that your model is being built appropriately.
In this module, you'll become familiar with the individual tests available in {{< var vm.product >}}, as well as how to run them and change parameters as necessary.
-:::
-
For now, **scroll through these test descriptions** to explore. When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
-:::
## Get your code snippet
@@ -209,7 +203,7 @@ For now, **scroll through these test descriptions** to explore. When you're done
::: {.f5}
{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your developer environment:
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **Getting Started**.
3. Locate the code snippet and click **Copy snippet to clipboard**.
@@ -230,16 +224,15 @@ Make sure you're logged in and have refreshed the page in a Chromium-based web b
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#initialize-the-validmind-library" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Connect to your model**
-::: {.f6}
With your code snippet copied to your clipboard:
1. Open **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
-2. Run the following cells in the Setting up section: **Initialize the {{< var validmind.developer >}}** / **Import sample dataset**.
-
-:::
+2. Run the following cells in the Setting up section:
+ - **Initialize the {{< var validmind.developer >}}**
+ - **Import sample dataset**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -247,33 +240,28 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#identify-qualitative-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Identify qualitative tests**
-::: {.f6}
Next, we'll use the [`list_tests()` function](/notebooks/EXECUTED/model_development/1-set_up_validmind.ipynb#explore-available-tests){target="_blank"} to pinpoint tests we want to run:
1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
2. Run all the cells under the Setting up section: **Identify qualitative tests**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
+
::::
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#initialize-the-validmind-datasets" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Initialize {{< var vm.product >}} datasets**
-::: {.f6}
Then, we'll use the [`init_dataset()` function](/validmind/validmind.qmd#init_dataset){target="_blank"} to connect the sample data with a {{< var vm.product >}} `Dataset` object in preparation for running tests:
1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
2. Run the following cell in the Setting up section: **Initialize the {{< var vm.product >}} datasets**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -284,16 +272,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#run-tabular-data-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Run tabular data tests**
-::: {.f6}
You run individual tests by calling the [`run_test()` function](/validmind/validmind/tests.qmd#run_test){target="_blank"} provided by the `validmind.tests` module:
1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
-2. Run all the cells under the Running tests section: **Run tabular data tests**.
-
-:::
+2. Run all the cells under the Running tests section: **Run tabular data tests**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -301,16 +286,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#utilize-test-output" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Utilize test output**
-::: {.f6}
You can utilize the output from a ValidMind test for further use, for example, if you want to remove highly correlated features:
1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
-2. Run all the cells under the Running tests section: **Utilize test output**.
-
-:::
+2. Run all the cells under the Running tests section: **Utilize test output**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -345,15 +327,13 @@ Every test result returned by the `run_test()` function has a `.log()` method th
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#run-and-log-multiple-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Run & log multiple tests**
-::: {.f6}
The [`run_documentation_tests()` function](/validmind/validmind.qmd#run_documentation_tests){target="_blank"} allows you to run multiple tests at once and automatically log the results to your documentation:
1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
-2. Run the following cell in the Documenting results section: **Run and log multiple tests**.
-:::
+2. Run the following cell in the Documenting results section: **Run and log multiple tests**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -361,16 +341,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#run-and-log-an-individual-test" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Run & log an individual test**
-::: {.f6}
Next, we'll run an individual test and log the result to the {{< var validmind.platform >}}:
1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
-2. Run the following cell in the Running tests section: **Run and log an individual test**.
-
-:::
+2. Run the following cell in the Running tests section: **Run and log an individual test**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -416,22 +393,27 @@ With the test results logged, let's head to the model we connected to at the beg
:::
::::
-
## {background-iframe="https://app.prod.validmind.ai/model-inventory/" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Insert a test-driven block**
-
-::: {.f6}
-Within your model's **{{< fa book-open >}} Documentation**: 2.3 Correlations and Interactions — `HighPearsonCorrelation:balanced_raw_dataset`
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. In the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**.
+3. Locate the Data Preparation section and click on **2.3 Correlations and Interactions** to expand that section.
+4. Hover under the Pearson Correlation Matrix content block until a horizontal dashed line with a **{{< fa plus >}}** button appears, indicating that you can insert a new block.
+5. Click **{{< fa plus >}}** and then select **Test-Driven Block** under [from library]{.smallcaps}:
+ - Click on **VM Library** under [test-driven]{.smallcaps} in the left sidebar.
+ - In the search bar, type in `HighPearsonCorrelation`.
+ - Select `HighPearsonCorrelation:balanced_raw_dataset` as the test.
+5. Finally, click **Insert 1 Test Result to Document** to add the test result to the documentation.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
-
# Test an existing model {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
## {.scrollable .center}
@@ -475,15 +457,13 @@ In this next example, we’ll focus on running the tests within the Model Develo
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#train-simple-logistic-regression-model" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Train your model**
-::: {.f6}
Using {{< var vm.product >}} tests, we'll train a simple logistic regression model on our dataset and evaluate its performance:
1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
-2. Run all the cells under the Model testing section: **Train simple logistic regression model**.
-:::
+2. Run all the cells under the Model testing section: **Train simple logistic regression model**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -491,16 +471,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#initialize-model-evaluation-objects" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Initialize a model object**
-::: {.f6}
Use the `init_dataset()` and [`init_model()` functions](/validmind/validmind.qmd#init_model){target="_blank"} to initialize these objects:
1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
-2. Run the cell under the following Model testing section: **Initialize model evaluation objects**.
-
-:::
+2. Run the cell under the following Model testing section: **Initialize model evaluation objects**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -508,16 +485,13 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#assign-predictions" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Assign predictions**
-::: {.f6}
Use the [`assign_predictions()` method](/validmind/validmind/vm_models.qmd#assign_predictions){target="_blank"} from the `Dataset` object to link existing predictions to any number of models:
1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
-2. Run the cell under the following Model testing section: **Assign predictions**.
-
-:::
+2. Run the cell under the following Model testing section: **Assign predictions**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -525,23 +499,18 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_development/2-start_development_process.html#run-the-model-evaluation-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Run the model evaluation tests**
-::: {.f6}
Finally, we'll run only the tests within the Model Development section of the model documentation:
1. Continue with **2 — Start the model development process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/2-start_development_process.ipynb){target="_blank"}
-2. Run the cell under the following Model testing section: **Run the model evaluation tests**.
-
-:::
+2. Run the cell under the following Model testing section: **Run the model evaluation tests**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
-
-
# In summary {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
## {.scrollable .center}
diff --git a/site/training/developer-fundamentals/using-validmind-for-model-development.qmd b/site/training/developer-fundamentals/using-validmind-for-model-development.qmd
index 8d6885e295..c6706415b3 100644
--- a/site/training/developer-fundamentals/using-validmind-for-model-development.qmd
+++ b/site/training/developer-fundamentals/using-validmind-for-model-development.qmd
@@ -16,6 +16,8 @@ format:
view-distance: 2
logo: /validmind.png
footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)"
+ revealjs-plugins:
+ - slideover
html:
# Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml
output-file: _using-validmind-for-model-development.html
@@ -140,92 +142,70 @@ In this first module, we'll run through **1 — Set up the {{< var validmind.dev
Let's start our journey with **1 — Set up the {{< var validmind.developer >}}** on the next page. {{< fa hand-point-right >}}
-## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html" background-interactive="yes" data-preload="yes"}
+## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html" background-interactive="yes" data-preload="yes"}
+
+:::: {.slideover--r .three-quarters}
-:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay}
**1 — Set up the {{< var validmind.developer >}}**
-::: {.f6}
During this course, we'll run through these notebooks together, and at the end of your learning journey you'll have a fully documented sample model ready for review.
-:::
For now, **scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue.
::::
-<!-- USING THE VARIABLE IN THE HEADING MESSES UP THE PAGE ANCHOR -->
-
# Register a model {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
## {background-iframe="https://app.prod.validmind.ai" background-interactive="true" data-preload="yes"}
-::: {.fr .f5 .mv4 .nr5 .pa5 .overlay}
+:::: {.slideover--r}
**Welcome to the {{< var validmind.platform >}}**
From here, you can:
-- Keep track of your models in the <br>customizable inventory ...
-- Review and edit model documentation <br>generated via the {{< var validmind.developer >}} ...
-- Collaborate with model validators <br>to get your model approved ...
+- Keep track of your models in the customizable inventory ...
+- Review and edit model documentation generated via the {{< var validmind.developer >}} ...
+- Collaborate with model validators to get your model approved ...
- ... and much more!
-::: {.f6 .pl3 .pr3 .embed}
+::: {.embed}
**Can't load the {{< var validmind.platform >}}?**
-Make sure you're logged in and have <br>refreshed the page in a Chromium-based <br>web browser.
-:::
-
-When you're done navigating around,
-<br>click [{{< fa chevron-right >}}]() to continue.
+Make sure you're logged in and have refreshed the page in a Chromium-based web browser.
:::
+When you're done navigating around, click [{{< fa chevron-right >}}]() to continue.
+
+::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.fr .f5 .mv6 .nr5 .pa4 .overlay}
+:::: {.slideover--r}
**Welcome to the {{< fa cubes >}} Inventory**
-Use the model inventory to track <br>
-comprehensive details for all your <br>
-models throughout the model lifecycle.
-
-The model inventory is customizable <br>
-and extensible, with a layout that <br>
-can be configured to suit your needs.
+Use the model inventory to track comprehensive details for all your models throughout the model lifecycle. The model inventory is customizable and extensible, with a layout that can be configured to suit your needs.
::: {.f6 .pl3 .pr3 .embed}
-To start the documentation process, <br>
-a model must already be registered <br>
-in the model inventory via the <br>
-**{{< fa plus >}} Register Model** modal.
+To start the documentation process, a model must already be registered in the model inventory via the **{{< fa plus >}} Register Model** modal.
:::
-**Let's register a model together** on <br>
-the next page. {{< fa hand-point-right >}}
+**Let's register a model together** on the next page. {{< fa hand-point-right >}}
::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory/?register=open" background-interactive="true" data-preload="yes"}
-:::: {.fr .f5 .mv5 .nr5 .pa4 .overlay}
-**Register a binary <br>classification model**
-
-1. Select the option <br>
-for a new model:
+:::: {.slideover--r .three-quarters .auto-collapse-10}
+**Register a binary classification model**
-::: {.f6 .nt2 .pl2}
+1. Select the option for a new model:
- **Documentation template** — <br>`Binary classification`
-- **Use case** — <br>`Attrition/Churn Management`
-
- You can fill in other options<br> according to your preference.
-:::
-
-2. Click **{{< fa plus >}} Register Model** to <br>
-add the model to your <br>
-inventory.
+- **Use case** — <br>`Attrition/Churn Management`<br><br>
+ You can fill in other options according to your preference.
+2. Click **{{< fa plus >}} Register Model** to add the model to your inventory.
-When you're done, <br>click [{{< fa chevron-right >}}]() to continue.
+When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -260,7 +240,7 @@ When you're done, <br>click [{{< fa chevron-right >}}]() to continue.
::: {.f5}
{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your developer environment:
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **Getting Started**.
3. Locate the code snippet and click **Copy snippet to clipboard**.
@@ -281,17 +261,14 @@ Make sure you're logged in and have refreshed the page in a Chromium-based web b
## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html#install-the-validmind-library" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Install & initialize the {{< var validmind.developer >}}**
-::: {.f6}
With your code snippet copied to your clipboard:
1. Open **1 — Set up the {{< var validmind.developer >}}**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/1-set_up_validmind.ipynb){target="_blank"}
2. Run all the cells in the sections under **Initializing the {{< var validmind.developer >}}**.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -302,17 +279,14 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html#preview-the-documentation-template" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Preview the documentation template**
-::: {.f6}
You can preview your model's documentation template right from the {{< var validmind.developer >}}:
1. Continue with **1 — Set up the {{< var validmind.developer >}}**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_development/1-set_up_validmind.ipynb){target="_blank"}
2. Run all the cells in the sections under **Getting to know ValidMind**.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -338,24 +312,23 @@ Try it **live** on the next page. {{< fa hand-point-right >}}
{{< include /guide/model-documentation/_review-model-documentation.qmd >}}
-
:::
+
::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Verify the documentation template**
+:::
-::: {.f6}
Once you've called `preview_template()`:
-1. From the model inventory, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**.
3. Note how the structure of the Document Overview reflects the previewed template.
-:::
-
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
diff --git a/site/training/training-templates/course-slides.qmd b/site/training/training-templates/course-slides.qmd
index 8635000fb0..c9c79f4e75 100644
--- a/site/training/training-templates/course-slides.qmd
+++ b/site/training/training-templates/course-slides.qmd
@@ -18,6 +18,8 @@ format:
view-distance: 2
logo: /validmind.png
footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)"
+ revealjs-plugins:
+ - slideover
html:
# Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml
output-file: _course-slides.html
@@ -42,29 +44,36 @@ This {{#th}} module is part of a {{#}}-part series:
## In this course {.center}
-:::: {.columns .f4}
+<!-- :::: {.columns .f4}
::: {.column width="50%" .mt4 .pr4}
### Introduction
- [Before you begin](#before-you-begin)
:::
-::::
+:::: -->
:::: {.columns .f4}
-::: {.column width="50%" .mt4 .pr4}
+::: {.column width="20%" .mt4 .pr4}
+### Introduction
+- [Before you begin](#before-you-begin)
+
+:::
+
+::: {.column width="35%" .mt4 .pr4}
### Section 1
1. [iFrame embed right](#iframe-embed-right)
2. [iFrame embed bottom](#iframe-embed-bottom)
-3. [iFrame embed footer](#iframe-embed-footer)
-4. [Scrollable single user guide](#scrollable-single-user-guide)
-5. [Scrollable tabset user guides](#scrollable-tabset-user-guide)
+3. [iFrame embed left](#iframe-embed-left)
+4. [iFrame embed top](#iframe-embed-top)
+5. [Scrollable single user guide](#scrollable-single-user-guide)
+6. [Scrollable tabset user guides](#scrollable-tabset-user-guide)
:::
-::: {.column width="50%" .mt4}
+::: {.column width="45%" .mt4}
### Section 2
Example embedded `iframe` for interactive copy/paste workaround:
@@ -122,7 +131,7 @@ Already logged in and refreshed this module? Click [{{< fa chevron-right >}}]()
Be sure to return to this page afterwards.
:::
-2. After you successfully log in, refresh the page to connect this training module up to the {{< var validmind.platform >}}:
+2. After you successfully log in, refresh the page to connect this training module up to the {{< var validmind.platform >}}:
::: {.tc}
<button class="button" onClick="window.location.reload();">Refresh Page</button>
@@ -138,48 +147,82 @@ Be sure to return to this page afterwards.
## {background-iframe="https://app.prod.validmind.ai/" data-preload="yes"}
-:::: {.fr .f5 .mv5 .nr4 .pa5 .overlay}
-This is a right-aligned modal `.overlay` <br>on top of the live {{< var vm.product >}} Platform.
+:::: {.slideover--r}
+This is a collapsible right-aligned modal (`slideover--r`) on top of the live {{< var vm.product >}} Platform.
+
+- This modal uses a version of the [`nrichers/slideover` extension](https://github.com/nrichers/slideover) ...
+- ... with the styling modified to mimic the style of the docs site callouts.
+- **Bold text shows up in a different colour.** So do *italics*.
+- You can use an additional callout style with this class: `.embed`
+
+::: {.embed}
+This is the nested embed, that has its own styling. Think of it as a callout within the slideovers.
-- It mimics the style of the docs site callouts.
-- **Bold text shows up in a different colour.**
-- You can use an additional callout style <br>with this class: `.embed`
+Speaking of important information in callouts ... if you update the extension, **make sure to reapply our custom CSS template from `/training/assets/validmind-slideover.css` to the extensions `custom-slideover.css`!**
-::: {.f6 .pl3 .pr3 .embed}
-**This is the nested embed**, <br>which also has its own bold styling.
:::
-As you can see, you need to get creative <br>with some of your `<br>`s to determine <br>the max-width of the overlay box.
+- And code blocks look like this:
+ ```python
+ import validmind as vm
+
+ vm.init(
+ # api_host="...",
+ # api_key="...",
+ # api_secret="...",
+ # model="...",
+ )
+ ```
::::
# iFrame embed bottom {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
-## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html" background-interactive="yes" data-preload="yes"}
+## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html" background-interactive="yes" data-preload="yes"}
-:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay}
-**This is a bottom-aligned modal `.overlay` on top of a rendered site file.**
+:::: {.slideover--b}
+**This is a collapsible bottom-aligned modal (`slideover--b`) on top of a rendered site file.**
+
+When you add `background-interactive="yes"` to the `background-iframe` element, users can fully interact with the page contents.
-::: {.f6}
-You can fully interact with the page contents.
-:::
::::
-# iFrame embed footer {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
+# iFrame embed left {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
-## {background-iframe="https://app.prod.validmind.ai/settings" background-interactive="true" data-preload="yes"}
+## {background-iframe="https://app.prod.validmind.ai/" data-preload="yes"}
-<!-- Avoid overlapping with UI elements by moving instructions into footer -->
-::: footer
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f3 .tc .pl4 .overlay}
-Sometimes you'll need to move the `overlay` to the footer in order to avoid overlap with UI elements.
+:::: {.slideover--l .auto-collapse .three-quarters}
+**This is a collapsible left-aligned modal (`slideover--l`) at three-quarters size on top of the live {{< var vm.product >}} Platform.**
-::: {.f4}
-This will also stretch the overlay to 95%.
-:::
+When you add `.auto-collapse` to the slideover, the modal will auto-collapse after five seconds for that slide.
+
+You can adjust the size of the modals, for more or less screen real estate:
+
+- `.three-quarters` — 3/4s or 75% of the default width.
+- `.half` — 1/2s or 50% of the default width.
+- `.third` — 1/3 or 33ish% of the default width.
+
+The box will also automatically scroll once past a certain length (a percentage of your screen ... if you don't see some scrolling, make your browser window smaller!)...
+
+> Eternal-return right chaos contradict strong enlightenment deceptions virtues strong. Love contradict marvelous of faith. Burying moral battle noble moral strong inexpedient fearful.
+>
+> Passion burying spirit moral dead endless madness hope. Joy play free value mountains prejudice derive decrepit.
+>
+> Burying faithful dead truth faithful law love zarathustra love. Truth ocean of superiority ultimate battle mountains prejudice enlightenment love passion merciful truth faithful.
+>
+> Ultimate pinnacle moral depths revaluation grandeur revaluation evil. Convictions spirit.
::::
-:::
+
+# iFrame embed top {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
+
+## {background-iframe="/notebooks/EXECUTED/model_development/1-set_up_validmind.html" background-interactive="yes" data-preload="yes"}
+
+:::: {.slideover--t .auto-collapse-10}
+**This is a collapsible top-aligned modal (`slideover--t`) on top of a rendered site file.**
+
+When you add `.auto-collapse-10` to the slideover, the modal will auto-collapse after 10 seconds for that slide.
+::::
# Scrollable single user guide {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
@@ -293,7 +336,7 @@ Try it **live** on the next page. {{< fa hand-point-right >}}
::: {.f5}
{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your developer environment:
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **Getting Started**.
3. Locate the code snippet and click **Copy snippet to clipboard**.
diff --git a/site/training/training.qmd b/site/training/training.qmd
index 9eb6d505cd..b52ee5b4a2 100644
--- a/site/training/training.qmd
+++ b/site/training/training.qmd
@@ -40,7 +40,7 @@ listing:
table-hover: true
image-align: left
contents:
- - path: https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/quickstart_customer_churn_full_suite.ipynb
+ - path: https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/quickstart/quickstart_model_documentation.ipynb
title: "Quickstart for model documentation"
subtitle: "Open notebook in JupyterHub {{< fa chevron-right >}}"
description: "Gets you started with the basic process of documenting models with {{< var vm.product >}}, from the {{< var vm.developer >}} to the {{< var vm.platform >}}."
diff --git a/site/training/validator-fundamentals/developing-challenger-models.qmd b/site/training/validator-fundamentals/developing-challenger-models.qmd
index c61eb49524..59e7f77b07 100644
--- a/site/training/validator-fundamentals/developing-challenger-models.qmd
+++ b/site/training/validator-fundamentals/developing-challenger-models.qmd
@@ -16,6 +16,8 @@ format:
view-distance: 2
logo: /validmind.png
footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)"
+ revealjs-plugins:
+ - slideover
html:
# Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml
output-file: _developing-challenger-models.html
@@ -183,14 +185,11 @@ Let's continue our journey with **Section 1** on the next page. {{< fa hand-poin
## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html" background-interactive="yes" data-preload="yes"}
-:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay}
+:::: {.slideover--r .three-quarters}
**3 — Developing a potential challenger model**
-::: {.f6}
This is the third notebook in our introductory series, which will walk you through how to evaluate your champion model against a potential challenger with {{< var vm.product >}}.
-:::
-
**Scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -222,7 +221,7 @@ This is the third notebook in our introductory series, which will walk you throu
::: {.f5}
{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your validation environment:
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **Getting Started**.
3. Locate the code snippet and click **Copy snippet to clipboard**.
@@ -243,34 +242,28 @@ Make sure you're logged in and have refreshed the page in a Chromium-based web b
## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#initialize-the-validmind-library" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Connect to your model**
-::: {.f6}
With your code snippet copied to your clipboard:
1. Open **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"}
2. Run all the cells under the **Setting up** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#import-the-champion-model" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Import the champion model**
-::: {.f6}
Next, let's import the champion model submitted by the model development team in the format of a `.pkl` file for evaluation:
1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"}
2. Run the cell under the **Import the champion model** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -315,34 +308,28 @@ We're curious how an alternate model compares to our champion model, so let's tr
## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#training-a-potential-challenger-model" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Random forest classification model**
-::: {.f6}
Let's train our potential challenger model:
1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"}
2. Run the cell under the following Training a potential challenger model section: **Random forest classification model**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#initializing-the-model-objects" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Initialize the model objects**
-::: {.f6}
In addition to the initialized datasets, you'll also need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data for each of our two models using [the `vm.init_model()` method](/validmind/validmind.qmd#init_model){target="_blank"}:
1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"}
2. Run all the cells under the section **Initializing the model objects**.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -389,17 +376,14 @@ Finally, we want to verify the relative influence of different input features on
## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#run-model-performance-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Run model performance tests**
-::: {.f6}
Use the [`list_tests()` function](/validmind/validmind/tests.qmd#list_tests){target="_blank"} to identify all the model performance tests for classification:
1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"}
2. Run all the cells under the Running model evaluation tests section: **Run model performance tests**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -428,7 +412,7 @@ As we can observe from the output in our notebook, our champion model doesn't pa
#### Create a finding via a validation report
-1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.
+1. From the **{{< fa cubes >}} Inventory** in the ValidMind Platform, go to the model you connected to earlier.
2. In the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**.
@@ -464,48 +448,49 @@ As we can observe from the output in our notebook, our champion model doesn't pa
## {background-iframe="https://app.prod.validmind.ai/model-inventory/" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Create a model finding**
-::: {.f6}
-Within your **{{< fa shield >}} Validation Report**: 2.2.2. Model Performance — **Link Finding to Report** {{< fa chevron-right >}} **{{< fa plus >}} Create New Finding** {{< fa chevron-right >}} **Update Linked Findings**
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. In the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**.
+3. Locate the Data Preparation section and click on **2.2.2. Model Performance** to expand that section.
+4. Under the Model Performance Metrics section, locate Findings then click **Link Finding to Report**.
+5. Click **{{< fa plus >}} Create New Finding** to add a finding.
+6. Enter in the details for your finding and click **Save**.
+7. Select the finding you just added to link to your validation report.
+8. Click **Update Linked Findings** to insert your finding.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#run-diagnostic-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Run diagnostic tests**
-::: {.f6}
This time, use `list_tests()` to identify all the model diagnosis tests for classification:
1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"}
2. Run all the cells under the Running model evaluation tests section: **Run diagnostic tests**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_validation/3-developing_challenger_model.html#run-feature-importance-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Run feature importance tests**
-::: {.f6}
Use `list_tests()` again to identify all the feature importance tests for classification:
1. Continue with **3 — Developing a potential challenger model**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/3-developing_challenger_model.ipynb){target="_blank"}
2. Run all the cells under the Running model evaluation tests section: **Run feature importance tests**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -514,12 +499,10 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.html" background-interactive="yes" data-preload="yes"}
-:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay}
+:::: {.slideover--r .three-quarters}
**4 — Finalize testing and reporting**
-::: {.f6}
This is the final notebook in our introductory series, which will walk you through how to supplement ValidMind tests with your own custom tests and include them as additional evidence in your validation report, and wrap up your validation testing.
-:::
**Scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue.
@@ -552,7 +535,7 @@ This is the final notebook in our introductory series, which will walk you throu
::: {.f4}
As usual, let's connect back up to your model in the {{< var validmind.platform >}}:
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **Getting Started**.
3. Locate the code snippet and click **Copy snippet to clipboard**.
@@ -566,17 +549,14 @@ When you're done, click [{{< fa chevron-right >}}]() to continue.
## {background-iframe="/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.html#initialize-the-validmind-library" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Connect to your model**
-::: {.f6}
With your code snippet copied to your clipboard:
1. Open **4 — Finalize testing and validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"}
2. Run all the cells under the **Setting up** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -604,34 +584,28 @@ Let's implement a *custom test* that calculates a confusion matrix:
## {background-iframe="/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.html#implement-a-custom-inline-test" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Implement a custom inline test**
-::: {.f6}
-An *inline test* refers to a test written and executed within the same environment as the code being tested — <br>in the following example, right in our Jupyter Notebook — without requiring a separate test file or framework:
+An *inline test* refers to a test written and executed within the same environment as the code being tested — in the following example, right in our Jupyter Notebook — without requiring a separate test file or framework:
1. Continue with **4 — Finalize testing and validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"}
2. Run all the cells in the following sections under Implementing custom tests: **Implement a custom inline test**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.html#use-external-test-providers" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Use external test providers**
-::: {.f6}
-Sometimes you may want to reuse the same set of custom tests across multiple models and share them with others in your organization, <br>like the model development team would have done with you in this example workflow featured in this series of notebooks:
+Sometimes you may want to reuse the same set of custom tests across multiple models and share them with others in your organization, like the model development team would have done with you in this example workflow featured in this series of notebooks:
1. Continue with **4 — Finalize testing and validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"}
2. Run all the cells in the following sections under Implementing custom tests: **Use external test providers**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -640,17 +614,14 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_validation/4-finalize_validation_reporting.html#verify-test-runs" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Verify model development testing**
-::: {.f6}
Our final task is to verify that all the tests provided by the model development team were run and reported accurately:
1. Continue with **4 — Finalize testing and validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/4-finalize_validation_reporting.ipynb){target="_blank"}
2. Run all the cells under the **Verify test runs** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
diff --git a/site/training/validator-fundamentals/finalizing-validation-reports.qmd b/site/training/validator-fundamentals/finalizing-validation-reports.qmd
index 2d0afc88e4..88fa5fe511 100644
--- a/site/training/validator-fundamentals/finalizing-validation-reports.qmd
+++ b/site/training/validator-fundamentals/finalizing-validation-reports.qmd
@@ -16,6 +16,8 @@ format:
view-distance: 2
logo: /validmind.png
footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)"
+ revealjs-plugins:
+ - slideover
html:
# Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml
output-file: _finalizing-validation-reports.html
@@ -190,17 +192,16 @@ For example:
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Add & edit content blocks**
-
-::: {.f6}
-Within your **{{< fa shield >}} Validation Report**:
-
-1. Click on **1. Executive Summary** to add and edit a content block.
-2. Click on **2.2.1. Data Quality** to edit the description for the linked Class Imbalance Assessment test results.
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. On the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**.
+3. Click on **1. Executive Summary** to add and edit a content block.
+4. Click on **2.2.1. Data Quality** to edit the description for the linked Class Imbalance Assessment test results.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -254,14 +255,15 @@ In the same Class Imbalance Assessment sub-section, above the Risk Assessment No
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Add your compliance assessments**
-
-::: {.f6}
-Within your **{{< fa shield >}} Validation Report**, click on **2.2.1. Data Quality** to assess compliance for the Class Imbalance Assessment sub-section.
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. On the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**.
+3. Click on **2.2.1. Data Quality** to assess compliance for the Class Imbalance Assessment sub-section.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
:::
@@ -312,28 +314,30 @@ Along with adding findings directly via validation reports, you can also add fin
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Add a finding via overview**
-
-::: {.f6}
-Within the model's **{{< fa book-open >}} Documentation**, click **{{< fa plus >}} Add Finding** to add a finding from the overview.
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. On the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**.
+3. Click **{{< fa plus >}} Add Finding** to add a finding from the overview.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
:::
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Add a finding via section**
-
-::: {.f6}
-Within the model's **{{< fa book-open >}} Documentation**, click on any section heading to expand and add a finding to that section via the **{{< var vm.product >}} Insights™** panel.
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. On the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**.
+3. Click on any section heading to expand and add a finding to that section via the **{{< var vm.product >}} Insights™** panel.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
:::
@@ -377,15 +381,15 @@ Along with model-specific findings, you can also view and filter a list of findi
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Update a model finding**
+:::
-::: {.f6}
-1. From the **{{< fa cubes >}} Inventory**, select the name the model you registered to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. In the left sidebar that appears for your model, click **{{< fa triangle-exclamation >}} Model Findings**.
3. Select one of the findings you logged during this course, and make some changes to any of the fields.
4. When you are finished editing, set the finding **[status]{.smallcaps}** to `Closed`.
-:::
When you're done, click [{{< fa chevron-right >}}]() to continue.
@@ -393,12 +397,12 @@ When you're done, click [{{< fa chevron-right >}}]() to continue.
## {background-iframe="https://app.prod.validmind.ai/model-findings" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
-**View all {{< fa triangle-exclamation >}} Model Findings**
+:::: {.slideover--b .three-quarters .auto-collapse-10}
+::: {.tc}
+**View all {{< fa triangle-exclamation >}} Model Findings**
+:::
-::: {.f6}
Filter this list to include only findings you want to see, or toggle visibilty for column headers.
-:::
When you're done, click [{{< fa chevron-right >}}]() to continue.
@@ -448,15 +452,15 @@ To transition through the approval workflow, all required workflow steps must be
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Submit for approval**
+:::
-::: {.f6}
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. Locate the **[model status]{.smallcaps}** section.
3. Open up the status transition panel, enter your **[notes]{.smallcaps}** and any other additional inventory fields, then click **Submit**.
-:::
When you're done, click [{{< fa chevron-right >}}]() to continue.
@@ -499,14 +503,15 @@ Have a question about the model? Collaborate with your developer right in the mo
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Comment on a text block**
-
-::: {.f6}
-**In the content block you added earlier** within your **{{< fa shield >}} Validation Report**: Post a comment, reply to it, and then resolve the thread.
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. On the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**.
+3. **In the content block you added earlier**: Post a comment, reply to it, and then resolve the thread.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -540,15 +545,14 @@ Track changes & updates
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Review model activity**
+:::
-::: {.f6}
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model you registered for this course to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. In the left sidebar that appears for your model, click **{{< fa wifi >}} Model Activity**.
-3. Filter the following activity: **Comments** | **Status Updates** | **Model Updates** | **Test Results**
-
-:::
+3. Filter the following activity: <br>**Comments** | **Status Updates** | **Model Updates** | **Test Results**
When you're done, click [{{< fa chevron-right >}}]() to continue.
@@ -558,15 +562,15 @@ When you're done, click [{{< fa chevron-right >}}]() to continue.
## {background-iframe="https://app.prod.validmind.ai/analytics" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--l .three-quarters .auto-collapse-10}
**Welcome to {{< fa square-poll-vertical >}} Analytics**
-::: {.f6}
-Here, you can find executive summaries, track information on models, findings, and more. For example:
+Here, you can find executive summaries, track information on models, findings, and more.
+
+For example:
1. Click **Findings** to review reports on findings.
3. Click into any widget to review the findings reported by that widget.
-:::
When you're done, click [{{< fa chevron-right >}}]() to continue.
diff --git a/site/training/validator-fundamentals/running-data-quality-tests.qmd b/site/training/validator-fundamentals/running-data-quality-tests.qmd
index 9d66c3db25..3c95a1139f 100644
--- a/site/training/validator-fundamentals/running-data-quality-tests.qmd
+++ b/site/training/validator-fundamentals/running-data-quality-tests.qmd
@@ -16,6 +16,8 @@ format:
view-distance: 2
logo: /validmind.png
footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)"
+ revealjs-plugins:
+ - slideover
html:
# Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml
output-file: _running-data-quality-tests.html
@@ -138,50 +140,42 @@ Our series of four introductory notebooks for model validators include sample co
<br>
::: {.f4 .pl3 .pr3 .embed}
-In this second module, we'll run through **2 — Start the model validation process** together.
+In this second module, we'll run through **2 — Start the model validation process** together.
:::
:::
+
::::
Let's continue our journey with **2 — Start the model validation process** on the next page. {{< fa hand-point-right >}}
## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html" background-interactive="yes" data-preload="yes"}
-:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay}
+:::: {.slideover--r .three-quarters}
**2 — Start the model validation process**
-::: {.f6}
During this course, we'll run through these notebooks together, and at the end of your learning journey you'll have a fully supported sample validation report ready for review.
-:::
-
For now, **scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue.
::::
-
<!-- USING THE VARIABLE IN THE HEADING MESSES UP THE PAGE ANCHOR -->
# Explore ValidMind tests {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
## {background-iframe="/developer/model-testing/test-descriptions.html" background-interactive="true" data-preload="yes"}
-::: footer
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f4 .tc .pl4 .overlay}
+:::: {.slideover--l .three-quarters .auto-collapse-5}
**{{< var vm.product >}} test repository**
-::: {.f5}
{{< var vm.product >}} provides a wealth out-of-the-box of tests to help you ensure that your model is being built appropriately.
In this module, you'll become familiar with the individual tests available in {{< var vm.product >}}, as well as how to run them and change parameters as necessary.
-:::
-
For now, **scroll through these test descriptions** to explore. When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
-:::
## Get your code snippet
@@ -210,7 +204,7 @@ For now, **scroll through these test descriptions** to explore. When you're done
::: {.f5}
{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your validation environment:
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **Getting Started**.
3. Locate the code snippet and click **Copy snippet to clipboard**.
@@ -231,67 +225,55 @@ Make sure you're logged in and have refreshed the page in a Chromium-based web b
## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#initialize-the-validmind-library" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Connect to your model**
-::: {.f6}
With your code snippet copied to your clipboard:
1. Open **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
2. Run the cell under the **Setting up** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#load-the-sample-dataset" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Load the sample dataset**
-::: {.f6}
After you've successfully initialized the {{< var validmind.developer >}}, let's import the sample dataset that was used to develop the dummy champion model:
1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
2. Run the cell under the **Load the sample dataset** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#identify-qualitative-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Identify qualitative tests**
-::: {.f6}
Next, we'll use the [`list_tests()` function](/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.ipynb#explore-available-tests){target="_blank"} to pinpoint tests we want to run:
1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
2. Run all the cells under the Verifying data quality adjustments section: **Identify qualitative tests**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#initialize-the-validmind-datasets" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Initialize {{< var vm.product >}} datasets**
-::: {.f6}
Then, we'll use the [`init_dataset()` function](/validmind/validmind.qmd#init_dataset){target="_blank"} to connect the sample data with a {{< var vm.product >}} `Dataset` object in preparation for running tests:
1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
2. Run the following cell in the Verifying data quality adjustments section: **Initialize the {{< var vm.product >}} datasets**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -303,34 +285,28 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#run-data-quality-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Run data quality tests**
-::: {.f6}
You run individual tests by calling the [`run_test()` function](/validmind/validmind/tests.qmd#run_test){target="_blank"} provided by the `validmind.tests` module:
1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
2. Run all the cells under the Verifying data quality adjustments section: **Run data quality tests**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#remove-highly-correlated-features" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Remove highly correlated features**
-::: {.f6}
You can utilize the output from a ValidMind test for further use, for example, if you want to remove highly correlated features:
1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
2. Run all the cells under the Verifying data quality adjustments section: **Remove highly correlated features**
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -365,15 +341,13 @@ Every test result returned by the `run_test()` function has a `.log()` method th
## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#configure-and-run-comparison-tests" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Configure and run comparison tests**
-::: {.f6}
You can leverage the {{< var validmind.developer >}} to easily run comparison tests, between both datasets and models. Here, we compare the original raw dataset and the final preprocessed dataset, then log the results to the {{< var validmind.platform >}}:
1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
2. Run all the cells under the Documenting test results section: **Configure and run comparison tests**
-:::
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -381,17 +355,14 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#log-tests-with-unique-identifiers" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Log tests with unique identifiers**
-::: {.f6}
When running individual tests, you can use a custom `result_id` to tag the individual result with a unique identifier:
1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
2. Run the cell under the following Documenting test results section: **Log tests with unique identifiers**.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
@@ -446,39 +417,39 @@ While the example below focuses on a specific test result, you can follow the sa
:::
:::
-::::
+::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory/" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Link validator evidence**
-
-::: {.f6}
-Within your **{{< fa shield >}} Validation Report**: 2.2.1. Data Quality — **ValidMind Data Validation Class Imbalance** {{< fa chevron-right >}} **Link Evidence to Report**
-
:::
+1. Select the name of your model you registered for this course to open up the model details page.
+2. In the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**.
+3. Locate the Data Preparation section and click on **2.2.1. Data Quality** to expand that section.
+4. Under the Class Imbalance Assessment section, locate Validator Evidence then click **Link Evidence to Report**.
+5. Select the Class Imbalance test results we logged: **ValidMind Data Validation Class Imbalance**
+6. Click **Update Linked Evidence** to add the test results to the validation report.
+
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
-
# Prepare datasets for model evaluation {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
## {background-iframe="/notebooks/EXECUTED/model_validation/2-start_validation_process.html#split-the-preprocessed-dataset" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Split the preprocessed dataset**
-::: {.f6}
So far, we've rebalanced our raw dataset and used the results of {{< var vm.product >}} tests to additionally remove highly correlated features from our dataset. Next, let's now spilt our dataset into train and test in preparation for model evaluation testing:
1. Continue with **2 — Start the model validation process**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/2-start_validation_process.ipynb){target="_blank"}
2. Run all the cells under the **Split the preprocessed dataset** section.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
diff --git a/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd b/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd
index 95aaa27c63..729c64786a 100644
--- a/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd
+++ b/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd
@@ -16,6 +16,8 @@ format:
view-distance: 2
logo: /validmind.png
footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)"
+ revealjs-plugins:
+ - slideover
html:
# Change this to the file name prepended by a _ to get around the global HTML output settings required by _metadata.yml
output-file: _using-validmind-for-model-validation.html
@@ -141,14 +143,12 @@ In this first module, we'll run through **1 — Set up the {{< var validmind.dev
Let's start our journey with **1 — Set up the {{< var validmind.developer >}} for validation** on the next page. {{< fa hand-point-right >}}
-## {background-iframe="/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.html" background-interactive="yes" data-preload="yes"}
+## {background-iframe="/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.html" background-interactive="yes" data-preload="yes"}
-:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay}
+:::: {.slideover--r .three-quarters}
**1 — Set up the {{< var validmind.developer >}} for validation**
-::: {.f6}
During this course, we'll run through these notebooks together, and at the end of your learning journey you'll have a fully supported sample validation report ready for review.
-:::
For now, **scroll through this notebook** to explore. When you are done, click [{{< fa chevron-right >}}]() to continue.
@@ -160,99 +160,74 @@ For now, **scroll through this notebook** to explore. When you are done, click [
## {background-iframe="https://app.prod.validmind.ai" background-interactive="true" data-preload="yes"}
-::: {.fr .f5 .nr5 .pa5 .overlay}
+:::: {.slideover--r}
**Welcome to the {{< var validmind.platform >}}**
From here, you can:
-- Keep track of your models in the <br>customizable inventory ...
-- Review model documentation <br> submitted by model developers ...
-- Assess the compliance of models by <br>logging findings and test results as <br>evidence on validation reports ...
-- View analytics on your models and findings, <br>and create custom reports ...
+- Keep track of your models in the customizable inventory ...
+- Review model documentation submitted by model developers ...
+- Assess the compliance of models by logging findings and test results as evidence on validation reports ...
+- View analytics on your models and findings, and create custom reports ...
- ... and much more!
-::: {.f6 .pl3 .pr3 .embed}
+::: {.embed}
**Can't load the {{< var validmind.platform >}}?**
-Make sure you're logged in and have refreshed<br> the page in a Chromium-based browser.
+Make sure you're logged in and have refreshed the page in a Chromium-based browser.
:::
-When you're done navigating around,
-<br>click [{{< fa chevron-right >}}]() to continue.
+When you're done navigating around, click [{{< fa chevron-right >}}]() to continue.
-:::
+::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.fr .f5 .mv5 .nr5 .pa4 .overlay}
+:::: {.slideover--r}
**Welcome to the {{< fa cubes >}} Inventory**
-Use the model inventory to track <br>
-comprehensive details for all your <br>
-models throughout the model lifecycle.
-
-The model inventory is customizable <br>
-and extensible, with a layout that <br>
-can be configured to suit your needs.
-
-::: {.f6 .pl3 .pr3 .embed}
-In a usual model lifecycle, <br>
-a champion model will have been <br>
-independently registered in your <br>
-model inventory and submitted to <br>
-you for validation.
-
-For this training course, <br>
-we'll have you register a dummy model <br>
-via the **{{< fa plus >}} Register Model** modal <br>
-and assign yourself as the validator <br>
-to familiarize you with the <br>
-{{< var validmind.platform >}} interface.
+Use the model inventory to track comprehensive details for all your models throughout the model lifecycle. The model inventory is customizable and extensible, with a layout that can be configured to suit your needs.
+
+::: {.embed}
+In a usual model lifecycle, a champion model will have been independently registered in your model inventory and submitted to you for validation.
+
+For this training course, we'll have you register a dummy model via the **{{< fa plus >}} Register Model** modal and assign yourself as the validator to familiarize you with the {{< var validmind.platform >}} interface.
:::
-**Let's register a model together** for use <br>
-with validation on the next page. {{< fa hand-point-right >}}
+**Let's register a model together** for use with validation on the next page. {{< fa hand-point-right >}}
::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory/?register=open" background-interactive="true" data-preload="yes"}
-:::: {.fr .f5 .mv5 .nr5 .pa4 .overlay}
-**Register a binary <br>classification model**
-
-1. Select the option <br>
-for a new model:
+:::: {.slideover--r .three-quarters .auto-collapse-10}
+**Register a binary classification model**
-::: {.f6 .nt2 .pl2}
+1. Select the option for a new model:
- **Documentation template** — <br>`Binary classification`
-- **Use case** — <br>`Attrition/Churn Management`
-
- You can fill in other options<br> according to your preference.
-:::
-
-2. Click **{{< fa plus >}} Register Model** to <br>
-add the model to your <br>
-inventory.
+- **Use case** — <br>`Attrition/Churn Management`<br><br>
+ You can fill in other options according to your preference.
+2. Click **{{< fa plus >}} Register Model** to add the model to your inventory.
-When you're done, <br>click [{{< fa chevron-right >}}]() to continue.
+When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=15 left=0 right=50 .w-100 .f5 .tc .pl4 .pr4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Assign validator credentials**
-::: {.f6}
+:::
+
In order to log tests as a validator instead of as a developer, we'll need to adjust your **model stakeholder** permissions:
-1. From the **{{< fa cubes >}} Inventory**, select the name the model you registered to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. Remove yourself as a model owner by clicking on the **[owners]{.smallcaps}** tile, then clicking on the **x** next to your name. Click **Save** to apply your changes to that role.
3. Remove yourself as a developer by clicking on the **[developers]{.smallcaps}** tile, then clicking on the **x** next to your name. Click **Save** to apply your changes to that role.
4. Add yourself as a validator by clicking on the **[validators]{.smallcaps}** tile, and selecting your name from the drop-down menu. Click **Save** to apply your changes to that role.
-:::
-
When you're done, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -286,7 +261,7 @@ When you're done, click [{{< fa chevron-right >}}]() to continue.
::: {.f5}
{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your validation environment:
-1. From the **{{< fa cubes >}} Inventory**, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **Getting Started**.
3. Locate the code snippet and click **Copy snippet to clipboard**.
@@ -307,16 +282,15 @@ Make sure you're logged in and have refreshed the page in a Chromium-based web b
## {background-iframe="/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.html#install-the-validmind-library" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Install & initialize the {{< var validmind.developer >}}**
-::: {.f6}
With your code snippet copied to your clipboard:
1. Open **1 — Set up the {{< var validmind.developer >}} for validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb){target="_blank"}
-2. Run all the cells under the Setting up sections: **Install the {{< var validmind.developer >}}** / **Initialize the {{< var validmind.developer >}}**
-
-:::
+2. Run all the cells under the Setting up sections:
+ - **Install the {{< var validmind.developer >}}**
+ - **Initialize the {{< var validmind.developer >}}**
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
@@ -326,35 +300,31 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to
## {background-iframe="/notebooks/EXECUTED/model_validation/1-set_up_validmind_for_validation.html#preview-the-validation-report-template" data-preload="yes"}
-:::: {.absolute bottom=15 .w-100 .f5 .tc .pl4 .overlay}
+:::: {.slideover--r .three-quarters}
**Preview the validation report template**
-::: {.f6}
You can preview your model's validation report template right from the {{< var validmind.developer >}}:
1. Continue with **1 — Set up the {{< var validmind.developer >}} for validation**: [{{< fa square-arrow-up-right >}} JupyterHub]({{< var url.jupyterhub >}}/hub/user-redirect/lab/tree/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb){target="_blank"}
2. Run all the cells in the sections under **Getting to know ValidMind**.
-:::
-
When you're done, return to this page and click [{{< fa chevron-right >}}]() to continue.
::::
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.absolute bottom=0 left=50 right=50 .w-95 .f5 .tc .pl4 .overlay}
+:::: {.slideover--b .auto-collapse-10}
+::: {.tc}
**Verify the report template**
+:::
-::: {.f6}
Once you've called `preview_template()`:
-1. From the model inventory, select the name of your model to open up the model details page.
+1. Select the name of your model you registered for this course to open up the model details page.
2. On the left sidebar that appears for your model, click **{{< fa shield >}} Validation Report**.
3. Note how the structure of the validation report reflects the previewed template.
-:::
-
When you're done taking a look around, click [{{< fa chevron-right >}}]() to continue.
::::
@@ -382,41 +352,30 @@ Try it **live** on the next page. {{< fa hand-point-right >}}
{{< include /guide/model-documentation/_review-model-documentation.qmd >}}
-
:::
::::
-
-
## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-:::: {.fr .f5 .mv6 .nr5 .pa4 .overlay}
+:::: {.slideover--r .auto-collapse-10}
**Explore sample model documentation**
-In a usual model lifecycle, a champion model <br>
-will have been submitted to you for validation <br>
-with completed model documentation.
+In a usual model lifecycle, a champion model will have been submitted to you for validation with completed model documentation.
+
+Here, review the empty sample model documentation for the model you registered earlier to familiarize you with what the structure of documentation could look like when presented to you for review:
-Here, review the empty sample model <br>
-documentation for the model you registered <br>
-earlier to familiarize you with what the structure <br>
-of documentation could look like when <br>
-presented to you for review.
+1. Select the name of your model you registered for this course to open up the model details page.
+2. In the left sidebar that appears for your model, click **{{< fa book-open >}} Documentation**.
-::: {.f6 .pl3 .pr3 .embed}
-The structure of the model documentation reflects <br>
-the documentation template applied to the model, <br>
-just like the validation report template.
+::: {.embed}
+The structure of the model documentation reflects the documentation template applied to the model, just like the validation report template.
:::
-When you're done taking a look <br>
-around the model documentation, <br>
-click [{{< fa chevron-right >}}]() to continue.
+When you're done taking a look around the model documentation, click [{{< fa chevron-right >}}]() to continue.
::::
-
# In summary {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
## {.scrollable .center}
diff --git a/site/training/validator-fundamentals/validator-fundamentals.qmd b/site/training/validator-fundamentals/validator-fundamentals.qmd
deleted file mode 100644
index 75feca543c..0000000000
--- a/site/training/validator-fundamentals/validator-fundamentals.qmd
+++ /dev/null
@@ -1,478 +0,0 @@
----
-title: "Validator <br>Fundamentals"
-subtitle: "docs.validmind.ai/training<br><br>_Click [{{< fa chevron-right >}}](#learning-objectives) to start_"
-lightbox: true
-search: false
-format:
- revealjs:
- controls: true
- controls-tutorial: true
- controls-back-arrows: visible
- transition: slide
- theme: [default, ../assets/slides.scss]
- slide-number: true
- chalkboard: false
- preview-links: auto
- view-distance: 2
- logo: /validmind.png
- footer: "{{< var validmind.training >}} | [Home {{< fa person-walking-dashed-line-arrow-right >}}](/training/training.qmd)"
- html:
- output-file: _validator-fundamentals.html
- search: false
-title-slide-attributes:
- data-background-color: "#083E44"
- data-background-image: "../assets/home-hero.svg"
-skip_preview: true
----
-
-# Learning objectives
-
-_"As a validator who is new to {{< var vm.product >}}, I want to learn how to review model documentation, prepare my validation report, track issues, and submit my report for approval."_
-
-
-
-## In this course
-
-:::: {.columns .f2}
-::: {.column width="55%"}
-### PART 1
-
-- [Review documentation](#review-documentation)
-- [Analyze test results](#analyze-test-results)
-- [Prepare validation reports](#prepare-validation-reports)
-:::
-
-::: {.column width="45%"}
-### PART 2
-
-- [Track issue resolution](#track-issue-resolution)
-- [Submit for review and approval](#submit-for-review-and-approval)
-:::
-::::
-
-First, let's make sure you can log in to {{< var vm.product >}}.
-
-{{< include /training/assets/_revealjs-navigation.qmd >}}
-
-## Can you log in?
-
-To try out this course, you need to have been [onboarded](validator-fundamentals-register.qmd#register) onto {{< var validmind.training >}} with the [**{{< fa circle-check >}} Validator**]{.bubble} role.
-
-<br>Log in to check your access:
-
-::: {.tc}
-[Log in to {{< var vm.product >}}](https://app.prod.validmind.ai/){.button target="_blank"}
-
-<br>
-Be sure to return to this page afterwards.
-
-:::
-
-<!---[Log in](https://app.prod.validmind.ai/){.f2 .f4 .fw6 .db .black .link .dim .ph3 .pv2 .br3 .mb2 .dib .dark-blue .bg-near-white}--->
-
-# You're in — let's show you around {{< var vm.product >}}.
-
-## {background-iframe="https://app.prod.validmind.ai" data-preload="yes"}
-
-:::: {.fr .f3 .mv5 .pa5 .overlay}
-**This is the {{< var validmind.platform >}}.**
-
-From here, you have access to:
-
-- Model inventory
-- Documentation & test results
-- Validation reports
-- Model findings
-- Analytics
-
-Try it **live** on the next page. {{< fa hand-point-right >}}
-::::
-
-## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-
-:::: {.fr .f4 .mv5 .pa5 .overlay}
-From the **{{< fa cubes >}} Inventory**, <br>you access:
-
-- Model documentation
-- Validation reports
-- Getting started for <br>developers
-
-Explore some models, then <br>click [{{< fa chevron-right >}}]() to continue.
-:::
-
-## {background-iframe="https://app.prod.validmind.ai/model-findings" background-interactive="true" data-preload="yes"}
-
-:::: {.fr .f4 .mv6 .nr4 .pa4 .overlay}
-From **{{< fa triangle-exclamation >}} Model Findings**, <br>you track:
-
-- Major or minor issues
-- Deficiencies
-- Model limitations
-- Robustness concerns
-- And more!
-
-Explore existing findings, <br>then click [{{< fa chevron-right >}}]() to continue.
-::::
-
-## {background-iframe="https://app.prod.validmind.ai/reports" background-interactive="true" data-preload="yes"}
-
-:::: {.fr .f4 .mv6 .nr4 .pa4 .overlay}
-Under **{{< fa square-poll-vertical >}} Analytics**, you can <br>find:
-
-- Executive summaries
-- Model tracking
-- Risk assessments
-- Compliance review
-- Remediation tracking
-
-Explore existing reports, <br>then click [{{< fa chevron-right >}}]() to continue.
-::::
-
-# PART 1 {background-color="#083E44" background-image="/assets/img/solutions-hero.png"}
-
-# Review documentation {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
-
-## {.scrollable}
-
-:::: {.columns}
-::: {.column width="30%" .pr4 .f2}
-<!-- Verify model documentation is complete and complies with regulatory standards by cross-checking against your guidelines. -->
-
-Evaluate the conceptual soundness, data preparation, model development, and ongoing monitoring and governance plans for the model.
-
-::: {.tc}
-[Learn more...](/guide/model-validation/review-model-documentation.qmd){.button target="_blank"}
-:::
-
-:::
-
-::: {.column width="70%" .bl .pl4 .f3}
-### Review model documentation
-
-{{< include /guide/model-documentation/_model-documentation-overview.qmd >}}
-
-{{< include /guide/model-documentation/_review-model-documentation.qmd >}}
-
-
-:::
-::::
-
-Try it **live** on the next page. {{< fa hand-point-right >}}
-
-## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-
-:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .pr4 .overlay}
-Review the documentation for the **[Quickstart] Customer Churn Model**.
-
-::: {.f5 .nt2}
-When you are done, click [{{< fa chevron-right >}}]() to continue.
-:::
-::::
-
-## {.scrollable}
-
-:::: {.columns}
-::: {.column width="30%" .pr4 .f2}
-Have a question about the model? Collaborate with your developer right in the model documentation.
-
-::: {.tc}
-[Learn more ...](/guide/model-documentation/collaborate-with-others.qmd){.button target="_blank"}
-:::
-
-<br>Try it **live** on the next page. {{< fa hand-point-right >}}
-
-:::
-
-::: {.column width="70%" .bl .pl4 .f3}
-::: {.f5 .nt2}
-:::
-
-{{< include /guide/model-documentation/_collaborate-with-others-activity.qmd >}}
-
-::: {.panel-tabset}
-
-{{< include /guide/model-documentation/_collaborate-with-others-comments.qmd >}}
-
-:::
-
-:::
-
-::::
-
-## {background-iframe="https://app.prod.validmind.ai//model-inventory" background-interactive="true" data-preload="yes"}
-
-:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .pr4 .overlay}
-In the documentation, post a comment, reply to it, and then resolve the comment thread. Review your comments in the **Recent Activity** feed on the front page.
-
-::: {.f5 .nt2}
-When you are done, click [{{< fa chevron-right >}}]() to continue.
-:::
-::::
-
-# Analyze test results {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
-
-## {.scrollable}
-
-:::: {.columns}
-::: {.column width="30%" .pr4 .f2}
-Locate the test results in the documentation, review the data, and identify issues with the model.
-
-::: {.tc}
-[Learn more ...](/guide/model-validation/review-model-documentation.qmd){.button target="_blank"}
-:::
-
-:::
-
-::: {.column width="70%" .bl .pl4 .f3}
-
-{{< include /guide/model-documentation/_review-model-documentation.qmd >}}
-
-1. Review the sections:
-
- - 2. Data Preparation
- - 3. Model Development
-
-:::
-
-::::
-
-<br>Try it **live** on the next page. {{< fa hand-point-right >}}
-
-## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-
-:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .pr4 .overlay}
-In the documentation, review **2. Data Preparation** and **3. Model Development**.
-
-::: {.f5 .nt2}
-When you are done, click [{{< fa chevron-right >}}]() to continue.
-:::
-::::
-
-# Prepare validation reports {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
-
-## {.scrollable}
-
-:::: {.columns}
-::: {.column width="30%" .pr4 .f2}
-Based on your review of the documentation, add some findings for your validation report.
-
-::: {.tc}
-[Learn more ...](/guide/model-validation/add-manage-model-findings.qmd){.button target="_blank"}
-:::
-
-<br>Try it **live** on the next page. {{< fa hand-point-right >}}
-:::
-
-::: {.column width="70%" .bl .pl4 .f3}
-### Add findings
-
-As part of the validation process, you may find issues with the model documentation that must be resolved. To indicate that there is an issue and to track the resolution later on, you add a new finding.
-
-- Findings are logged with your model documentation.
-- You can add findings both on the main documentation overview page and in each documentation section.
-
-{{< include /guide/model-validation/_add-model-findings.qmd >}}
-
-:::
-::::
-
-## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-
-:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .pr4 .overlay}
-Add some findings for your validation report.
-
-::: {.f5 .nt2}
-When you are done, click [{{< fa chevron-right >}}]() to continue.
-:::
-::::
-
-## {.scrollable}
-
-:::: {.columns}
-::: {.column width="30%" .pr4 .f2}
-Link your findings and the evidence from the test results you analyzed to the validation report.
-
-::: {.f5 .nt2 .pl2 .mb4}
-(Scroll down for the full instructions.)
-:::
-
-::: {.tc}
-[Learn more ...](/guide/model-validation/assess-compliance.qmd){.button target="_blank"}
-:::
-
-<br>Try it **live** on the next page. {{< fa hand-point-right >}}
-:::
-
-::: {.column width="70%" .bl .pl4 .f3}
-::: {.panel-tabset}
-### Link finding
-
-{{< include /guide/model-validation/_assess-compliance-link-finding.qmd >}}
-
-### Link evidence
-
-{{< include /guide/model-validation/_assess-compliance-developer-evidence.qmd >}}
-
-:::
-
-:::
-::::
-
-## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-
-:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .pr4 .overlay}
-Link findings and link evidence to your validation report.
-
-::: {.f5 .nt2}
-When you are done, click [{{< fa chevron-right >}}]() to continue.
-:::
-::::
-
-## {.scrollable}
-
-:::: {.columns}
-::: {.column width="30%" .pr4 .f2}
-Based on the evidence you analyzed and your model, with your guidelines.
-
-::: {.f5 .nt2 .pl2 .mb4}
-(Scroll down for the full instructions.)
-:::
-
-::: {.tc}
-[Learn more ...](/guide/model-validation/assess-compliance.qmd#provide-compliance-assessments){.button target="_blank"}
-:::
-
-<br>Try it **live** on the next page. {{< fa hand-point-right >}}
-:::
-
-::: {.column width="70%" .bl .pl4 .f4}
-
-### Assess compliance
-
-{{< include /guide/model-validation/_assess-compliance-assess.qmd >}}
-
-:::
-::::
-
-## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-
-:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .pr4 .overlay}
-Assess compliance for your validation report.
-
-::: {.f5 .nt2}
-When you are done, click [{{< fa chevron-right >}}]() to continue.
-:::
-::::
-
-# PART 2 {background-color="#083E44" background-image="/assets/img/solutions-hero.png"}
-
-# Track issue resolution {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
-
-## {.scrollable}
-
-:::: {.columns}
-::: {.column width="30%" .pr4 .f2}
-As you prepare your report, review open or past due findings, close resolved ones, or add a mitigation plan.
-
-::: {.f5 .nt2 .pl2 .mb4}
-(Scroll down for the full instructions.)
-:::
-
-::: {.tc}
-[Learn more ...](/guide/model-validation/add-manage-model-findings.qmd#track-issue-resolution){.button target="_blank"}
-:::
-
-<br>Try it **live** on the next page. {{< fa hand-point-right >}}
-:::
-
-::: {.column width="70%" .bl .pl4 .f3}
-### Track issue resolution
-
-{{< include /guide/model-validation/_track_issue_resolution.qmd >}}
-
-:::
-::::
-
-## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-
-:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .pr4 .overlay}
-To track issue resolution, review open or past due findings, close resolved ones, or add a mitigation plan.
-
-::: {.f5 .nt2}
-When you are done, click [{{< fa chevron-right >}}]() to continue.
-:::
-::::
-
-# Submit for review and approval {background-color="#083E44" background-image="/training/assets/home-hero.svg"}
-
-## {.scrollable}
-
-:::: {.columns}
-::: {.column width="30%" .pr4 .f2}
-When you’re ready, verify the approval workflow, and then submit your validation report for approval.
-
-::: {.f5 .nt2}
-(Scroll down for the full instructions.)
-:::
-
-::: {.tc}
-[Learn more ...](/guide/model-documentation/submit-for-approval.qmd){.button target="_blank"}
-:::
-
-<br>Try it **live** on the next page. {{< fa hand-point-right >}}
-:::
-
-::: {.column width="70%" .bl .pl4 .f4}
-
-::: {.panel-tabset}
-### Verify workflow
-
-Workflow states and transitions are configured by an administrator in advance, but you should verify that the expected people are included in the approval process.
-
-{{< include /guide/model-workflows/_model-workflows-see.qmd >}}
-
-### Submit for approval
-
-To transition through the approval workflow, all required workflow steps must be completed. For example, you cannot submit a validation report for review until the model documentation itself has been submitted.
-
-{{< include /guide/model-workflows/_model-workflows-transition.qmd >}}
-:::
-
-:::
-::::
-
-## {background-iframe="https://app.prod.validmind.ai/model-inventory" background-interactive="true" data-preload="yes"}
-
-:::: {.absolute bottom=15 .w-100 .f4 .tc .pl4 .pr4 .overlay}
-Verify the workflow, then submit your validation report for review & approval.
-
-::: {.f5 .nt2}
-When you are done, click [{{< fa chevron-right >}}]() to continue.
-:::
-::::
-
-# About validation reports
-
-There is more that {{< var vm.product >}} can do to help you prepare validation reports, from using your own template to configuring the full approval workflow.
-
-::: {.tc}
-[All model validation guides](/guide/guides.qmd#model-validation){.button target="_blank"}
-:::
-<br>
-Or, find your next learning resource on [{{< var validmind.training >}}](/training/training.qmd).
-
-<!-- ## PREVIEW: AI documentation assistant
-
-:::: {.columns}
-::: {.column width="50%"}
-Use the {{< var vm.product >}} AI assistant to auto-generate initial report content based on model metadata and to gain insight into test results:
-
-{fig-alt="A screenshot showing a text editor with a toolbar and the Generate with AI button on the right"}
-:::
-
-::: {.column width="50%"}
-Accept the text or try again:
-
-{fig-alt="A screenshot showing a modal with AI-generated text"}
-:::
-:::: -->