diff --git a/.github/actions/setup-r/action.yml b/.github/actions/setup-r/action.yml new file mode 100644 index 0000000000..5b2f53deac --- /dev/null +++ b/.github/actions/setup-r/action.yml @@ -0,0 +1,33 @@ +name: "Setup R and install packages" +description: "Sets up R and installs required packages for displaying interactive tables from .CSV" +runs: + using: "composite" + steps: + - name: Set up R + uses: r-lib/actions/setup-r@v2 + with: + r-version: '4.3' + + - name: Cache R packages + uses: actions/cache@v3 + with: + path: ${{ runner.os == 'Linux' && '/home/runner/work/_temp/Library' || runner.os == 'macOS' && '~/Library/R/4.3/library' }} + key: r-packages-${{ runner.os }}-${{ hashFiles('**/DESCRIPTION', '**/renv.lock') }} + restore-keys: | + r-packages-${{ runner.os }}- + + - name: Install R packages + shell: bash + run: | + Rscript -e 'install.packages(c("DT", "readr", "stringr", "lubridate"), repos="https://cran.rstudio.com/")' + + - name: Verify R installation + shell: bash + run: | + Rscript -e 'sessionInfo()' + which R + + - name: Verify Quarto detects R + shell: bash + run: | + quarto check diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6b84df5b98..0947af5637 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,8 +8,9 @@ PR instructions for release notes: - `internal` (skip Step 2, no release notes required) - `highlight` - `enhancement` -- `bug` +- `breaking-change` - `deprecation` +- `bug` - `documentation` 2. In the next section, describe the changes so that an external user can understand them. Keep it simple and link to the docs with [Learn more ...](), if available. diff --git a/.github/workflows/deploy-docs-prod.yaml b/.github/workflows/deploy-docs-prod.yaml index 8948ec862a..8edb0c8710 100644 --- a/.github/workflows/deploy-docs-prod.yaml +++ b/.github/workflows/deploy-docs-prod.yaml @@ -25,6 +25,9 @@ jobs: - name: Fetch Quarto uses: ./.github/actions/fetch-quarto + + - name: Setup R environment + uses: ./.github/actions/setup-r - name: Render prod docs site run: | diff --git a/.github/workflows/deploy-docs-staging.yaml b/.github/workflows/deploy-docs-staging.yaml index b08982f28c..c5ddf48fd3 100644 --- a/.github/workflows/deploy-docs-staging.yaml +++ b/.github/workflows/deploy-docs-staging.yaml @@ -25,6 +25,9 @@ jobs: - name: Fetch Quarto uses: ./.github/actions/fetch-quarto + + - name: Setup R environment + uses: ./.github/actions/setup-r - name: Render staging docs site run: | diff --git a/.github/workflows/validate-docs-site.yaml b/.github/workflows/validate-docs-site.yaml index bba96bf603..890ba6eab5 100644 --- a/.github/workflows/validate-docs-site.yaml +++ b/.github/workflows/validate-docs-site.yaml @@ -25,6 +25,9 @@ jobs: - name: Fetch Quarto uses: ./.github/actions/fetch-quarto + - name: Setup R environment + uses: ./.github/actions/setup-r + - name: Render demo docs site run: | cd site diff --git a/README.md b/README.md index c377e6bb91..54c4275e89 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,12 @@ You need: - The Quarto extension for your IDE, such as [VS Code](https://marketplace.visualstudio.com/items?itemName=quarto.quarto) - For Windows operating systems, install the `make` command via [Cygwin](https://cygwin.com/install.html) +### Additional dependencies + +Some interactive tables, such as our breaking changes and dependency history rely you have R and some R packages installed in order for you to be able to preview or render certain pages of the docs site locally. + +**Refer to the [Breaking changes and deprecations](site/releases/breaking-changes/README.md) guide** for more information on how to install R and set up these tables. + ## How to contribute > [!IMPORTANT] diff --git a/release-scripts/generate-release-notes.ipynb b/release-scripts/generate-release-notes.ipynb index 8c8f8e7496..1158fb9951 100644 --- a/release-scripts/generate-release-notes.ipynb +++ b/release-scripts/generate-release-notes.ipynb @@ -155,7 +155,7 @@ "metadata": {}, "outputs": [], "source": [ - "label_hierarchy = [\"highlight\", \"enhancement\", \"deprecation\", \"bug\", \"documentation\"]\n", + "label_hierarchy = [\"highlight\", \"enhancement\", \"breaking-change\", \"deprecation\", \"bug\", \"documentation\"]\n", "gro.display_list(label_hierarchy)" ] }, diff --git a/release-scripts/generate_release_objects.py b/release-scripts/generate_release_objects.py index 53fa50e48e..b67ade140a 100644 --- a/release-scripts/generate_release_objects.py +++ b/release-scripts/generate_release_objects.py @@ -338,6 +338,7 @@ def setup_openai_api(env_location): label_to_category = { "highlight": "## Release highlights", "enhancement": "## Enhancements", + "breaking-change": "## Breaking changes", "deprecation": "## Deprecations", "bug": "## Bug fixes", "documentation": "## Documentation" @@ -346,6 +347,7 @@ def setup_openai_api(env_location): categories = { "highlight": [], "enhancement": [], + "breaking-change": [], "deprecation": [], "bug": [], "documentation": [] @@ -899,7 +901,7 @@ def main(): setup_openai_api(env_location) print() - label_hierarchy = ["highlight", "enhancement", "deprecation", "bug", "documentation"] + label_hierarchy = ["highlight", "enhancement", "breaking-change", "deprecation", "bug", "documentation"] display_list(label_hierarchy) print() diff --git a/site/Makefile b/site/Makefile index 7ac934a36e..11ed08e6b0 100644 --- a/site/Makefile +++ b/site/Makefile @@ -90,6 +90,11 @@ docs-site: get-source quarto render --profile production @$(MAKE) execute PROFILE=exe-prod +# Get all source files +docs-site-lite: get-source + @echo "\nRendering the static HTML site ..." + quarto render --profile production + # Deployment to https://docs-demo.vm.validmind.ai/ deploy-demo: @if [ "`git rev-parse --abbrev-ref HEAD`" != "docs-demo" ]; then \ @@ -152,7 +157,7 @@ yearly-releases: execute: quarto render --profile $(PROFILE) $(FILE_PATH) -docker-build: docs-site +docker-build: docs-site-lite @echo "\nBuilding the Docker image ..." @docker build -f ../Dockerfile -t validmind-docs .. diff --git a/site/_quarto.yml b/site/_quarto.yml index 84c238b7d7..ab80ddfb9a 100644 --- a/site/_quarto.yml +++ b/site/_quarto.yml @@ -2,6 +2,9 @@ project: type: website post-render: make python-docs +metadata-files: + - training/_sidebar.yaml + website: announcement: content: "[**{{< fa book-open-reader >}} EU AI Act Compliance**](https://validmind.com/download-whitepaper-the-eu-ai-act/) — Read our original regulation brief on how the EU AI Act aims to balance innovation with safety and accountability, setting standards for responsible AI use" @@ -170,6 +173,7 @@ website: - text: "---" - text: "Releases" # MAKE-RELEASE-NOTES-EMBED-MARKER + - releases/2025/2025-mar-07/release-notes.qmd - releases/2025/2025-jan-31/release-notes.qmd # CURRENT-YEAR-END-MARKER - file: releases/2024/2024-releases.qmd @@ -197,6 +201,8 @@ website: - releases/2023/release-notes-2023-jul-24.qmd - releases/2023/release-notes-2023-jun-22.qmd - releases/2023/release-notes-2023-may-30.qmd + - text: "Breaking changes & deprecations" + file: releases/breaking-changes/breaking-changes.qmd - text: "---" - text: "Fine Print" - about/fine-print/data-privacy-policy.qmd diff --git a/site/about/fine-print/data-privacy-policy.qmd b/site/about/fine-print/data-privacy-policy.qmd index 2442deb451..3de8cd4e5a 100644 --- a/site/about/fine-print/data-privacy-policy.qmd +++ b/site/about/fine-print/data-privacy-policy.qmd @@ -37,9 +37,9 @@ Service Organization Control 2 (SOC 2) is a type of audit report that evaluates The {{< var validmind.platform >}} is a secure, multi-tenant solution that can be hosted on Amazon Web Services (AWS), Microsoft Azure Cloud (Azure), or Google Cloud Platform (GCP). For organizations that require a stricter trust model and the highest level of security, such as financial services organizations handling highly sensitive data, {{< var vm.product >}} also offers a _{{< var validmind.vpv >}} (VPV)_ option to host our solution in a dedicated single-tenant cloud instance. -The {{< var validmind.vpv >}} option provides all the features and services of other editions of our products, but hosted within a separate environment that is isolated from other {{< var vm.product >}} accounts. VPV accounts do not share resources with non-VPV accounts. +The {{< var validmind.vpv >}} option provides all our features and services but hosted within a separate environment that is isolated from other {{< var vm.product >}} accounts. VPV accounts do not share resources with non-VPV accounts. -Access to any edition is available through AWS PrivateLink, Azure Private Link, or GCP Private Service Connect, all of which provide private connectivity between {{< var vm.product >}} and your on-premises network without exposing your traffic to the public internet. +Access is available through AWS PrivateLink, Azure Private Link, or GCP Private Service Connect, all of which provide private connectivity between {{< var vm.product >}} and your on-premises network without exposing your traffic to the public internet. ## What model artifacts are imported into documentation? @@ -56,5 +56,4 @@ The {{< var validmind.developer >}} does not send any personally identifiable in - \ No newline at end of file +- [Configure AWS PrivateLink](/guide/configuration/configure-aws-privatelink.qmd) --> \ No newline at end of file diff --git a/site/developer/model-testing/testing-overview.qmd b/site/developer/model-testing/testing-overview.qmd index 99666a993f..7ea340428e 100644 --- a/site/developer/model-testing/testing-overview.qmd +++ b/site/developer/model-testing/testing-overview.qmd @@ -6,6 +6,7 @@ aliases: listing: - id: tests-beginner type: grid + grid-columns: 2 max-description-length: 250 sort: false fields: [title, description] @@ -28,7 +29,7 @@ listing: max-description-length: 250 sort: false fields: [title, description] - contents: + contents: - ../../notebooks/code_samples/custom_tests/integrate_external_test_providers.ipynb - ../../notebooks/how_to/configure_dataset_features.ipynb - ../../notebooks/how_to/run_documentation_sections.ipynb @@ -43,6 +44,7 @@ listing: contents: - ../../notebooks/how_to/document_multiple_results_for_the_same_test.ipynb - ../../notebooks/how_to/load_datasets_predictions.ipynb + - ../../notebooks/how_to/understand_utilize_rawdata.ipynb - id: tests-custom type: grid max-description-length: 250 diff --git a/site/faq/_faq-editions.qmd b/site/faq/_faq-editions.qmd deleted file mode 100644 index 38bb3d243c..0000000000 --- a/site/faq/_faq-editions.qmd +++ /dev/null @@ -1,14 +0,0 @@ -## What solutions does {{< var vm.product >}} offer? - -{{< var vm.product >}} is a {{< var vm.developer >}} and cloud {{< var vm.platform >}} available in multiple editions catering to different organizational needs: - -- **Standard Edition**: Our introductory offering, providing essential features and services. -- **Enterprise Edition**: Builds upon the Standard Edition by adding features tailored for large-scale organizations. -- **{{< var validmind.vpv >}} (VPV)**:^[[{{< var validmind.vpv >}} (VPV)](/about/deployment/virtual-private-validmind.qmd)] Our most secure offering for organizations requiring a higher level of privacy, such as financial services handling sensitive data. Includes all Enterprise Edition features but in a separate, isolated {{< var vm.product >}} environment. VPV accounts do not share resources with accounts outside the VPV. - -::: {.callout} -## {{< var vm.product >}} does not send any personally identifiable information (PII) through our {{< var vm.api >}}. - -Access to any edition is facilitated through AWS PrivateLink, which provides private connectivity between {{< var vm.product >}} and your on-premises networks without exposing your traffic to the public internet.^[[Configure AWS PrivateLink](/guide/configuration/configure-aws-privatelink.qmd)] - -::: \ No newline at end of file diff --git a/site/faq/faq-organizations.qmd b/site/faq/faq-organizations.qmd index e6f5fef48a..3737e4db7e 100644 --- a/site/faq/faq-organizations.qmd +++ b/site/faq/faq-organizations.qmd @@ -13,8 +13,6 @@ listing: - ../guide/configuration/managing-users.qmd --- -{{< include _faq-editions.qmd >}} - ## How do I get access to {{< var vm.product >}}? #### 1. Register with ValidMind @@ -40,7 +38,7 @@ By default, the [{{< fa hand >}} Customer Admin]{.bubble} role[^5] has sufficien - {{< var vm.product >}} provides a built-in user management interface that allows new users to be registered on the {{< var validmind.platform >}} and assigned user roles[^6] and user groups.[^7] - User roles, user groups, and access permissions are configured during initial onboarding and can be updated at any time. -- In addition, {{< var vm.product >}} also provides support for Single Sign-On (SSO) integration as part of our Enterprise and our {{< var validmind.vpv >}} (VPV) edition. +- In addition, {{< var vm.product >}} also offers support for Single Sign-On (SSO) integration. ## How do user roles, user groups, and access permissions work? diff --git a/site/faq/faq-privacy.qmd b/site/faq/faq-privacy.qmd index 65a9f570c6..d2c0662a0b 100644 --- a/site/faq/faq-privacy.qmd +++ b/site/faq/faq-privacy.qmd @@ -18,11 +18,15 @@ listing: - ../guide/model-inventory/view-model-activity.qmd --- -{{< include _faq-editions.qmd >}} +## How does {{< var vm.product >}} handle the processing of PII? + +{{< var vm.product >}} does not send any personally identifiable information (PII) through our {{< var validmind.api >}}.[^1] + +Access to the {{< var validmind.platform >}} is facilitated through AWS PrivateLink, which provides private connectivity between {{< var vm.product >}} and your on-premises networks without exposing your traffic to the public internet.[^2] ## What model artifacts are automatically imported into {{< var vm.product >}}? -{{< var vm.product >}} stores the following artifacts in the documentation via our {{< var validmind.api >}}:[^1] +{{< var vm.product >}} stores the following artifacts in the documentation via our {{< var validmind.api >}}: - Dataset and model metadata which allow generating documentation snippets programmatically (example: stored definition for "common logistic regression limitations" when a logistic regression model has been passed to the {{< var vm.product >}} test suite execution) - Quality and performance metrics collected from the dataset and model @@ -35,7 +39,7 @@ listing: - With multi-tenant deployments, infrastructure is shared but with strict data isolation protocols that ensure that no tenant can access another's sensitive information. ::: {.callout} -## For organizations requiring the highest degree of data security, {{< var vm.product >}} offers a {{< var validmind.vpv >}} (VPV) option to host the solution in a dedicated single-tenant cloud instance on the {{< var vm.product >}} cloud account. +## For organizations requiring the highest degree of data security, {{< var vm.product >}} offers a {{< var validmind.vpv >}} (VPV)[^3] option to host the solution in a dedicated single-tenant cloud instance on the {{< var vm.product >}} cloud account. Furthermore, {{< var vm.product >}}'s data retention policy complies with the SOC 2 security standard. ::: @@ -82,4 +86,8 @@ Additionally, {{< var vm.product >}} is developing a feature that enables develo -[^1]: [{{< var validmind.developer >}}](/validmind/validmind.html) \ No newline at end of file +[^1]: [{{< var validmind.developer >}}](/validmind/validmind.html) + +[^2]: [Configure AWS PrivateLink](/guide/configuration/configure-aws-privatelink.qmd) + +[^3]: [{{< var validmind.vpv >}} (VPV)](/about/deployment/virtual-private-validmind.qmd) \ No newline at end of file diff --git a/site/faq/faq-testing.qmd b/site/faq/faq-testing.qmd index 5f1e0db840..3f4f195bc3 100644 --- a/site/faq/faq-testing.qmd +++ b/site/faq/faq-testing.qmd @@ -37,10 +37,10 @@ Yes, {{< var vm.product >}} allows tests to be manipulated at several levels: - You can configure which tests are required to run programmatically depending on the model use case.[^4] - You can change the thresholds and parameters for default tests already available in the {{< var vm.developer >}} — for instance, changing the threshold parameter for the class imbalance flag.[^5] - You can also connect your own custom tests with the {{< var validmind.developer >}}. These custom tests are configurable and are able to run programmatically, just like the rest of the {{< var vm.developer >}}.[^6] +- Personalize tests further for your use case by using {{< var vm.product >}}'s `RawData` feature[^7] to customize the output of tests. ::: {.callout} -In addition to custom tests, you can also add use case and test-specific context for any test to enhance the LLM-generated test descriptions using the {{< var validmind.developer >}}.[^7] - +In addition to custom tests, you can also add use case and test-specific context for any test to enhance the LLM-generated test descriptions using the {{< var validmind.developer >}}.[^8] ::: {{< include _faq-explainability.qmd >}} @@ -69,4 +69,6 @@ In addition to custom tests, you can also add use case and test-specific context [^6]: [Can I use my own tests?](/developer/model-testing/testing-overview.qmd#can-i-use-my-own-tests) -[^7]: [Add context to LLM-generated test descriptions](/notebooks/how_to/add_context_to_llm_descriptions.ipynb) \ No newline at end of file +[^7]: [Understand and utilize `RawData` in {{< var vm.product >}} tests](/notebooks/how_to/understand_utilize_rawdata.ipynb) + +[^8]: [Add context to LLM-generated test descriptions](/notebooks/how_to/add_context_to_llm_descriptions.ipynb) \ No newline at end of file diff --git a/site/faq/faq.qmd b/site/faq/faq.qmd index 4dd76f2c0d..fe1f439ef6 100644 --- a/site/faq/faq.qmd +++ b/site/faq/faq.qmd @@ -12,7 +12,7 @@ listing: contents: - path: faq-organizations.qmd title: "Access and permissions" - categories: ["editions", "access", "permissions", "organizations", "user registration", "validmind platform"] + categories: ["access", "permissions", "organizations", "user registration", "validmind platform"] - path: faq-workflows.qmd title: "Model workflows" categories: ["workflows", "model lifecycle", "lifecycle statuses", "validmind platform", "validmind library"] @@ -39,7 +39,7 @@ listing: categories: ["supported libraries", "supported languages", "integrations", "images", "large language models", "explainability", "deployment options", "validmind library"] - path: faq-privacy.qmd title: "Data handling and privacy" - categories: ["editions", "data handling", "privacy", "confidentiality", "model activity", "auditing", "validmind platform", "validmind library"] + categories: ["data handling", "privacy", "confidentiality", "model activity", "auditing", "validmind platform", "validmind library"] --- ## Answers by topic diff --git a/site/guide/guides.qmd b/site/guide/guides.qmd index 6dca7d24ba..991a80a9ab 100644 --- a/site/guide/guides.qmd +++ b/site/guide/guides.qmd @@ -145,17 +145,6 @@ Then, work with documentation and customizable templates, and collaborate with m :::{#guides-model-documentation} ::: -::: {.grid} -::: {.g-col-8} -{{< video https://www.youtube.com/embed/videoseries?si=aP7yLOGO60h4vi8C&list=PL7I7ZrCoVCCP876aLRCjitZqYif96Whlz title='Developer Fundamentals' >}} - -::: {.video-cap .tc} -(10 videos) -::: - -::: -::: - ## Model validation Set up validation guidelines and prepare validation reports, work with findings and evidence, and collaborate with model developers within the {{< var vm.platform >}}: @@ -163,16 +152,6 @@ Set up validation guidelines and prepare validation reports, work with findings :::{#guides-model-validation} ::: -::: {.grid} -::: {.g-col-8} -{{< video https://www.youtube.com/embed/videoseries?si=JfBNClmsZYLS7m9F&list=PL7I7ZrCoVCCObEiuLLAL0Wm9GQ6x5U0q5 title='Validating Models 101' >}} - -::: {.video-cap .tc} -(5 videos) -::: - -::: -::: Review reports or export your documentation for external records: diff --git a/site/guide/model-documentation/work-with-content-blocks.qmd b/site/guide/model-documentation/work-with-content-blocks.qmd index 613061dc4f..a203c7ce2d 100644 --- a/site/guide/model-documentation/work-with-content-blocks.qmd +++ b/site/guide/model-documentation/work-with-content-blocks.qmd @@ -149,7 +149,7 @@ Test-driven or metric over time blocks can be re-added later on but **text block -[^1]: [Add mathematical formuals](#add-mathematical-formulas) +[^1]: [Add mathematical formulas](#add-mathematical-formulas) [^2]: [Collaborate with others](collaborate-with-others.qmd) @@ -169,4 +169,4 @@ Test-driven or metric over time blocks can be re-added later on but **text block [^10]: [View model activity](/guide/model-inventory/view-model-activity.qmd) -[^11]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) \ No newline at end of file +[^11]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) diff --git a/site/index.qmd b/site/index.qmd index 7a6d0c5f51..0a5f17a873 100644 --- a/site/index.qmd +++ b/site/index.qmd @@ -46,12 +46,12 @@ listing: fields: [title, description] contents: # MAKE-RELEASE-NOTES-LATEST-MARKER + - /releases/2025/2025-mar-07/release-notes.qmd - /releases/2025/2025-jan-31/release-notes.qmd - /releases/2024/2024-dec-24/release-notes.qmd - /releases/2024/2024-dec-06/release-notes.qmd - /releases/2024/2024-oct-22/release-notes.qmd - /releases/2024/2024-sep-25/release-notes.qmd - - /releases/2024/2024-sep-09/release-notes.qmd # MAKE-RELEASE-NOTES-OLDEST-MARKER - id: validmind-academy type: grid diff --git a/site/notebooks.zip b/site/notebooks.zip index 115a60ad76..06efc737df 100644 Binary files a/site/notebooks.zip and b/site/notebooks.zip differ diff --git a/site/notebooks/how_to/understand_utilize_rawdata.ipynb b/site/notebooks/how_to/understand_utilize_rawdata.ipynb new file mode 100644 index 0000000000..7354eae858 --- /dev/null +++ b/site/notebooks/how_to/understand_utilize_rawdata.ipynb @@ -0,0 +1,571 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c18ba8a2", + "metadata": {}, + "source": [ + "# Understand and utilize `RawData` in ValidMind tests\n", + "\n", + "Test functions in ValidMind can return a special object called *`RawData`*, which holds intermediate or unprocessed data produced somewhere in the test logic but not returned as part of the test's visible output, such as in tables or figures.\n", + "\n", + "- The `RawData` feature allows you to customize the output of tests, making it a powerful tool for creating custom tests and post-processing functions.\n", + "- `RawData` is useful when running post-processing functions with tests to recompute tabular outputs, redraw figures, or even create new outputs entirely.\n", + "\n", + "In this notebook, you'll learn how to access, inspect, and utilize `RawData` from ValidMind tests." + ] + }, + { + "cell_type": "markdown", + "id": "5b5b248c", + "metadata": {}, + "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [Setup](#toc1_) \n", + " - [Installation and intialization](#toc1_1_) \n", + " - [Load the sample dataset](#toc1_2_) \n", + " - [Initialize the ValidMind objects](#toc1_3_) \n", + "- [`RawData` usage examples](#toc2_) \n", + " - [Using `RawData` from the ROC Curve Test](#toc2_1_) \n", + " - [Pearson Correlation Matrix](#toc2_2_) \n", + " - [Precision-Recall Curve](#toc2_3_) \n", + " - [Using `RawData` in custom tests](#toc2_4_) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "6dd79a98", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Setup\n", + "\n", + "Before we can run our examples, we'll need to set the stage to enable running tests with the ValidMind Library. Since the focus of this notebook is on the `RawData` object, this section will merely summarize the steps instead of going into greater detail. \n", + "\n", + "\n", + "**To learn more about running tests with ValidMind:** [Run tests and test suites](https://docs.validmind.ai/developer/model-testing/testing-overview.html)" + ] + }, + { + "cell_type": "markdown", + "id": "5b6d8d15", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Installation and intialization\n", + "\n", + "First, let's make sure that the ValidMind Library is installed and ready to go, and our Python environment set up for data analysis:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04eb084e", + "metadata": {}, + "outputs": [], + "source": [ + "# Install the ValidMind Library\n", + "%pip install -q validmind\n", + "\n", + "# Initialize the ValidMind Library\n", + "import validmind as vm\n", + "\n", + "# Import the `xgboost` library with an alias\n", + "import xgboost as xgb\n" + ] + }, + { + "cell_type": "markdown", + "id": "5e6aa2cb", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Load the sample dataset\n", + "\n", + "Then, we'll import a sample ValidMind dataset and preprocess it:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50d72eba", + "metadata": {}, + "outputs": [], + "source": [ + "# Import the `customer_churn` sample dataset\n", + "from validmind.datasets.classification import customer_churn\n", + "raw_df = customer_churn.load_data()\n", + "\n", + "# Preprocess the raw dataset\n", + "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)\n", + "\n", + "# Separate features and targets\n", + "x_train = train_df.drop(customer_churn.target_column, axis=1)\n", + "y_train = train_df[customer_churn.target_column]\n", + "x_val = validation_df.drop(customer_churn.target_column, axis=1)\n", + "y_val = validation_df[customer_churn.target_column]\n", + "\n", + "# Create an `XGBClassifier` object\n", + "model = xgb.XGBClassifier(early_stopping_rounds=10)\n", + "model.set_params(\n", + " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", + ")\n", + "\n", + "# Train the model using the validation set\n", + "model.fit(\n", + " x_train,\n", + " y_train,\n", + " eval_set=[(x_val, y_val)],\n", + " verbose=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e3895d35", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind objects" + ] + }, + { + "cell_type": "markdown", + "id": "c0e441f4", + "metadata": {}, + "source": [ + "Before you can run tests, you'll need to initialize a ValidMind dataset object, as well as a ValidMind model object that can be passed to other functions for analysis and tests on the data:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2310bc4", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the dataset object\n", + "vm_raw_dataset = vm.init_dataset(\n", + " dataset=raw_df,\n", + " input_id=\"raw_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " class_labels=customer_churn.class_labels,\n", + " __log=False,\n", + ")\n", + "\n", + "# Initialize the datasets into their own dataset objects\n", + "vm_train_ds = vm.init_dataset(\n", + " dataset=train_df,\n", + " input_id=\"train_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " __log=False,\n", + ")\n", + "vm_test_ds = vm.init_dataset(\n", + " dataset=test_df,\n", + " input_id=\"test_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " __log=False,\n", + ")\n", + "\n", + "# Initialize a model object\n", + "vm_model = vm.init_model(\n", + " model,\n", + " input_id=\"model\",\n", + " __log=False,\n", + ")\n", + "\n", + "# Assign predictions to the datasets\n", + "vm_train_ds.assign_predictions(\n", + " model=vm_model,\n", + ")\n", + "\n", + "vm_test_ds.assign_predictions(\n", + " model=vm_model,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "25ec99fc", + "metadata": {}, + "source": [ + "\n", + "\n", + "## `RawData` usage examples\n", + "\n", + "Once you're set up to run tests, you can then try out the following examples:\n", + "\n", + " - [Using `RawData` from the ROC Curve Test](#toc2_1_) \n", + " - [Pearson Correlation Matrix](#toc2_2_) \n", + " - [Precision-Recall Curve](#toc2_3_) \n", + " - [Using `RawData` in custom tests](#toc2_4_) " + ] + }, + { + "cell_type": "markdown", + "id": "33d79841", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Using `RawData` from the ROC Curve Test\n", + "\n", + "In this introductory example, we run the [ROC Curve](https://docs.validmind.ai/tests/model_validation/sklearn/ROCCurve.html) test, inspect its `RawData` output, and then create a custom ROC curve using the raw data values.\n", + "\n", + "First, let's run the default ROC Curve test for comparsion with later iterations:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58a3a779", + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.tests import run_test\n", + "\n", + "# Run the ROC Curve test normally\n", + "result_roc = run_test(\n", + " \"validmind.model_validation.sklearn.ROCCurve\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " generate_description=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "66c44fe0", + "metadata": {}, + "source": [ + "Now let's assume we want to create a custom version of the above figure. First, let's inspect the raw data that this test produces so we can see what we have to work with.\n", + "\n", + "`RawData` objects have a `inspect()` method that will pretty print the attributes of the object to be able to quickly see the data and its types:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "513ce01e", + "metadata": {}, + "outputs": [], + "source": [ + "# Inspect the RawData output from the ROC test\n", + "print(\"RawData from ROC Curve Test:\")\n", + "result_roc.raw_data.inspect()" + ] + }, + { + "cell_type": "markdown", + "id": "586f3a12", + "metadata": {}, + "source": [ + "As we can see, the ROC Curve returns a `RawData` object with the following attributes:\n", + "- **`fpr`:** A list of false positive rates\n", + "- **`tpr`:** A list of true positive rates\n", + "- **`auc`:** The area under the curve\n", + "\n", + "This should be enough to create our own custom ROC curve via a post-processing function without having to create a whole new test from scratch and without having to recompute any of the data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "613778d2", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "from validmind.vm_models.result import TestResult\n", + "\n", + "\n", + "def custom_roc_curve(result: TestResult):\n", + " # Extract raw data from the test result\n", + " fpr = result.raw_data.fpr\n", + " tpr = result.raw_data.tpr\n", + " auc = result.raw_data.auc\n", + "\n", + " # Create a custom ROC curve plot\n", + " fig = plt.figure()\n", + " plt.plot(fpr, tpr, label=f\"Custom ROC (AUC = {auc:.2f})\", color=\"blue\")\n", + " plt.plot([0, 1], [0, 1], linestyle=\"--\", color=\"gray\", label=\"Random Guess\")\n", + " plt.xlabel(\"False Positive Rate\")\n", + " plt.ylabel(\"True Positive Rate\")\n", + " plt.title(\"Custom ROC Curve from RawData\")\n", + " plt.legend()\n", + "\n", + " # close the plot to avoid it automatically being shown in the notebook\n", + " plt.close()\n", + "\n", + " # remove existing figure\n", + " result.remove_figure(0)\n", + "\n", + " # add new figure\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "# test it on the existing result\n", + "modified_result = custom_roc_curve(result_roc)\n", + "\n", + "# show the modified result\n", + "modified_result.show()" + ] + }, + { + "cell_type": "markdown", + "id": "794d026c", + "metadata": {}, + "source": [ + "Now that we have created a post-processing function and verified that it works on our existing test result, we can use it directly in `run_test()` from now on:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c7566f3", + "metadata": {}, + "outputs": [], + "source": [ + "result = run_test(\n", + " \"validmind.model_validation.sklearn.ROCCurve\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " post_process_fn=custom_roc_curve,\n", + " generate_description=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "1d0b94aa", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Pearson Correlation Matrix\n", + "\n", + "In this next example, try commenting out the `post_process_fn` argument in the following cell and see what happens between different runs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c57fb01b", + "metadata": {}, + "outputs": [], + "source": [ + "import plotly.graph_objects as go\n", + "\n", + "\n", + "def custom_heatmap(result: TestResult):\n", + " corr_matrix = result.raw_data.correlation_matrix\n", + "\n", + " heatmap = go.Heatmap(\n", + " z=corr_matrix.values,\n", + " x=list(corr_matrix.columns),\n", + " y=list(corr_matrix.index),\n", + " colorscale=\"Viridis\",\n", + " )\n", + " fig = go.Figure(data=[heatmap])\n", + " fig.update_layout(title=\"Custom Heatmap from RawData\")\n", + "\n", + " plt.close()\n", + "\n", + " result.remove_figure(0)\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "\n", + "result_corr = run_test(\n", + " \"validmind.data_validation.PearsonCorrelationMatrix\",\n", + " inputs={\"dataset\": vm_test_ds},\n", + " generate_description=False,\n", + " # COMMENT OUT `post_process_fn`\n", + " post_process_fn=custom_heatmap,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "0a7cbbc6", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Precision-Recall Curve\n", + "\n", + "Then, let's try the same thing with the [Precision-Recall Curve](https://docs.validmind.ai/tests/model_validation/sklearn/PrecisionRecallCurve.html) test:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d16c5209", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_pr_curve(result: TestResult):\n", + " precision = result.raw_data.precision\n", + " recall = result.raw_data.recall\n", + "\n", + " fig = plt.figure()\n", + " plt.plot(recall, precision, label=\"Precision-Recall Curve\")\n", + " plt.xlabel(\"Recall\")\n", + " plt.ylabel(\"Precision\")\n", + " plt.title(\"Custom Precision-Recall Curve from RawData\")\n", + " plt.legend()\n", + "\n", + " plt.close()\n", + " result.remove_figure(0)\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "result_pr = run_test(\n", + " \"validmind.model_validation.sklearn.PrecisionRecallCurve\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " generate_description=False,\n", + " # COMMENT OUT `post_process_fn`\n", + " post_process_fn=custom_pr_curve,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e25391a4", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Using `RawData` in custom tests\n", + "\n", + "These examples demonstrate some very simple ways to use the `RawData` feature of ValidMind tests. The majority of ValidMind-developed tests return some form of raw data that can be used to customize the output of the test, but you can also create your own tests that return `RawData` objects and use them in the same way.\n", + "\n", + "Let's take a look at how this can be done in custom tests. To start, define and run your custom test:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc6a389f", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from validmind import test, RawData\n", + "from validmind.vm_models import VMDataset, VMModel\n", + "\n", + "\n", + "@test(\"custom.MyCustomTest\")\n", + "def MyCustomTest(dataset: VMDataset, model: VMModel) -> tuple[go.Figure, RawData]:\n", + " \"\"\"Custom test that produces a figure and a RawData object\"\"\"\n", + " # pretend we are using the dataset and model to compute some data\n", + " # ...\n", + "\n", + " # create some fake data that will be used to generate a figure\n", + " data = pd.DataFrame({\"x\": [10, 20, 30, 40, 50], \"y\": [10, 20, 30, 40, 50]})\n", + "\n", + " # create the figure (scatter plot)\n", + " fig = go.Figure(data=go.Scatter(x=data[\"x\"], y=data[\"y\"]))\n", + "\n", + " # now let's create a RawData object that holds the \"computed\" data\n", + " raw_data = RawData(scatter_data_df=data)\n", + "\n", + " # finally, return both the figure and the raw data\n", + " return fig, raw_data\n", + "\n", + "\n", + "my_result = run_test(\n", + " \"custom.MyCustomTest\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " generate_description=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "854c219c", + "metadata": {}, + "source": [ + "We can see that the test result shows the figure. But since we returned a `RawData` object, we can also inspect the contents and see how we could use it to customize or regenerate the figure in the post-processing function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1cb661d1", + "metadata": {}, + "outputs": [], + "source": [ + "my_result.raw_data.inspect()" + ] + }, + { + "cell_type": "markdown", + "id": "55ad4acd", + "metadata": {}, + "source": [ + "We can see that we get a nicely-formatted preview of the dataframe we stored in the raw data object. Let's go ahead and use it to re-plot our data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1242083", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_plot(result: TestResult):\n", + " data = result.raw_data.scatter_data_df\n", + "\n", + " # use something other than a scatter plot\n", + " fig = go.Figure(data=go.Bar(x=data[\"x\"], y=data[\"y\"]))\n", + " fig.update_layout(title=\"Custom Bar Chart from RawData\")\n", + " fig.update_xaxes(title=\"X Axis\")\n", + " fig.update_yaxes(title=\"Y Axis\")\n", + "\n", + " result.remove_figure(0)\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "result = run_test(\n", + " \"custom.MyCustomTest\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " post_process_fn=custom_plot,\n", + " generate_description=False,\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/site/python-docs.zip b/site/python-docs.zip index 127abb3922..67d109e059 100644 Binary files a/site/python-docs.zip and b/site/python-docs.zip differ diff --git a/site/releases/2024/2024-jan-26/highlights.qmd b/site/releases/2024/2024-jan-26/highlights.qmd index e4d6506b96..9325503592 100644 --- a/site/releases/2024/2024-jan-26/highlights.qmd +++ b/site/releases/2024/2024-jan-26/highlights.qmd @@ -295,6 +295,21 @@ To work with Azure OpenAI API endpoints, you need to set the following environme - We have updated our demonstration notebooks for large language models (LLMs) to provide the correct support for `openai >= 1.0.0`. - Previously, some notebooks were using an older version of the OpenAI client API. +## Breaking changes + +### {{< var validmind.developer >}} (v1.25.3) + + +#### Removed deprecated {{< var validmind.api >}} methods + +The {{< var vm.api >}} methods `run_template` and `run_test_plan` had been deprecated previously. They have now been removed from the {{< var validmind.developer >}}. + +You'll need to update your code to use the recommended high-level API methods: + +- **`run_template` (removed):** Use [`run_documentation_tests()`](/validmind/validmind.html#run_documentation_tests){target="_blank"} +- **`run_test_plan` (removed):** Use [`run_test_suite()`](/validmind/validmind.html#run_test_suite){target="_blank"} + + ## Deprecations ### {{< var validmind.developer >}} (v1.25.3) @@ -337,16 +352,6 @@ The {{< var validmind.developer >}} now supports passing custom inputs[^3] as an :::: - -#### Removed deprecated {{< var validmind.api >}} methods - -The {{< var vm.api >}} methods `run_template` and `run_test_plan` had been deprecated previously. They have now been removed from the {{< var validmind.developer >}}. - -You'll need to update your code to use the recommended high-level API methods: - -- **`run_template` (removed):** Use [`run_documentation_tests()`](/validmind/validmind.html#run_documentation_tests){target="_blank"} -- **`run_test_plan` (removed):** Use [`run_test_suite()`](/validmind/validmind.html#run_test_suite){target="_blank"} - ## Documentation ### User guide updates diff --git a/site/releases/2025/2025-jan-31/release-notes.qmd b/site/releases/2025/2025-jan-31/release-notes.qmd index c73e95078e..8502ae9693 100644 --- a/site/releases/2025/2025-jan-31/release-notes.qmd +++ b/site/releases/2025/2025-jan-31/release-notes.qmd @@ -34,7 +34,7 @@ listing: max-description-length: 250 # image-height: 100% contents: - - path: https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb) + - path: https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb title: "Ongoing monitoring for application scorecard" description: "Open notebook in JupyterHub {{< fa chevron-right >}}" - id: e2e-template diff --git a/site/releases/2025/2025-mar-07/export-validation-report.png b/site/releases/2025/2025-mar-07/export-validation-report.png new file mode 100644 index 0000000000..4228b79636 Binary files /dev/null and b/site/releases/2025/2025-mar-07/export-validation-report.png differ diff --git a/site/releases/2025/2025-mar-07/release-notes.qmd b/site/releases/2025/2025-mar-07/release-notes.qmd new file mode 100644 index 0000000000..6a4c547b05 --- /dev/null +++ b/site/releases/2025/2025-mar-07/release-notes.qmd @@ -0,0 +1,164 @@ +--- +title: "March 7, 2025" +listing: + - id: academy + type: grid + grid-columns: 1 + max-description-length: 250 + # image-height: 100% + contents: + - path: ../../../training/training.qmd + title: "{{< var validmind.training >}} {{< fa chevron-right >}}" + description: "Our training modules are interactive. They combine instructional content with our live product and are easy to use." + fields: [title, description] +--- + +This release focuses on quality of life improvements for our users: {{< var validmind.training >}} training journeys, transparent history for breaking changes and deprecations, and additional validation report configuration options in the {{< var validmind.platform >}}. + +::: {.highlights} + +## Release highlights — `25.02` + +::: {.callout} +Our documentation now follows the new **unified versioning scheme** for our software as of our [`25.01` release on January 31, 2025](/releases/2025/2025-jan-31/release-notes.qmd). +::: + +### Documentation + + +#### A guided {{< var validmind.training >}} experience + +Wondering you can get started with {{< var vm.product >}}? Check out our free training material, now with an upgraded experience! + +- We've added a training overview, learning paths, and a sample training plan to help you navigate your training journey. +- We've also introduced an frequently asked questions section mapping common questions to relevant learning paths. + +::: {.column-margin} +:::{#academy} +::: + +::: + + +#### Breaking changes and deprecation history + +We've introduced an interactive historical record of breaking changes and deprecations to the {{< var validmind.product >}} to better inform users about important updates to our product. You can review this history under our Releases section. + +- In addition to announcement and planned obsolescence dates, this history provides easy access to any relevant messaging associated with the change, such as version release notes and blog posts. +- These histories grouped by calendar year are searchable and can be filtered by product area ({{< var validmind.developer >}}, {{< var validmind.platform >}}), associated versioning, and the type of change. + +::: {.column-margin} +[Breaking changes & deprecations](/releases/breaking-changes/breaking-changes.qmd){.button .button-green target="_blank"} + +::: + +::: + +## Enhancements + +### {{< var validmind.platform >}} (v1.29.20) + + +#### Additional validation report export options + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-70-ns .pr4} +We now support additional configuration when exporting validation reports, allowing you to choose what gets included in your generated document. + +::: {.tc} + +::: + +::: + +::: {.w-30-ns} +[Export documentation](/guide/model-documentation/export-documentation.qmd#export-validation-report){.button} +::: + +:::: + +![New Export Validation Report options menu](export-validation-report.png){width=80% fig-alt="A screenshot showing the new Export Validation Report options menu" .screenshot} + + + + + + + + + + + + + +{{< include /releases/_how-to-upgrade.qmd >}} + + + + diff --git a/site/releases/breaking-changes/README.md b/site/releases/breaking-changes/README.md new file mode 100644 index 0000000000..f8aba4fc11 --- /dev/null +++ b/site/releases/breaking-changes/README.md @@ -0,0 +1,94 @@ +# Breaking changes and deprecations + +This guide walks you through how to add a breaking change or deprecation to our [~/releases/breaking-changes/breaking-changes.qmd](breaking-changes.qmd) history. + +## Before you begin + +In order to get Quarto display the neat interactive table in our HTML output, you need to have R and some dependencies installed in your environment. + +### Install R + +1. **Download and install R** from the Comprehensive R Archive Network: [CRAN](https://cran.r-project.org/index.html) + - If you're using VS Code, you may also want to install the following extensions: [R](https://marketplace.visualstudio.com/items?itemName=REditorSupport.r), [R Extension Pack](https://marketplace.visualstudio.com/items?itemName=Ikuyadeu.r-pack) +2. **Open up an R terminal.** In VS Code you can press `CTRL/CMD + SHIFT + P` to open up the command palette, then type in and select `R: Create R Terminal`. + +### Install R dependencies + +In the R terminal, run: + +```bash +install.packages("DT") +install.packages("readr") +install.packages("stringr") +install.packages("lubridate") +``` + +These are the packages used to generate cool interactive tables from the yearly CSVs: + +- **[DataTables](https://rstudio.github.io/DT/)** — Displays a R data object as an HTML table. +- **[readr](https://readr.tidyverse.org/)** — Parses the yearly `.csv` files for the table. +- **[stringr](https://readr.tidyverse.org/)** — Parses markdown links in the `.csv` and turns them into HTML for the output table so they can be displayed properly. +- **[lubridate](https://lubridate.tidyverse.org/)** — Parses and reformats the ISO 8601 date in the CSV into the Mon DD, YYYY format in the output table. + +## Creating entries + +### Add a year + +If you're adding the first entry for that year, you'll need to add that year to the history section first: + +#### Create the yearly CSV + +Make a copy of `~/releases/breaking-changes/year.csv` in the `~/releases/breaking-changes/history/` directory. For example: + +> `~/releases/breaking-changes/year.csv` > `~/releases/breaking-changes/history/2025.csv` + +> [!CAUTION] +> Due to the way DataTables processes the input, there needs to be an empty row/line break at the end of the sheet. + +#### Insert CSV as table + +1. In `~/releases/breaking-changes/breaking-changes.qmd` under ``, insert the new year `h3` header. For example: + + ```markdown + ### 2025 + ``` + +2. Then under that header, call the functions to display the interactive table for that year under a named and fenced `{r}` code block. For example: + + ```R + ```{r 2025} + ## Render 2025.csv + my_data_2025 <- read_csv_data(2025) + my_data_2025 <- convert_markdown_links(my_data_2025) + my_data_2025 <- format_dates_in_data(my_data_2025) + render_table(my_data_2025) + ``` + ``` + + Replace the year with the actual year you're intending to display. + +### Add an entry + +#### Create collateral + +You'll need the links for the release notes associated with the breaking change or deprecation to include in the table, so make sure those are published that first unless you want to fill them in at a later date. + +#### Enter an entry + +Open up the `.csv` for the year you want to update and create a new entry under the header row with the following information: + +| Column name | Content | Example | +|---|---|---| +| Change | Name of the feature being broken or deprecated | `Standard inputs are deprecated` | +| Product area | Applicable variable for the ValidMind Library or ValidMind Platform | `{{< var validmind.developer >}} `| +| Version | Associated library or platform version attached to the announcement prefaced by a `v` (Linked to release tags) | `v1.25.3` | +| Type | Whether it's a breaking change or deprecation | `Deprecation` | +| Date announced | ISO 8601 date of announcement | `2024-01-26` | +| Release notes | Markdown format **HTML** link to the associated release notes enclosed by `""` | `"[Read](/releases/2024/2024-jan-26/highlights.html#standard-inputs-are-deprecated)"` | +| Date effective | ISO 8601 date of feature removal/deprecation | `2024-01-26` | + +> [!NOTE] +> You can refer to the **[example `.csv`](example.csv)** for formatting guidance. +> +> If you're using VS Code, [Excel Viewer](https://marketplace.visualstudio.com/items?itemName=GrapeCity.gc-excelviewer) is a great extension to make editing `.csv` files more accessible. + diff --git a/site/releases/breaking-changes/breaking-changes.qmd b/site/releases/breaking-changes/breaking-changes.qmd new file mode 100644 index 0000000000..7fb72b7d33 --- /dev/null +++ b/site/releases/breaking-changes/breaking-changes.qmd @@ -0,0 +1,36 @@ +--- +title: "Breaking changes and deprecations" +date: last-modified +# DON'T DISPLAY THE SOURCE CODE FOR THE TABLES ON THE PAGE +execute: + echo: false +--- + +```{r import-script} +# Import custom `breaking_changes` script +source("breaking_changes.R") +``` + +## Definitions + +deprecation +: - Feature not recommended for use. +- May be removed in a future major release. + +breaking change +: - Feature will be removed in a future major release. +- Requires users to make a corresponding change to their code, settings, or workflows. + +## History + + + +### 2024 + +```{r 2024} +## Render 2024.csv +my_data_2024 <- read_csv_data(2024) +my_data_2024 <- convert_markdown_links(my_data_2024) +my_data_2024 <- format_dates_in_data(my_data_2024) +render_table(my_data_2024) +``` diff --git a/site/releases/breaking-changes/breaking_changes.R b/site/releases/breaking-changes/breaking_changes.R new file mode 100644 index 0000000000..30e202206e --- /dev/null +++ b/site/releases/breaking-changes/breaking_changes.R @@ -0,0 +1,71 @@ +# Load necessary libraries with output messages supressed +suppressPackageStartupMessages(library(DT)) +suppressPackageStartupMessages(library(readr)) +suppressPackageStartupMessages(library(stringr)) +suppressPackageStartupMessages(library(lubridate)) + +# Read CSV while preserving column names and treating all text as characters +read_csv_data <- function(year) { + file_name <- paste0("history/", year, ".csv") + data <- read.csv(file_name, check.names = FALSE, stringsAsFactors = FALSE) + return(data) +} + +# Cnvert Markdown-style links to proper HTML +convert_markdown_links <- function(data) { + data[] <- lapply(data, function(column) { + column <- str_replace_all(column, "\\[(.*?)\\]\\((.*?)\\)", "\\1") + return(column) + }) + return(data) +} + +# Reformat ISO dates to Mon DD, YYYY +format_date <- function(date_column) { + parsed_dates <- ymd(date_column) + formatted_dates <- format(parsed_dates, "%b %d, %Y") + return(formatted_dates) +} + +# Apply date formatting to `Date announced` and `Date effective` columns +format_dates_in_data <- function(data) { + if ("Date announced" %in% names(data)) { + data[["Date announced"]] <- format_date(data[["Date announced"]]) + } + if ("Date effective" %in% names(data)) { + data[["Date effective"]] <- paste0( + "", format_date(data[["Date effective"]]), "" + ) + } + return(data) +} + +# Render an interactive searchable and filterable table +render_table <- function(data) { + datatable( + data, + options = list( + pageLength = 10, # Define default length of entries + dom = 'fltpi', # Table display options: Search/filter, length menu, the table istself, pagination, table info + initComplete = JS( # Filter `Product area`, `Version`, & `Type` using JavaScript + "function(settings, json) { + [1, 2, 3].forEach(function(index) { + var column = this.api().column(index); + var header = $(column.header()); // Keep original header text + var select = $('
') + .appendTo(header) // Append instead of replacing + .on('change', function() { + var val = $.fn.dataTable.util.escapeRegex($(this).val()); + column.search(val ? '^' + val + '$' : '', true, false).draw(); + }); + column.data().unique().sort().each(function(d, j) { + select.append('') + }); + }.bind(this)); + }" + ) + ), + escape = FALSE, # Allow HTML rendering + rownames = FALSE # Disable row numbering + ) +} \ No newline at end of file diff --git a/site/releases/breaking-changes/example.csv b/site/releases/breaking-changes/example.csv new file mode 100644 index 0000000000..403a38f8c3 --- /dev/null +++ b/site/releases/breaking-changes/example.csv @@ -0,0 +1,6 @@ +Change,Product area,Version,Type,Date announced,Release notes,Date effective +ValidMind Library deprecation,{{< var validmind.developer >}},v00.0,Deprecation,YYYY-MM-DD,"[Read](url.html)",YYYY-MM-DD +ValidMind Platform breaking change,{{< var validmind.platform >}},v00.0,Breaking change,YYYY-MM-DD,"[Read](url.html)","YYYY-MM-DD + + + diff --git a/site/releases/breaking-changes/history/2024.csv b/site/releases/breaking-changes/history/2024.csv new file mode 100644 index 0000000000..c2ed26fdb9 --- /dev/null +++ b/site/releases/breaking-changes/history/2024.csv @@ -0,0 +1,7 @@ +Change,Product area,Version,Type,Date announced,Release notes,Date effective +Standard inputs are deprecated,{{< var validmind.developer >}},v1.25.3,Deprecation,2024-01-26,"[Read](/releases/2024/2024-jan-26/highlights.html#standard-inputs-are-deprecated)",2024-01-26 +Removed deprecated Python Library API methods,{{< var validmind.developer >}},v1.25.3,Breaking change,2024-01-26,"[Read](/releases/2024/2024-jan-26/highlights.html#removed-deprecated-methods)",2024-01-26 + + + + diff --git a/site/releases/breaking-changes/year.csv b/site/releases/breaking-changes/year.csv new file mode 100644 index 0000000000..1c23d3b62f --- /dev/null +++ b/site/releases/breaking-changes/year.csv @@ -0,0 +1,6 @@ +Change,Product area,Version,Type,Date announced,Release notes,,Date effective +ValidMind Library deprecation,{{< var validmind.developer >}},v00.0,Deprecation,YYYY-MM-DD,"[Read](url.html)",YYYY-MM-DD +ValidMind Platform breaking change,{{< var validmind.platform >}},v00.0,Breaking change,YYYY-MM-DD,"[Read](url.html)",YYYY-MM-DD + + + diff --git a/site/styles.css b/site/styles.css index 48d0b2873b..f2f5ab5127 100644 --- a/site/styles.css +++ b/site/styles.css @@ -270,6 +270,14 @@ pre, pre.python, pre.bash, pre.yaml, pre.markdown { font-size: larger; } +.smallercaps { + font-weight: bold; + font-size: smaller; + font-variant: small-caps; + position: relative; + top: -5px; +} + input[type="checkbox"] { width: 1em; height: 1rem; @@ -299,6 +307,45 @@ input[type="checkbox"][checked] { color: #083E44; } +.attn-yellow { + margin-top: 20px; + margin-bottom: 40px; + border: 2px solid #083E44; + border-left-width: 0px; + border-right-width: 0px; + padding-top: 20px; + padding-bottom: 20px; + padding-left: 30px; + padding-right: 40px; + background-color: rgb(250, 245, 145); +} + +.attn-orange { + margin-top: 20px; + margin-bottom: 40px; + border: 2px solid #083E44; + border-left-width: 0px; + border-right-width: 0px; + padding-top: 20px; + padding-bottom: 20px; + padding-left: 30px; + padding-right: 40px; + background-color: rgb(252, 169, 110); +} + +.attn-red { + margin-top: 20px; + margin-bottom: 40px; + border: 2px solid #083E44; + border-left-width: 0px; + border-right-width: 0px; + padding-top: 20px; + padding-bottom: 20px; + padding-left: 30px; + padding-right: 40px; + background-color: rgb(255, 109, 109); +} + .highlights { margin-top: 20px; margin-bottom: 40px; @@ -434,6 +481,26 @@ figcaption { border: 1px solid #083E44; } +.button-small { + /* font-size: 0.6em; */ + font-variant: small-caps; + color: #083E44; + background-color: #B5B5B510; + border: 1px solid #083E44; + /* font-weight: bold; */ + border-radius: 3px; + padding: 2px 5px; + vertical-align: middle; + line-height: 35px; +} + +.button-small:hover { + color: #083E44; + text-decoration: none; + background-color: #EAF8FA; + border: 1px solid #083E44; +} + .cta { text-decoration: none; padding-left: 1rem; @@ -597,10 +664,7 @@ hr.sidebar-divider.hi { nav#TOC { padding: 13px; - background-color: white; border: 1px solid #bdc1c4; - /* border-bottom: 2px solid #196972; - border-right: 2px solid #196972; */ border-radius: 3px; background-color: #FAFBFB; } diff --git a/site/training/_metadata.yml b/site/training/_metadata.yml index d0a3cf5471..ec31361d29 100644 --- a/site/training/_metadata.yml +++ b/site/training/_metadata.yml @@ -1,4 +1,4 @@ format: html: page-layout: full - css: assets/training.css \ No newline at end of file + css: assets/training.css diff --git a/site/training/_sidebar.yaml b/site/training/_sidebar.yaml new file mode 100644 index 0000000000..e84275859d --- /dev/null +++ b/site/training/_sidebar.yaml @@ -0,0 +1,14 @@ +website: + sidebar: + - id: validmind-training + title: "ValidMind Training" + collapsed: false + collapse-level: 2 + contents: + - text: " Training" + file: "training/training.qmd" + - text: "---" + - training/program/program-overview.qmd + - training/program/learning-paths.qmd + - training/program/sample-training-plan.qmd + - training/program/training-faq.qmd diff --git a/site/training/assets/training.css b/site/training/assets/training.css index 9238f58bcf..a56f84ca58 100644 --- a/site/training/assets/training.css +++ b/site/training/assets/training.css @@ -87,9 +87,16 @@ div.callout.callout { text-decoration: none; } +#quarto-sidebar.sidebar { + background-color: #FFFFFF !important; +} + .sidebar-header .sidebar-title a { text-decoration: none; - /* color: #DE257E; */ +} + +.sidebar.sidebar-navigation:not(.rollup) { + border-right: none !important; } .quarto-listing-default .no-anchor.listing-title a { @@ -104,6 +111,11 @@ div.callout.callout { background-color: #FAFBFB !important; } +nav#TOC { + border: 1px solid #FFFFFF !important; + background-color: #FFFFFF !important; +} + nav#TOC ul > li > a.nav-link { color: #3E6C69 !important; text-decoration: none; diff --git a/site/training/program/learning-paths.qmd b/site/training/program/learning-paths.qmd new file mode 100644 index 0000000000..5de373bb44 --- /dev/null +++ b/site/training/program/learning-paths.qmd @@ -0,0 +1,485 @@ +--- +title: "Learning paths" +sidebar: validmind-training +toc-expand: true +--- + +Each path provides a structured, hands-on approach to mastering key {{< var vm.product >}}’s features that you can apply in your work. + +::: {.callout title="We are in the process of expanding our training program."} +If you have a specific topic in mind, [let us know](/training/training.qmd#connect-with-us). +::: + +## End-to-end courses + +Develop role-based expertise for your entire organization. + +### Developer Fundamentals + +As a developer who is new to ValidMind, learn how to generate model documentation, add your own tests, edit the content, and then submit your documentation for approval. + +::: {.attn .hidden} +:::: {.flex .flex-wrap .justify-around} + +::: {.w-30-ns} +{{< fa check >}} **Course content** + +- Split 3 modules -> 4 modules +- Expand process overview content +- Expand template content to include editing templates and adding tests +- Rename "Module 2: Start Documenting & Testing" to "Module 2: Start Documenting" +- Add missing content for "Register a model" in Module 1 +::: + +::: {.w-30-ns} +{{< fa check >}} **Videos ** + +- 10 videos +- Minor update to collaboration video (multi-person comment threads) +::: + +::: {.w-30-ns .pl4 .bl} +**Notes** + +- FUTURE: 1 intro notebook -> 4 notebooks that progressively build on each other +::: + +:::: +::: + +::: {.attn} +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 1
Initialize the ValidMind Library + +- Install the ValidMind Library +- Initialize your development environment +- Register a model +- Get your code snippet +- Work with documentation templates +::: + +::: {.w-50-ns} +#### Module 2
Start Documenting & Testing + +- Initialize your datasets +- Run tabular data tests +- Utilize test output +- Document results based on datasets +- Test your model + +- Run the model evaluation tests +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 3
Implement Custom Tests + +- Create a confusion matrix plot +- Add parameters to custom tests +- Pass parameters to custom tests +- Log the confusion matrix results +- Using external test providers +- Initializing a local test provider +::: + +::: {.w-50-ns} +#### Module 4
Finalize Testing & Documentation + +- Include custom test results in your documentation +- Viewing and updating the configuration for the entire model documentation template +- Update the config +::: + +:::: +::: + +### Validator Fundamentals + +As a validator who is new to ValidMind, learn how to review model documentation, prepare your validation report, track issues, and submit your report for approval. + +::: {.attn .hidden} + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-30-ns} +{{< fa check >}} **Course content** + +- Split 2 modules -> 3 modules +- Add missing content: + - Preview templates for validation reports + - How to work with validation reports + - Expand process overview content +::: + +::: {.w-30-ns} +{{< fa minus >}} **Video** + +- 5 videos +- Replace stale '101' intro +- Review for stale content +::: + +::: {.w-30-ns} + +::: + +:::: +::: + +::: {.attn} +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 1
Reviewing & Analyzing Models + +- Access validation reports +- Preview templates for validation reports +- How to work with validation reports +::: + +::: {.w-50-ns} +#### Module 2
Reviewing & Analyzing Models + +- Reviewing model documentation +- Analyzing test results +- Add findings +- Link evidence +- Assessing compliance +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 3
Issue Resolution & Submission + +- Tracking issue resolution +- Submitting validation reports for approval +::: + +::: {.w-50-ns} + +::: + +:::: +::: + +### Administrator Fundamentals + +As an administrator who is new to ValidMind, learn how to onboard your organization, manage users and roles, and manage permissions for specific roles. + +::: {.attn .hidden} + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-30-ns} +{{< fa check >}} **Most course content exists** + +- Split 2 modules -> 4 modules +- Expand workflow content section +- Add model inventory content +- Add missing content: + - Assigning users to workflows + - Mapping users to groups and roles +::: + +::: {.w-30-ns} +{{< fa x >}} **No videos** + +- Create 4 videos, one per module +::: + +::: {.w-30-ns .pl4 .bl} +**Notes** + +- FUTURE: Add admin UI coverage +::: + +:::: + +::: + +::: {.attn} +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 1
Onboarding + +- Setting up an organization +- Inviting users via email +::: + +::: {.w-50-ns} +#### Module 2
User Management + +- Managing roles, permissions, and groups +- Mapping users to groups and roles +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 3
Workflows + +- Configuring workflows +- Assigning users to workflows +::: + +::: {.w-50-ns} +#### Module 4
Templates & Model Inventory + +- Customize templates +- Configure model inventory fields +::: + +:::: +::: + +### Monitoring Fundamentals
[coming soon]{.smallercaps .pink} + +As a monitoring user who is new to ValidMind, learn how to generate ongoing monitoring documentation, schedule monitoring runs, and review the results. + +::: {.attn .hidden} +:::: {.flex .flex-wrap .justify-around} + +::: {.w-30-ns} +{{< fa x >}} **No course** + +- Create complete course with 3 modules +- Create course registration page & add to training portal +::: + +::: {.w-30-ns} +{{< fa x >}} **No videos** + +- Create 3 videos +::: + +::: {.w-30-ns .pl4 .bl} +**Notes** + +- Based on [ongoing monitoring notebook](https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb) +::: + +:::: +::: + +::: {.attn} +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 1
Ongoing Monitoring of Models + +- How monitoring integrates with model documentation +- Enable monitoring for a model +- Preview the monitoring template +::: + +::: {.w-50-ns} +#### Module 2
Monitoring Setup + +- Load the reference and monitoring datasets +- Load the production model +- Initialize datasets and a model object +- Assign predictions to the datasets +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 3
Scheduling & Execution + +- Run the ongoing monitoring tests +- Conduct target and feature drift testing +- Model performance monitoring tests +- How to schedule quarterly or monthly monitoring runs +::: + +::: {.w-50-ns} +::: + +:::: +::: + +## Standalone courses + +Useful skills for anyone who wants to use ValidMind. + +### {{< var vm.product >}} Guided Tour
[coming soon]{.smallercaps .pink} + +As a new user of ValidMind, learn how to navigate the platform's key features and understand how they work together to support AI governance and compliance with regulations like the EU AI Act. + +::: {.attn .hidden} + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-30-ns} +{{< fa check >}} **Most course content exists** + +- Create course with 4 modules + +- Create course registration page & add to training portal +::: + +::: {.w-30-ns} +{{< fa x >}} **No videos** + +- Create 1 video +::: + +::: {.w-30-ns .pl4 .bl} +**Notes** + +- Derived from [EU AI Act (V0)](https://docs-demo.vm.validmind.ai/product-demos/eu-ai-act/v0/index.html#/title-slide) +- Switch to training template +- Fix display issues using V1 fix (don't use absolute negative placement for columns) +::: + +:::: +::: + +::: {.attn} +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 1
Getting Started with the Model Inventory + +- Understand how the model inventory serves as your central hub for registered models +- Learn to register new models with appropriate templates +- Configure custom inventory fields and business units +- Calculate risk classifications for AI systems +::: + +::: {.w-50-ns} +#### Module 2
Model Documentation and Testing + +- Navigate the documentation templates and structure +- Explore the library of 250+ out-of-the-box tests +- Learn to use AI-powered documentation features +- Understand how to customize workflows for your needs +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 3
Validation and Collaboration + +- Create and manage validation reports +- Learn to assess compliance and link evidence +- Create and track findings with severity and due dates +- Use collaboration features for developer-validator interaction +::: + +::: {.w-50-ns} +#### Module 4
Monitoring and Governance + +- Set up ongoing monitoring for production models +- Track model performance and identify data drift +- Access and interpret governance analytics +- Generate reports for non-technical stakeholders +::: + +:::: +::: + +### {{< var vm.product >}} Content Features
[coming soon]{.smallercaps .pink} + +As a content author who is new to ValidMind, learn how to use the content features to enhance my documentation, and to enable or disable text generation with GenAI. + +::: {.attn .hidden} + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-30-ns} +{{< fa x >}} **No course** + +- Create course with 2 modules +- Create course registration page & add to training portal +::: + +::: {.w-30-ns} +{{< fa x >}} **No videos** + +- Create 2 videos +::: + +::: {.w-30-ns} + +::: + +:::: +::: + +::: {.attn} +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns} +#### Module 1
Using text generation + +- When to use GenAI to draft content +- How to turn GenAI on and off +::: + +::: {.w-50-ns} +#### Module 2
Enhancing content + +- How to edit content +- Write optional assessment prompts +::: + +:::: +::: + + + diff --git a/site/training/program/program-overview.qmd b/site/training/program/program-overview.qmd new file mode 100644 index 0000000000..762f85adf6 --- /dev/null +++ b/site/training/program/program-overview.qmd @@ -0,0 +1,139 @@ +--- +title: "Learn with confidence" +sidebar: validmind-training +listing: + - id: learning-paths + type: grid + grid-columns: 1 + max-description-length: 250 + sort: false + fields: [title, description] + contents: + - learning-paths.qmd + - id: sample-training-plan + type: grid + grid-columns: 1 + max-description-length: 250 + sort: false + fields: [title, description] + contents: + - sample-training-plan.qmd +--- + +Empower your team with ValidMind's comprehensive training program, designed to accelerate adoption and maximize value. + +::: {.attn} +## {{< fa check >}} Training that delivers + +Our training program is built on three pillars: + +1. **Efficient onboarding** - Get teams productive quickly with role-based training +2. **Real-world application** - Apply skills directly to your organization's needs +3. **Continuous growth** - Progress from fundamentals to advanced techniques + +::: + +## Invest in your success + +ValidMind's training program ensures your organization can: + +- Accelerate time-to-value with structured learning paths +- Build internal expertise across key roles +- Maintain compliance with industry standards +- Scale model risk management capabilities efficiently + +::: {.column-margin} +::: {#learning-paths} +::: +::: + +### Role-based expertise + +Our training develops competency across your entire model risk management team: + +- **Model developers** master efficient documentation and testing +- **Model validators** learn systematic validation approaches +- **Governance administrators** implement robust oversight processes + +## Options for every organization + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .pr2} +### Base program includes + +- 10 hours of instructor-led training +- Self-paced learning platform access +- Community support resources +- Standard training materials +::: + +::: {.w-50-ns .pl2} +### Enterprise enhancements + +- Customized on-site training +- Extended instructor sessions +- Specialized topic workshops +- Dedicated office hours +- Team-specific programs +::: + +:::: + +## Enterprise flexibility + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .pr2} +### Global delivery + +- Regional tracks (North America/UK) +- Time zone aligned scheduling +- Local compliance considerations +::: + +::: {.w-50-ns} +### Flexible scheduling + +- Flexible session booking +- Parallel training tracks +- Custom group timing +::: + +:::: + +## Enterprise support + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .pr2} +### Implementation services + +- System integration +- Platform customization +- Workflow optimization +- Enterprise deployment +::: + +::: {.w-50-ns} +### Ongoing partnership + +- Expert consultation +- Advanced training +- Community resources +- Technical assistance +::: + +:::: + +::: {.column-margin} +::: {#sample-training-plan} +::: +::: + +::: {.feature} +## Get started + +Ready to begin your ValidMind training journey? Contact your customer success representative to create a customized training plan for your organization. + +::: diff --git a/site/training/program/sample-training-plan.qmd b/site/training/program/sample-training-plan.qmd new file mode 100644 index 0000000000..03162919a7 --- /dev/null +++ b/site/training/program/sample-training-plan.qmd @@ -0,0 +1,130 @@ +--- +title: "Sample training plan" +sidebar: validmind-training +description: "Sample 3-week training plan for enterprise deployment with enhanced support options." +--- + +This enterprise training plan demonstrates how ValidMind can be deployed across a large organization using both standard and enhanced training options. The plan includes comprehensive role-based training, custom workshops, and ongoing support to ensure successful adoption. + +## Timeline overview + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-30-ns} +### Week 1
Foundation & core skills + +- Initial platform introduction +- Role-based fundamentals +- Hands-on workshops +::: + +::: {.w-30-ns} +### Week 2
Advanced implementation + +- Specialized workflows +- Custom integrations +- Team-specific scenarios +::: + +::: {.w-30-ns} +### Week 3
Optimization & scale + +- Advanced features +- Best practices +- Knowledge transfer +::: + +:::: + + +## Regional delivery + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .pr2} +### North America (EST-based) +- Live morning sessions: 10:00 AM - 12:00 PM EST +- Interactive workshops: 2:00 PM - 4:00 PM EST +- Office hours: Alternating AM/PM slots +::: + +::: {.w-50-ns .pl2} +### UK/Europe (GMT-based) +- Live morning sessions: 10:00 AM - 12:00 PM GMT +- Interactive workshops: 2:00 PM - 4:00 PM GMT +- Office hours: Aligned with regional working hours +::: + +:::: + +::: {.attn} +## Detailed training structure + +### Foundation phase (Week 1) + +| Module | Format | Duration | Delivery | +|--------|---------|----------|-----------| +| ValidMind 101 | Live + Labs | 2 hours | Regional tracks | +| Developer foundations | Interactive | 4 hours | Role-specific | +| Validator essentials | Interactive | 4 hours | Role-specific | +| Admin core skills | Interactive | 4 hours | Role-specific | + +### Implementation phase (Week 2) + +| Module | Format | Duration | Delivery | +|--------|---------|----------|-----------| +| Custom workflows | Workshop | 4 hours | Team-based | +| Integration practices | Technical | 4 hours | Development teams | +| Advanced validation | Workshop | 4 hours | Validation teams | +| Governance setup | Technical | 4 hours | Admin teams | + +### Optimization phase (Week 3) + +| Module | Format | Duration | Delivery | +|--------|---------|----------|-----------| +| Best practices | Workshop | 4 hours | All teams | +| Advanced features | Technical | 4 hours | Role-specific | +| Enterprise scale | Strategy | 2 hours | Leadership | +| Future roadmap | Planning | 2 hours | Key stakeholders | +::: + +## Enhanced support package + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .pr2} +### Dedicated office hours +- Weekly technical office hours +- Bi-weekly strategic reviews +- Monthly progress assessments +::: + +::: {.w-50-ns .pl2} +### Ongoing support +- Documentation reviews +- Implementation guidance +- Technical troubleshooting +- Best practice updates +::: + +:::: + +## Success metrics & reporting + +- Weekly adoption metrics +- Monthly progress reports +- Quarterly effectiveness reviews +- ROI assessment framework + +::: {.feature} + +## Next steps + +1. Confirm regional participant lists +2. Schedule kick-off sessions +3. Complete pre-training assessments +4. Begin foundation phase rollout + +Contact your ValidMind representative to customize this plan for your organization's specific needs and timeline requirements. + +::: \ No newline at end of file diff --git a/site/training/program/training-faq.qmd b/site/training/program/training-faq.qmd new file mode 100644 index 0000000000..481897b9c4 --- /dev/null +++ b/site/training/program/training-faq.qmd @@ -0,0 +1,320 @@ +--- +title: "Where do I find … ?" +sidebar: validmind-training +--- + +Here's what we have been asked about ValidMind during training sessions. + +## Answers by topic + + + +### Model inventory + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I configure the model inventory? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-inventory/managing-model-inventory.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I add model inventory fields? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-inventory/manage-inventory-custom-fields.html){.button-small target="blank"} +::: + +:::: + +### User management and roles + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I create new users? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/administrator-fundamentals/administrator-fundamentals.html#/section-4){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/guide/manage-users.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I create new user roles? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/administrator-fundamentals/administrator-fundamentals.html#/section-6){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/guide/manage-users.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I add users to roles? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/administrator-fundamentals/administrator-fundamentals.html#/section-4){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/guide/manage-roles.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I map users to specific groups? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/administrator-fundamentals/administrator-fundamentals.html#/section-4){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/guide/manage-groups.html){.button-small target="blank"} +::: + +:::: + +### Workflows for model lifecycle management + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I modify or configure workflows for my use case? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/administrator-fundamentals/administrator-fundamentals.html#/section-12){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-workflows/working-with-model-workflows.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I configure workflows for MRM? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/administrator-fundamentals/administrator-fundamentals.html#/section-12){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-workflows/set-up-model-workflows.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I add users into workflows? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa book-open >}} docs](https://docs.validmind.ai/guide/manage-roles.html){.button-small target="blank"} +::: + +:::: + +### Templates for documentation & validation + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I edit templates? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-documentation/customize-documentation-templates.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I add tests to templates? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-documentation/customize-documentation-templates.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I swap templates? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-documentation/customize-documentation-templates.html){.button-small target="blank"} +::: + +:::: + +### Model documentation + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I register a model with existing template? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/developer-fundamentals/developer-fundamentals.html){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I generate documentation for models? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/developer-fundamentals/developer-fundamentals.html){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/developers/documenting-models.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I submit my model documentation for validation? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/developer-fundamentals/developer-fundamentals.html){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-documentation/submit-for-approval.html){.button-small target="blank"} +::: + +:::: + +### Model testing + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I create custom tests? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/developer-fundamentals/developer-fundamentals.html){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/developers/run-tests-and-test-suites.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I reuse existing tests for comparison tests? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/developer-fundamentals/developer-fundamentals.html){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/developers/run-tests-and-test-suites.html){.button-small target="blank"} +::: + +:::: + +### Model validation + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I work with validation reports? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/validator-fundamentals/validator-fundamentals.html#/prepare-validation-reports){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-validation/working-with-model-findings.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I submit validation reports for approval? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa graduation-cap >}} training](https://docs.validmind.ai/training/validator-fundamentals/validator-fundamentals.html#/submit-for-review-and-approval){.button-small target="blank"} [{{< fa book-open >}} docs](https://docs.validmind.ai/guide/model-documentation/submit-for-approval.html){.button-small target="blank"} +::: + +:::: + +### Monitoring + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I set up monitoring? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa book-open >}} docs](https://docs.validmind.ai/guide/monitoring/enable-monitoring.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I integrate monitoring with documentation process? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa book-open >}} docs](https://docs.validmind.ai/guide/monitoring/ongoing-monitoring.html){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I schedule quarterly/monthly runs? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa book-open >}} docs](https://docs.validmind.ai/guide/monitoring/work-with-metrics-over-time.html){.button-small target="blank"} +::: + +:::: + +### Data management + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- How do I work with reference datasets? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa book-open >}} code sample](https://docs.validmind.ai/notebooks/code_samples/nlp_and_llm/llm_summarization_demo.html#initialize-the-python-environment){.button-small target="blank"} +::: + +:::: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-60-ns .nb3 .pr2} +- Can we connect to external data sources? +::: + +::: {.w-40-ns .nb3 .pl2} +[{{< fa brands python >}} pandas docs](https://pandas.pydata.org/docs/getting_started/intro_tutorials/02_read_write.html){.button-small target="blank"} +::: + +:::: + +{{< include /training/_training-support.qmd >}} + +{{< include /support/_submit-feedback.qmd >}} \ No newline at end of file diff --git a/site/training/training.qmd b/site/training/training.qmd index 683f6fe125..6b9d443e6a 100644 --- a/site/training/training.qmd +++ b/site/training/training.qmd @@ -9,6 +9,17 @@ aliases: - training-overview.html - ../guide/get-started-sandbox.html listing: + - id: training-curriculum + type: grid + grid-columns: 4 + max-description-length: 250 + sort: false + fields: [title, description] + contents: + - program/program-overview.qmd + - program/learning-paths.qmd + - program/sample-training-plan.qmd + - program/training-faq.qmd - id: training-courses type: grid max-description-length: 250 @@ -77,11 +88,50 @@ Our **free** training courses make navigating **AI and governance** straightforw ::: {.column-page} +## Our program + +:::{#training-curriculum} +::: + ## Available courses :::{#training-courses} ::: +## Available videos + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .pa4} +::: {.grid} +::: {.g-col-12} +{{< video https://www.youtube.com/embed/videoseries?si=aP7yLOGO60h4vi8C&list=PL7I7ZrCoVCCP876aLRCjitZqYif96Whlz title='Developer Fundamentals' >}} + +::: {.video-cap .tc} +(10 videos) +::: + +::: +::: + +::: + +::: {.w-50-ns .pa4} +::: {.grid} +::: {.g-col-12} +{{< video https://www.youtube.com/embed/videoseries?si=JfBNClmsZYLS7m9F&list=PL7I7ZrCoVCCObEiuLLAL0Wm9GQ6x5U0q5 title='Validating Models 101' >}} + +::: {.video-cap .tc} +(5 videos) +::: + +::: +::: + +::: + +:::: + ::: {.attn} ## ![](/validmind.png){height=30px} Learn by doing