diff --git a/.github/workflows/validate-docs-site.yaml b/.github/workflows/validate-docs-site.yaml index 6838a4ca34..bba96bf603 100644 --- a/.github/workflows/validate-docs-site.yaml +++ b/.github/workflows/validate-docs-site.yaml @@ -60,7 +60,7 @@ jobs: # Only execute the demo notebook if .env file is created - name: Execute demo Intro for Model Developers notebook - if: ${{ steps.create_env.outcome == 'success' }} + if: ${{ env.ENABLE_DEMO_NOTEBOOK == 'true' && steps.create_env.outcome == 'success' }} uses: ./.github/actions/demo-notebook id: execute-demo-notebook with: diff --git a/site/_quarto.yml b/site/_quarto.yml index 04e111ed8d..84c238b7d7 100644 --- a/site/_quarto.yml +++ b/site/_quarto.yml @@ -170,6 +170,7 @@ website: - text: "---" - text: "Releases" # MAKE-RELEASE-NOTES-EMBED-MARKER + - releases/2025/2025-jan-31/release-notes.qmd # CURRENT-YEAR-END-MARKER - file: releases/2024/2024-releases.qmd contents: diff --git a/site/index.qmd b/site/index.qmd index 4558c52ea3..7a6d0c5f51 100644 --- a/site/index.qmd +++ b/site/index.qmd @@ -46,12 +46,12 @@ listing: fields: [title, description] contents: # MAKE-RELEASE-NOTES-LATEST-MARKER + - /releases/2025/2025-jan-31/release-notes.qmd - /releases/2024/2024-dec-24/release-notes.qmd - /releases/2024/2024-dec-06/release-notes.qmd - /releases/2024/2024-oct-22/release-notes.qmd - /releases/2024/2024-sep-25/release-notes.qmd - /releases/2024/2024-sep-09/release-notes.qmd - - /releases/2024/2024-aug-13/release-notes.qmd # MAKE-RELEASE-NOTES-OLDEST-MARKER - id: validmind-academy type: grid diff --git a/site/python-docs.zip b/site/python-docs.zip index 2c32f416d5..c9baa66229 100644 Binary files a/site/python-docs.zip and b/site/python-docs.zip differ diff --git a/site/releases/2024/2024-dec-24/release-notes.qmd b/site/releases/2024/2024-dec-24/release-notes.qmd index 91a9d75059..2884f518e9 100644 --- a/site/releases/2024/2024-dec-24/release-notes.qmd +++ b/site/releases/2024/2024-dec-24/release-notes.qmd @@ -2,6 +2,17 @@ title: "December 24, 2024" aliases: - ../../2024-dec-24/release-notes.html +listing: + - id: deep-dive + type: grid + grid-columns: 1 + max-description-length: 250 + # image-height: 100% + contents: + - path: https://validmind.com/blog/tech-deep-dive-custom-reports-dashboards/ + title: "Tech Deep Dive: Custom Reports & Dashboards {{< fa chevron-right >}}" + description: "For an introduction to how this feature can benefit your organization, check out our companion blog post." + fields: [title, description] --- This release brings additional customization features to the {{< var validmind.platform >}}, improvements to our documentation site, a brand new Help Center, and more! @@ -89,6 +100,20 @@ Widget types include saved inventory and model findings views[^1], analytics rep ![Custom `High Risk` dashboard showing widgets for Tier 1 models and High Severity findings](custom-dashboard.png){width=80% fig-alt="A screenshot of a custom `High Risk` dashboard showing widgets for Tier 1 models and High Severity findings" .screenshot} +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .pr4} +You can add as many dashboards as you need to suit different use cases, whether it's managing model documentation and testing as a developer, tracking findings in validation reports as a validator, or monitoring analytics as a model owner. +::: + +::: {.w-50-ns .tc} +:::{#deep-dive} +::: + +::: + +:::: + ### Documentation #### User guide updates diff --git a/site/releases/2025/2025-jan-31/create-new-organization.png b/site/releases/2025/2025-jan-31/create-new-organization.png new file mode 100644 index 0000000000..0274dd6fcf Binary files /dev/null and b/site/releases/2025/2025-jan-31/create-new-organization.png differ diff --git a/site/releases/2025/2025-jan-31/dashboard-edit-mode.png b/site/releases/2025/2025-jan-31/dashboard-edit-mode.png new file mode 100644 index 0000000000..8b486d3b29 Binary files /dev/null and b/site/releases/2025/2025-jan-31/dashboard-edit-mode.png differ diff --git a/site/releases/2025/2025-jan-31/dashboard-view-mode.png b/site/releases/2025/2025-jan-31/dashboard-view-mode.png new file mode 100644 index 0000000000..7ca793f0fb Binary files /dev/null and b/site/releases/2025/2025-jan-31/dashboard-view-mode.png differ diff --git a/site/releases/2025/2025-jan-31/math-editor.png b/site/releases/2025/2025-jan-31/math-editor.png new file mode 100644 index 0000000000..22bf2f97e3 Binary files /dev/null and b/site/releases/2025/2025-jan-31/math-editor.png differ diff --git a/site/releases/2025/2025-jan-31/metric-threshold-lines.png b/site/releases/2025/2025-jan-31/metric-threshold-lines.png new file mode 100644 index 0000000000..aa1fa53265 Binary files /dev/null and b/site/releases/2025/2025-jan-31/metric-threshold-lines.png differ diff --git a/site/releases/2025/2025-jan-31/release-notes.qmd b/site/releases/2025/2025-jan-31/release-notes.qmd new file mode 100644 index 0000000000..acdbc1b288 --- /dev/null +++ b/site/releases/2025/2025-jan-31/release-notes.qmd @@ -0,0 +1,589 @@ +--- +title: "January 31, 2025" +listing: + - id: test-desc + type: grid + grid-columns: 1 + max-description-length: 250 + # image-height: 100% + contents: + - path: https://github.com/validmind/validmind-library/blob/main/notebooks/code_sharing/llm/add_context_to_llm_descriptions.ipynb + title: "Add context to LLM-generated test descriptions" + description: "Open notebook on GitHub {{< fa chevron-right >}}" + - id: credit-risk + type: grid + grid-columns: 1 + max-description-length: 250 + # image-height: 100% + contents: + - path: https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/code_samples/credit_risk/application_scorecard_with_ml.ipynb + title: "Document an application scorecard model" + categories: ["Individual Tests"] + description: "Open notebook in JupyterHub {{< fa chevron-right >}}" + - path: https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/code_samples/credit_risk/application_scorecard_full_suite.ipynb + title: "Document an application scorecard model" + categories: ["Full Test Suite"] + description: "Open notebook in JupyterHub {{< fa chevron-right >}}" + - path: https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/code_samples/credit_risk/application_scorecard_executive.ipynb + title: "Document an application scorecard model" + categories: ["Single Function"] + description: "Open notebook in JupyterHub {{< fa chevron-right >}}" + - id: ongoing-monitoring + type: grid + grid-columns: 1 + max-description-length: 250 + # image-height: 100% + contents: + - path: https://jupyterhub.validmind.ai/hub/user-redirect/lab/tree/code_samples/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb) + title: "Ongoing monitoring for application scorecard" + description: "Open notebook in JupyterHub {{< fa chevron-right >}}" + - id: e2e-template + type: grid + grid-columns: 1 + max-description-length: 250 + # image-height: 100% + contents: + - path: https://github.com/validmind/validmind-library/blob/main/notebooks/templates/e2e-notebook.ipynb + title: "End-to-end notebook template generation" + description: "Open notebook on GitHub {{< fa chevron-right >}}" + fields: [title, categories, description] +--- + +This release includes our new unified versioning scheme for our software, support for thresholds in unit metrics and custom context for test descriptions within the {{< var validmind.developer >}}, and many more enhancements. + +::: {.highlights} + +## Release highlights — `25.01` + +::: {.callout} +Our documentation now follows the new **unified versioning scheme** for our software, starting with this `25.01` release. Included in this release are: + +- **{{< var validmind.developer >}} — `v2.7.7`** +- **{{< var validmind.platform >}} — `v1.29.10`** + +#### Why a unified versioning scheme? + +We manage multiple repositories, each with its own version tags. The new versioning scheme replaces the {{< var validmind.developer >}} version in the documentation to clarify that each release includes code from multiple repositories rather than a single source. + +This change simplifies tracking changes for each {{< var vm.product >}} release and streamlines version management for you. Release frequency and the upgrade process remain unchanged. +::: + +### {{< var validmind.developer >}} (v2.7.7) + + + +#### Threshold lines in unit metric plots + +When logging metrics using `log_metric()`, you can now include a `thresholds` dictionary. For example, use `thresholds={"target": 0.8, "minimum": 0.6}` to define multiple reference levels. + +![Threshold lines in unit metric plots](metric-threshold-lines.png){width="949" fig-alt="A screenshot showing threshold lines in unit metric plots" .screenshot} + + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-70-ns .pr4} +- These thresholds automatically appear as horizontal reference lines when you add a *Metric Over Time* block to the documentation. + +- The visualization uses a distinct color palette to differentiate between thresholds. It displays only the most recent threshold configuration and includes threshold information in both the chart legend and data table. +::: + +::: {.w-30-ns .tr} +[log_metric()](/validmind/validmind.html#log_metric){.button .button-green target="_blank"} + +[Add metrics over time](/guide/monitoring/work-with-metrics-over-time.qmd){.button .button-green} + +::: + +:::: + +::: {.column-margin} +**Usage example:** + +```python +log_metric( + key="AUC Score", + value=auc, + recorded_at=datetime(2024, 1, 1), + thresholds={ + "high_risk": 0.6, + "medium_risk": 0.7, + "low_risk": 0.8, + } +) +``` + +::: + +This enhancement provides immediate visual context for metric values. It helps track metric performance against multiple defined thresholds over time. + + + + + +#### Add context to enhance LLM-based text generation for model test results + +You can now include contextual information to enhance LLM-based generation of test results descriptions and interpretations. This enhancement improves test result descriptions by incorporating additional context that can be specified through environment variables. + +::: {.column-margin} +:::{#test-desc} +::: + +::: + +A new notebook demonstrates adding context to LLM-based descriptions with examples of: + +- Setting up the environment +- Initializing datasets and models +- Running tests with and without context + + + + + +#### Document credit risk scorecard models using XGBoost + +We've introduced enhancements to the {{< var validmind.developer >}} that focus on documenting credit risk scorecard models: + +- **New notebooks**: Learn how to document application scorecard models using the {{< var vm.developer >}}. These notebooks provide a step-by-step guide for loading a demo dataset, preprocessing data, training models, and documenting the model. + + You can choose from three different approaches: **running individual tests**, **running a full test suite**, or **using a single function** to document a model. + +::: {.column-margin} +:::{#credit-risk} +::: + +::: + +- **New tests**: + + - [`MutualInformation`](/tests/data_validation/MutualInformation.md): Evaluates feature relevance by calculating mutual information scores between features and the target variable. + - [`ScoreBandDefaultRates`](/tests/data_validation/ScoreBandDefaultRates.md): Analyzes default rates and population distribution across credit score bands. + - [`CalibrationCurve`](/tests/model_validation/sklearn/CalibrationCurve.md): Assesses calibration by comparing predicted probabilities against observed frequencies. + - [`ClassifierThresholdOptimization`](/tests/model_validation/sklearn/ClassifierThresholdOptimization.md): Visualizes threshold optimization methods for binary classification models. + - [`ModelParameters`](/tests/model_validation/sklearn/ModelParameters.md): Extracts and displays model parameters for transparency and reproducibility. + - [`ScoreProbabilityAlignment`](/tests/model_validation/sklearn/ScoreProbabilityAlignment.md): Evaluates alignment between credit scores and predicted probabilities. + +Modifications have also been made to existing tests to improve functionality and accuracy. The [`TooManyZeroValues`](/tests/data_validation/TooManyZeroValues.md) test now includes a row count and applies a percentage threshold for zero values. + +The [`split`](/validmind/validmind/datasets/regression/lending_club.html#preprocess){target="_blank"} function in `lending_club.py` has been enhanced to support an optional validation set, allowing for more flexible dataset splitting. + +A new utility function, [`get_demo_test_config`](/validmind/validmind/datasets/credit_risk/lending_club.html#get_demo_test_config){target="_blank"}, has been added to generate a default test configuration for demo purposes. + + +#### Ongoing monitoring notebook for application scorecard model + +Several enhancements to the {{< var validmind.developer >}} focus on ongoing monitoring capabilities: + +- **New notebook**: Learn how to use ongoing monitoring with credit risk datasets in this step-by-step guide for the {{< var validmind.developer >}}. + + - Use our new metrics for data and model drift, and populate the ongoing monitoring documentation for a scorecard model.[^1] + +::: {.column-margin} +:::{#ongoing-monitoring} +::: + +::: + +- **Custom tests**: Define and run your own tests using the {{< var vm.developer >}}: + + - [`ScoreBandDiscriminationMetrics.py`](https://github.com/validmind/validmind-library/blob/main/notebooks/code_samples/credit_risk/custom_tests/ScoreBandDiscriminationMetrics.py): Evaluates discrimination metrics across different score bands. + +- **New tests**: + + - [`CalibrationCurveDrift`](/tests/ongoing_monitoring/CalibrationCurveDrift.md): Evaluates changes in probability calibration. + - [`ClassDiscriminationDrift`](/tests/ongoing_monitoring/ClassDiscriminationDrift.md): Compares classification discrimination metrics. + - [`ClassImbalanceDrift`](/tests/ongoing_monitoring/ClassImbalanceDrift.md): Evaluates drift in class distribution. + - [`ClassificationAccuracyDrift`](/tests/ongoing_monitoring/ClassificationAccuracyDrift.md): Compares classification accuracy metrics. + - [`ConfusionMatrixDrift`](/tests/ongoing_monitoring/ConfusionMatrixDrift.md): Compares confusion matrix metrics. + - [`CumulativePredictionProbabilitiesDrift`](/tests/ongoing_monitoring/CumulativePredictionProbabilitiesDrift.md): Compares cumulative prediction probability distributions. + - [`FeatureDrift`](/tests/ongoing_monitoring/FeatureDrift.md): Evaluates changes in feature distribution. + - [`PredictionAcrossEachFeature`](/tests/ongoing_monitoring/PredictionAcrossEachFeature.md): Assesses prediction distributions across features. + - [`PredictionCorrelation`](/tests/ongoing_monitoring/PredictionCorrelation.md): Assesses correlation changes between predictions and features. + - [`PredictionProbabilitiesHistogramDrift`](/tests/ongoing_monitoring/PredictionProbabilitiesHistogramDrift.md): Compares prediction probability distributions. + - [`PredictionQuantilesAcrossFeatures`](/tests/ongoing_monitoring/PredictionQuantilesAcrossFeatures.md): Assesses prediction distributions across features using quantiles. + - [`ROCCurveDrift`](/tests/ongoing_monitoring/ROCCurveDrift.md): Compares ROC curves. + - [`ScoreBandsDrift`](/tests/ongoing_monitoring/ScoreBandsDrift.md): Analyzes drift in score bands. + - [`ScorecardHistogramDrift`](/tests/ongoing_monitoring/ScorecardHistogramDrift.md): Compares score distributions. + - [`TargetPredictionDistributionPlot`](/tests/ongoing_monitoring/TargetPredictionDistributionPlot.md): Assesses differences in prediction distributions. + +We also improved dataset loading, preprocessing, and feature engineering functions with verbosity control for cleaner output. + + +#### Jupyter Notebook templates + +Want to create your own code samples using ValidMind's? We've now made it easier for contributors to submit custom code samples. + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .pr4} +Our **end-to-end notebook template generation notebook** will generate a new file with all the bits and pieces of a standard ValidMind notebook to get you started. + +The same functionality is also accessible from our [Makefile](https://github.com/validmind/validmind-library/blob/main/Makefile#L83C1-L85C44): + +```bash +make notebook +``` + +:::{#e2e-template} +::: + + +::: + +::: {.w-50-ns .pl4 .nt4} +##### Mini-templates + +The template generation notebook draws from a number of mini-templates, should you need to revise them or grab the information from them manually: + +- `about-validmind.ipynb`: Conceptual overview of ValidMind & prerequisites. +- `install-initialize-validmind.ipynb`: ValidMind Library installation & initialization instructions. +- `next-steps.ipynb`: Directions to review the generated documentation within the ValidMind Platform & additional learning resources. +- `upgrade-validmind.ipynb`: Instructions for comparing & upgrading versions of the ValidMind Library. +::: + +:::: + +### {{< var validmind.platform >}} (v1.29.10) + + +#### Edit your dashboards + +We’ve streamlined dashboard configuration with dedicated view and edit modes. Click **Edit Mode** to make changes, then click **Done Editing** to save and return to view mode: + +::: {.column-margin} +[Customize your dashboard](/guide/configuration/customize-your-dashboard.qmd){.button .button-green} + +::: + +![Edit mode for your dashboard](dashboard-edit-mode.png){width="809" fig-alt="A screenshot showing edit mode" .screenshot} + + + +To prevent any confusion when multiple people are working on the same dashboard, we've added some helpful safeguards: + +- If someone else makes changes while you're editing, you'll get a friendly notification to reload the page +- The system automatically detects if you're looking at an older version of the dashboard and prompts you to get the latest updates + + + + + + +#### Optional prompt for risk assessments + +Risk assessment generation has been enhanced to allow you to provide an optional prompt before starting text generation. This feature lets you guide the output, ensuring that the generated text aligns more closely with your specific requirements. + +::: {.column-margin} +[Assess compliance](/guide/model-validation/assess-compliance.qmd){.button} + +::: + +![Optional prompt for risk assessments](risk-assessment-prompt.gif){.screenshot} + +::: + +## Enhancements + +### {{< var validmind.developer >}} (v2.7.7) + + +#### Static descriptions in test results + +The `TestResult` class now exposes pre-populated test descriptions through the `doc` property, separating them from dynamically generated GenAI descriptions: + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-80-ns} +- `result.doc` — contains the original docstring of the test. +- `result.description` — contains the dynamically generated description. +::: + +::: {.w-20-ns .tr} +[`TestResult`](/validmind/validmind/vm_models.html#TestResult){.button target="_blank"} +::: + +:::: + +This enhancement makes it easier to distinguish between ValidMind's standard test documentation and the dynamic, context-aware descriptions generated for your specific test results. + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-70-ns} + You can browse the full catalog of official test descriptions in our test documentation: +::: + +::: {.w-30-ns .tr} +[Test descriptions](/developer/model-testing/test-descriptions.qmd){.button target="_blank"} +::: + +:::: + + +#### Raw data storage for tests + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-80-ns} +We added raw data storage across all {{< var validmind.developer >}} tests. Every test now returns a `RawData` object, allowing post-processing functions to recreate any test output. This feature enhances flexibility and customizability. +::: + +::: {.w-20-ns .tr} +[`RawData`](/validmind/validmind.html#RawData){.button target="_blank"} +::: + +:::: + + +#### New `print_env` function + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-80-ns} +We've added a new diagnostic `print_env()` utility function that displays comprehensive information about your running environment. This function is particularly useful when: +::: + +::: {.w-20-ns .tr} +[print_env()](/validmind/validmind.html#print_env){.button target="_blank"} +::: + +:::: +- Troubleshooting issues in your code +- Seeking support from the ValidMind team +- Verifying your environment configuration + +::: {.column-margin} + +**Usage example:** + +```python +import validmind + +validmind.print_env() +``` +::: + +This function outputs key details, such as Python version, installed package versions, and relevant environment variables, making it easier to diagnose issues and share your setup with others. + + + +### {{< var validmind.platform >}} (v1.29.10) + + +#### Simplified workflow nodes + +Workflows are now easier to read when zoomed out, helped by a larger modal window and simplified nodes: + +::: {.column-margin} +[Working with model workflows](/guide/model-workflows/working-with-model-workflows.qmd){.button} + +::: + +![Workflow visualization showing simplified nodes](workflow-simplified-nodes.png){width=90% fig-alt="A screenshot showing the simplified workflow visualization with nodes" .screenshot} + +Zooming in reveals more details: + +![Workflow visualization in zoomed-out view](workflow-zoomed-view.png){width=90% fig-alt="A screenshot showing the simplified workflow visualization" .screenshot} + +Hovering over a node highlights all `in` and `out` connections, making relationships clearer: + +![Workflow connection highlighting on hover](workflow-connection-hover.png){width=90% fig-alt="A screenshot showing the workflow connection highlighting" .screenshot} + + + + + + + + + + + + + + + + + + + + + + + + + + + + +#### New editor for mathematical formulas + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-70-ns .pr4} +We replaced the plugin for the editor of mathematical equations and formulas. The new plugin provides an improved interface for adding and editing LaTeX expressions in your documentation. + +The new editor also includes a real-time preview and common mathematical symbols for easier equation creation. + +::: {.tc} +[Add mathematical formulas](/guide/model-documentation/work-with-content-blocks.html#add-mathematical-formulas.qmd){.button} +::: + +::: + +::: {.w-30-ns} +![New editor for mathematical equations and formulas](math-editor.png){fig-alt="A screenshot showing the new editor for mathematical equations and formulas" .screenshot} + +::: + +:::: + + + + + + + + + + + + + + +{{< include /releases/_how-to-upgrade.qmd >}} + + + + +[^1]: [Document credit risk scorecard models using XGBoost](#credit-risk) diff --git a/site/releases/2025/2025-jan-31/risk-assessment-prompt.gif b/site/releases/2025/2025-jan-31/risk-assessment-prompt.gif new file mode 100644 index 0000000000..f00ab448ff Binary files /dev/null and b/site/releases/2025/2025-jan-31/risk-assessment-prompt.gif differ diff --git a/site/releases/2025/2025-jan-31/workflow-connection-hover.png b/site/releases/2025/2025-jan-31/workflow-connection-hover.png new file mode 100644 index 0000000000..b697fe3b25 Binary files /dev/null and b/site/releases/2025/2025-jan-31/workflow-connection-hover.png differ diff --git a/site/releases/2025/2025-jan-31/workflow-simplified-nodes.png b/site/releases/2025/2025-jan-31/workflow-simplified-nodes.png new file mode 100644 index 0000000000..a1b911157f Binary files /dev/null and b/site/releases/2025/2025-jan-31/workflow-simplified-nodes.png differ diff --git a/site/releases/2025/2025-jan-31/workflow-zoomed-view.png b/site/releases/2025/2025-jan-31/workflow-zoomed-view.png new file mode 100644 index 0000000000..27dcae58a3 Binary files /dev/null and b/site/releases/2025/2025-jan-31/workflow-zoomed-view.png differ