From 1f2fc7e1bef9e2d6449e58447d20c69020b2538a Mon Sep 17 00:00:00 2001 From: John Walz Date: Tue, 4 Feb 2025 13:37:22 -0500 Subject: [PATCH 1/8] feat: add new notebook and quick little update to how inspect preview is shown for raw data --- notebooks/code_sharing/rawdata_usage.ipynb | 437 +++++++++++++++++++++ validmind/utils.py | 23 +- 2 files changed, 455 insertions(+), 5 deletions(-) create mode 100644 notebooks/code_sharing/rawdata_usage.ipynb diff --git a/notebooks/code_sharing/rawdata_usage.ipynb b/notebooks/code_sharing/rawdata_usage.ipynb new file mode 100644 index 000000000..1d66c9eef --- /dev/null +++ b/notebooks/code_sharing/rawdata_usage.ipynb @@ -0,0 +1,437 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Understanding and Utilizing RawData in ValidMind Tests" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In ValidMind, test functions can return a special object called **RawData**. This object holds, as the name suggests, intermediate or unprocessed data that is produced somewhere in the test logic but is not returned as part of the test's visible output (tables or figures). This data can be useful when running post-processing functions with tests to recompute tabular outputs, redraw figure or even create new outputs entirely. In this notebook, we demonstrate how to access, inspect, and utilize RawData from ValidMind tests with a couple of examples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import xgboost as xgb\n", + "import validmind as vm\n", + "from validmind.datasets.classification import customer_churn\n", + "\n", + "raw_df = customer_churn.load_data()\n", + "\n", + "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)\n", + "\n", + "x_train = train_df.drop(customer_churn.target_column, axis=1)\n", + "y_train = train_df[customer_churn.target_column]\n", + "x_val = validation_df.drop(customer_churn.target_column, axis=1)\n", + "y_val = validation_df[customer_churn.target_column]\n", + "\n", + "model = xgb.XGBClassifier(early_stopping_rounds=10)\n", + "model.set_params(\n", + " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", + ")\n", + "model.fit(\n", + " x_train,\n", + " y_train,\n", + " eval_set=[(x_val, y_val)],\n", + " verbose=False,\n", + ")\n", + "\n", + "vm_raw_dataset = vm.init_dataset(\n", + " dataset=raw_df,\n", + " input_id=\"raw_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " class_labels=customer_churn.class_labels,\n", + " __log=False,\n", + ")\n", + "\n", + "vm_train_ds = vm.init_dataset(\n", + " dataset=train_df,\n", + " input_id=\"train_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " __log=False,\n", + ")\n", + "\n", + "vm_test_ds = vm.init_dataset(\n", + " dataset=test_df,\n", + " input_id=\"test_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " __log=False,\n", + ")\n", + "\n", + "vm_model = vm.init_model(\n", + " model,\n", + " input_id=\"model\",\n", + " __log=False,\n", + ")\n", + "\n", + "vm_train_ds.assign_predictions(\n", + " model=vm_model,\n", + ")\n", + "\n", + "vm_test_ds.assign_predictions(\n", + " model=vm_model,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example 1: Using RawData from the ROC Curve Test\n", + "\n", + "In this example, we run the ROC Curve test, inspect its RawData output, and then create a custom ROC curve using the raw data values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.tests import run_test\n", + "\n", + "# Run the ROC Curve test normally\n", + "result_roc = run_test(\n", + " \"validmind.model_validation.sklearn.ROCCurve\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " generate_description=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's assume we want to create a custom version of the above figure. First, let's inspect the raw data that this test produces so we can see what we have to work with.\n", + "\n", + "`RawData` objects have a `inspect()` method that will pretty print the attributes of the object to be able to quickly see the data and its types." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Inspect the RawData output from the ROC test\n", + "print(\"RawData from ROC Curve Test:\")\n", + "result_roc.raw_data.inspect()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, the ROC Curve returns a `RawData` object with the following attributes:\n", + "- `fpr`: A list of false positive rates\n", + "- `tpr`: A list of true positive rates\n", + "- `auc`: The area under the curve\n", + "\n", + "This should be enough to create our own custom ROC curve via a post-processing function without having to create a whole new test from scratch and without having to recompute any of the data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "from validmind.vm_models.result import TestResult\n", + "\n", + "\n", + "def custom_roc_curve(result: TestResult):\n", + " # Extract raw data from the test result\n", + " fpr = result.raw_data.fpr\n", + " tpr = result.raw_data.tpr\n", + " auc = result.raw_data.auc\n", + "\n", + " # Create a custom ROC curve plot\n", + " fig = plt.figure()\n", + " plt.plot(fpr, tpr, label=f\"Custom ROC (AUC = {auc:.2f})\", color=\"blue\")\n", + " plt.plot([0, 1], [0, 1], linestyle=\"--\", color=\"gray\", label=\"Random Guess\")\n", + " plt.xlabel(\"False Positive Rate\")\n", + " plt.ylabel(\"True Positive Rate\")\n", + " plt.title(\"Custom ROC Curve from RawData\")\n", + " plt.legend()\n", + "\n", + " # close the plot to avoid it automatically being shown in the notebook\n", + " plt.close()\n", + "\n", + " # remove existing figure\n", + " result.remove_figure(0)\n", + "\n", + " # add new figure\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "# test it on the existing result\n", + "modified_result = custom_roc_curve(result_roc)\n", + "\n", + "# show the modified result\n", + "modified_result.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have created a post-processing function and verified that it works on our existing test result, we can use it directly in `run_test()` from now on:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = run_test(\n", + " \"validmind.model_validation.sklearn.ROCCurve\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " post_process_fn=custom_roc_curve,\n", + " generate_description=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## More Examples\n", + "\n", + "### Pearson Correlation Matrix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Try commenting out the `post_process_fn` argument in the following cell and see what happens between different runs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import plotly.graph_objects as go\n", + "\n", + "\n", + "def custom_heatmap(result: TestResult):\n", + " corr_matrix = result.raw_data.correlation_matrix\n", + "\n", + " heatmap = go.Heatmap(\n", + " z=corr_matrix.values,\n", + " x=list(corr_matrix.columns),\n", + " y=list(corr_matrix.index),\n", + " colorscale=\"Viridis\",\n", + " )\n", + " fig = go.Figure(data=[heatmap])\n", + " fig.update_layout(title=\"Custom Heatmap from RawData\")\n", + "\n", + " plt.close()\n", + "\n", + " result.remove_figure(0)\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "\n", + "result_corr = run_test(\n", + " \"validmind.data_validation.PearsonCorrelationMatrix\",\n", + " inputs={\"dataset\": vm_test_ds},\n", + " generate_description=False,\n", + " post_process_fn=custom_heatmap,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Precision-Recall Curve" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's try the same thing with the Precision-Recall Curve test." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def custom_pr_curve(result: TestResult):\n", + " precision = result.raw_data.precision\n", + " recall = result.raw_data.recall\n", + "\n", + " fig = plt.figure()\n", + " plt.plot(recall, precision, label=\"Precision-Recall Curve\")\n", + " plt.xlabel(\"Recall\")\n", + " plt.ylabel(\"Precision\")\n", + " plt.title(\"Custom Precision-Recall Curve from RawData\")\n", + " plt.legend()\n", + "\n", + " plt.close()\n", + " result.remove_figure(0)\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "result_pr = run_test(\n", + " \"validmind.model_validation.sklearn.PrecisionRecallCurve\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " generate_description=False,\n", + " post_process_fn=custom_pr_curve,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using RawData in Custom Tests\n", + "\n", + "These examples demonstrate some very simple ways to use the `RawData` feature of ValidMind tests. The majority of ValidMind-developed tests return some form of raw data that can be used to customize the output of the test. But you can also create your own tests that return `RawData` objects and use them in the same way.\n", + "\n", + "Let's take a look at how this can be done in custom tests." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from validmind import test, RawData\n", + "from validmind.vm_models import VMDataset, VMModel\n", + "\n", + "\n", + "@test(\"custom.MyCustomTest\")\n", + "def MyCustomTest(dataset: VMDataset, model: VMModel) -> tuple[go.Figure, RawData]:\n", + " \"\"\"Custom test that produces a figure and a RawData object\"\"\"\n", + " # pretend we are using the dataset and model to compute some data\n", + " # ...\n", + "\n", + " # create some fake data that will be used to generate a figure\n", + " data = pd.DataFrame({\"x\": [10, 20, 30, 40, 50], \"y\": [10, 20, 30, 40, 50]})\n", + "\n", + " # create the figure (scatter plot)\n", + " fig = go.Figure(data=go.Scatter(x=data[\"x\"], y=data[\"y\"]))\n", + "\n", + " # now let's create a RawData object that holds the \"computed\" data\n", + " raw_data = RawData(scatter_data_df=data)\n", + "\n", + " # finally, return both the figure and the raw data\n", + " return fig, raw_data\n", + "\n", + "\n", + "my_result = run_test(\n", + " \"custom.MyCustomTest\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " generate_description=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that the test result shows the figure. But since we returned a `RawData` object, we can also inspect the contents and see how we could use it to customize or regenerate the figure in the post-processing function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "my_result.raw_data.inspect()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that we get a nicely-formatted preview of the dataframe we stored in the raw data object. Let's go ahead and use it to re-plot our data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def custom_plot(result: TestResult):\n", + " data = result.raw_data.scatter_data_df\n", + "\n", + " # use something other than a scatter plot\n", + " fig = go.Figure(data=go.Bar(x=data[\"x\"], y=data[\"y\"]))\n", + " fig.update_layout(title=\"Custom Bar Chart from RawData\")\n", + " fig.update_xaxes(title=\"X Axis\")\n", + " fig.update_yaxes(title=\"Y Axis\")\n", + "\n", + " result.remove_figure(0)\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "result = run_test(\n", + " \"custom.MyCustomTest\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " post_process_fn=custom_plot,\n", + " generate_description=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "This notebook has demonstrated how to use the `RawData` feature of ValidMind tests to customize the output of tests. It has also shown how to create custom tests that return `RawData` objects and use them in the same way.\n", + "\n", + "This feature is a powerful tool for creating custom tests and post-processing functions that can be used to generate a wide variety of outputs from ValidMind tests." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/validmind/utils.py b/validmind/utils.py index 7393b8883..77c448983 100644 --- a/validmind/utils.py +++ b/validmind/utils.py @@ -165,11 +165,12 @@ def is_generic_object(self, obj): return isinstance(obj, object) def handle_generic_object(self, obj): - return ( - obj.__str__() - if type(obj).__dict__.get("__str__") - else str(obj).split(".")[1].split(" ")[0] - ) + try: + if hasattr(obj, "__str__"): + return obj.__str__() + return obj.__class__.__name__ + except Exception: + return str(type(obj).__name__) def encode(self, obj): obj = nan_to_none(obj) @@ -190,6 +191,18 @@ def __init__(self, *args, **kwargs): else obj.tolist() ) + def default(self, obj): + if self.is_dataframe(obj): + return { + "type": str(type(obj)), + "preview": obj.head(5).to_dict(orient="list"), + "shape": f"{obj.shape[0]} rows x {obj.shape[1]} columns", + } + return super().default(obj) + + def is_dataframe(self, obj): + return isinstance(obj, pd.DataFrame) + def get_full_typename(o: Any) -> Any: """We determine types based on type names so we don't have to import From 309faf52692772f4abf130b09382eefd3126759d Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Tue, 4 Feb 2025 11:58:24 -0800 Subject: [PATCH 2/8] Working branch for notebook edits --- .../understand_utilize_rawdata.ipynb | 309 ++++++++++++++++++ 1 file changed, 309 insertions(+) create mode 100644 notebooks/code_sharing/understand_utilize_rawdata.ipynb diff --git a/notebooks/code_sharing/understand_utilize_rawdata.ipynb b/notebooks/code_sharing/understand_utilize_rawdata.ipynb new file mode 100644 index 000000000..51e29fe4c --- /dev/null +++ b/notebooks/code_sharing/understand_utilize_rawdata.ipynb @@ -0,0 +1,309 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4f9a3237", + "metadata": {}, + "source": [ + "# Understand and utilize `RawData` in ValidMind tests\n", + "\n", + "Test functions in ValidMind can return a special object called *`RawData`*, which holds intermediate or unprocessed data produced somewhere in the test logic but not returned as part of the test's visible output, such as in tables or figures.\n", + "\n", + "This data is useful when running post-processing functions with tests to recompute tabular outputs, redraw figures, or even create new outputs entirely. In this notebook, learn how to access, inspect, and utilize `RawData` from ValidMind tests with a couple of examples." + ] + }, + { + "cell_type": "markdown", + "id": "7d0ceab4", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "markdown", + "id": "install-library-eeed10fa-6015-4d97-bcaa-32098df7f764", + "metadata": {}, + "source": [ + "### Install and intialize the ValidMind Library\n", + "\n", + "First, let's make sure that the ValidMind Library is installed and ready to go:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "install-python-b56f71e6-6faf-4125-bb78-203a240f65b3", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q validmind\n", + "import validmind as vm" + ] + }, + { + "cell_type": "markdown", + "id": "fbaab316", + "metadata": {}, + "source": [ + "### Initialize the Python environment\n", + "\n", + "Next, we'll import the necessary libraries and set up your Python environment for data analysis:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "578e5075", + "metadata": {}, + "outputs": [], + "source": [ + "import xgboost as xgb" + ] + }, + { + "cell_type": "markdown", + "id": "67a2992e", + "metadata": {}, + "source": [ + "### Load the sample dataset\n", + "\n", + "Then, we'll import a sample ValidMind dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc19d4f8", + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.datasets.classification import customer_churn\n", + "raw_df = customer_churn.load_data()" + ] + }, + { + "cell_type": "markdown", + "id": "b9f27fcb", + "metadata": {}, + "source": [ + "#### Preprocess the raw dataset\n", + "\n", + "We'll also perform a number of operations to get ready for the subsequent steps:\n", + "\n", + "\n", + "- **Preprocess the data:** Splits the DataFrame (`df`) into multiple datasets (`train_df`, `validation_df`, and `test_df`) using `demo_dataset.preprocess` to simplify preprocessing.\n", + "- **Separate features and targets:** Drops the target column to create feature sets (`x_train`, `x_val`) and target sets (`y_train`, `y_val`).\n", + "- **Initialize XGBoost classifier:** Creates an `XGBClassifier` object with early stopping rounds set to 10.\n", + "- **Set evaluation metrics:** Specifies metrics for model evaluation as `error`, `logloss`, and `auc`.\n", + "- **Fit the model:** Trains the model on `x_train` and `y_train` using the validation set `(x_val, y_val)`. Verbose output is disabled." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8913f5c8", + "metadata": {}, + "outputs": [], + "source": [ + "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)\n", + "\n", + "x_train = train_df.drop(customer_churn.target_column, axis=1)\n", + "y_train = train_df[customer_churn.target_column]\n", + "x_val = validation_df.drop(customer_churn.target_column, axis=1)\n", + "y_val = validation_df[customer_churn.target_column]\n", + "\n", + "model = xgb.XGBClassifier(early_stopping_rounds=10)\n", + "model.set_params(\n", + " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", + ")\n", + "model.fit(\n", + " x_train,\n", + " y_train,\n", + " eval_set=[(x_val, y_val)],\n", + " verbose=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "7f904540", + "metadata": {}, + "source": [ + "### Initialize the ValidMind objects" + ] + }, + { + "cell_type": "markdown", + "id": "c0e441f4", + "metadata": {}, + "source": [ + "#### Initialize the datasets\n", + "\n", + "Before you can run tests, you'll need to initialize a ValidMind dataset object using the [`init_dataset`](https://docs.validmind.ai/validmind/validmind.html#init_dataset) function from the ValidMind (`vm`) module.\n", + "\n", + "We'll include the following arguments:\n", + "\n", + "- **`dataset`** — the raw dataset that you want to provide as input to tests\n", + "- **`input_id`** - a unique identifier that allows tracking what inputs are used when running each individual test\n", + "- **`target_column`** — a required argument if tests require access to true values. This is the name of the target column in the dataset\n", + "- **`class_labels`** — an optional value to map predicted classes to class labels\n", + "\n", + "With all datasets ready, you can now initialize the raw, training, and test datasets (`raw_df`, `train_df` and `test_df`) created earlier into their own dataset objects using [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c072304", + "metadata": {}, + "outputs": [], + "source": [ + "vm_raw_dataset = vm.init_dataset(\n", + " dataset=raw_df,\n", + " input_id=\"raw_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " class_labels=customer_churn.class_labels,\n", + " __log=False,\n", + ")\n", + "\n", + "vm_train_ds = vm.init_dataset(\n", + " dataset=train_df,\n", + " input_id=\"train_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " __log=False,\n", + ")\n", + "\n", + "vm_test_ds = vm.init_dataset(\n", + " dataset=test_df,\n", + " input_id=\"test_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " __log=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "47339b81", + "metadata": {}, + "source": [ + "#### Initialize a model object\n", + "\n", + "Additionally, you'll need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data. \n", + "\n", + "Simply intialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82d74165", + "metadata": {}, + "outputs": [], + "source": [ + "vm_model = vm.init_model(\n", + " model,\n", + " input_id=\"model\",\n", + " __log=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8c5a2b7c", + "metadata": {}, + "source": [ + "#### Assign predictions to the datasets\n", + "\n", + "We can now use the `assign_predictions()` method from the Dataset object to link existing predictions to any model.\n", + "\n", + "If no prediction values are passed, the method will compute predictions automatically:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cee7ab79", + "metadata": {}, + "outputs": [], + "source": [ + "vm_train_ds.assign_predictions(\n", + " model=vm_model,\n", + ")\n", + "\n", + "vm_test_ds.assign_predictions(\n", + " model=vm_model,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e5b08241", + "metadata": {}, + "source": [ + "## Examples\n", + "\n", + "Once you're set up, you can then run the following examples:" + ] + }, + { + "cell_type": "markdown", + "id": "18808f71", + "metadata": {}, + "source": [ + "### Using `RawData` from the ROC Curve Test" + ] + }, + { + "cell_type": "markdown", + "id": "ed1b2ff9", + "metadata": {}, + "source": [ + "### Pearson Correlation Matrix" + ] + }, + { + "cell_type": "markdown", + "id": "d4006753", + "metadata": {}, + "source": [ + "### Precision-Recall Curve" + ] + }, + { + "cell_type": "markdown", + "id": "eaca281f", + "metadata": {}, + "source": [ + "### Using `RawData` in custom tests" + ] + }, + { + "cell_type": "markdown", + "id": "c3e825ea", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "ValidMind's `RawData` feature allows you to customize the output of tests, including custom tests. \n", + "\n", + "This notebook has demonstrated how to use the `RawData` feature of ValidMind tests to customize the output of tests. It has also shown how to create custom tests that return `RawData` objects and use them in the same way.\n", + "\n", + "This feature is a powerful tool for creating custom tests and post-processing functions that can be used to generate a wide variety of outputs from ValidMind tests." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 5a27f323e0e17a6208a7ea8ead3b261a68c16cc9 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Tue, 4 Feb 2025 14:17:25 -0800 Subject: [PATCH 3/8] Redid Setup section --- .../understand_utilize_rawdata.ipynb | 152 ++++-------------- 1 file changed, 32 insertions(+), 120 deletions(-) diff --git a/notebooks/code_sharing/understand_utilize_rawdata.ipynb b/notebooks/code_sharing/understand_utilize_rawdata.ipynb index 51e29fe4c..2c6a9c047 100644 --- a/notebooks/code_sharing/understand_utilize_rawdata.ipynb +++ b/notebooks/code_sharing/understand_utilize_rawdata.ipynb @@ -9,7 +9,10 @@ "\n", "Test functions in ValidMind can return a special object called *`RawData`*, which holds intermediate or unprocessed data produced somewhere in the test logic but not returned as part of the test's visible output, such as in tables or figures.\n", "\n", - "This data is useful when running post-processing functions with tests to recompute tabular outputs, redraw figures, or even create new outputs entirely. In this notebook, learn how to access, inspect, and utilize `RawData` from ValidMind tests with a couple of examples." + "- The `RawData` feature allows you to customize the output of tests, making it a powerful tool for creating custom tests and post-processing functions.\n", + "- `RawData` is useful when running post-processing functions with tests to recompute tabular outputs, redraw figures, or even create new outputs entirely.\n", + "\n", + "In this notebook, you'll learn how to access, inspect, and utilize `RawData` from ValidMind tests." ] }, { @@ -17,7 +20,11 @@ "id": "7d0ceab4", "metadata": {}, "source": [ - "## Setup" + "## Setup\n", + "\n", + "Before we can run our examples, we'll need to set the stage to enable running tests with the ValidMind Library. Since the focus of this notebook is on the `RawData` object, this section will merely summarize the steps instead of going into greater detail. \n", + "\n", + "To learn more about running tests with ValidMind: **[Run tests and test suites](https://docs.validmind.ai/developer/model-testing/testing-overview.html)**" ] }, { @@ -25,9 +32,9 @@ "id": "install-library-eeed10fa-6015-4d97-bcaa-32098df7f764", "metadata": {}, "source": [ - "### Install and intialize the ValidMind Library\n", + "### Installation and intialization\n", "\n", - "First, let's make sure that the ValidMind Library is installed and ready to go:" + "First, let's make sure that the ValidMind Library is installed and ready to go, and our Python environment set up for data analysis:" ] }, { @@ -37,28 +44,14 @@ "metadata": {}, "outputs": [], "source": [ + "# Install the ValidMind Library\n", "%pip install -q validmind\n", - "import validmind as vm" - ] - }, - { - "cell_type": "markdown", - "id": "fbaab316", - "metadata": {}, - "source": [ - "### Initialize the Python environment\n", "\n", - "Next, we'll import the necessary libraries and set up your Python environment for data analysis:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "578e5075", - "metadata": {}, - "outputs": [], - "source": [ - "import xgboost as xgb" + "# Initialize the ValidMind Library\n", + "import validmind as vm\n", + "\n", + "# Import the `xgboost` library with an allias\n", + "import xgboost as xgb\n" ] }, { @@ -68,7 +61,7 @@ "source": [ "### Load the sample dataset\n", "\n", - "Then, we'll import a sample ValidMind dataset:" + "Then, we'll import a sample ValidMind dataset and preprocess it:" ] }, { @@ -78,45 +71,26 @@ "metadata": {}, "outputs": [], "source": [ + "# Import the `customer_churn` sample dataset\n", "from validmind.datasets.classification import customer_churn\n", - "raw_df = customer_churn.load_data()" - ] - }, - { - "cell_type": "markdown", - "id": "b9f27fcb", - "metadata": {}, - "source": [ - "#### Preprocess the raw dataset\n", + "raw_df = customer_churn.load_data()\n", "\n", - "We'll also perform a number of operations to get ready for the subsequent steps:\n", - "\n", - "\n", - "- **Preprocess the data:** Splits the DataFrame (`df`) into multiple datasets (`train_df`, `validation_df`, and `test_df`) using `demo_dataset.preprocess` to simplify preprocessing.\n", - "- **Separate features and targets:** Drops the target column to create feature sets (`x_train`, `x_val`) and target sets (`y_train`, `y_val`).\n", - "- **Initialize XGBoost classifier:** Creates an `XGBClassifier` object with early stopping rounds set to 10.\n", - "- **Set evaluation metrics:** Specifies metrics for model evaluation as `error`, `logloss`, and `auc`.\n", - "- **Fit the model:** Trains the model on `x_train` and `y_train` using the validation set `(x_val, y_val)`. Verbose output is disabled." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8913f5c8", - "metadata": {}, - "outputs": [], - "source": [ + "# Preprocess the raw dataset\n", "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)\n", "\n", + "# Separate features and targets\n", "x_train = train_df.drop(customer_churn.target_column, axis=1)\n", "y_train = train_df[customer_churn.target_column]\n", "x_val = validation_df.drop(customer_churn.target_column, axis=1)\n", "y_val = validation_df[customer_churn.target_column]\n", "\n", + "# Create an `XGBClassifier` object\n", "model = xgb.XGBClassifier(early_stopping_rounds=10)\n", "model.set_params(\n", " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", ")\n", + "\n", + "# Train the model using the validation set\n", "model.fit(\n", " x_train,\n", " y_train,\n", @@ -138,18 +112,7 @@ "id": "c0e441f4", "metadata": {}, "source": [ - "#### Initialize the datasets\n", - "\n", - "Before you can run tests, you'll need to initialize a ValidMind dataset object using the [`init_dataset`](https://docs.validmind.ai/validmind/validmind.html#init_dataset) function from the ValidMind (`vm`) module.\n", - "\n", - "We'll include the following arguments:\n", - "\n", - "- **`dataset`** — the raw dataset that you want to provide as input to tests\n", - "- **`input_id`** - a unique identifier that allows tracking what inputs are used when running each individual test\n", - "- **`target_column`** — a required argument if tests require access to true values. This is the name of the target column in the dataset\n", - "- **`class_labels`** — an optional value to map predicted classes to class labels\n", - "\n", - "With all datasets ready, you can now initialize the raw, training, and test datasets (`raw_df`, `train_df` and `test_df`) created earlier into their own dataset objects using [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset):" + "Before you can run tests, you'll need to initialize a ValidMind dataset object, as well as a ValidMind model object that can be passed to other functions for analysis and tests on the data:\n" ] }, { @@ -159,6 +122,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Initialize the dataset object\n", "vm_raw_dataset = vm.init_dataset(\n", " dataset=raw_df,\n", " input_id=\"raw_dataset\",\n", @@ -167,66 +131,28 @@ " __log=False,\n", ")\n", "\n", + "# Initialize the datasets into their own dataset objects\n", "vm_train_ds = vm.init_dataset(\n", " dataset=train_df,\n", " input_id=\"train_dataset\",\n", " target_column=customer_churn.target_column,\n", " __log=False,\n", ")\n", - "\n", "vm_test_ds = vm.init_dataset(\n", " dataset=test_df,\n", " input_id=\"test_dataset\",\n", " target_column=customer_churn.target_column,\n", " __log=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "47339b81", - "metadata": {}, - "source": [ - "#### Initialize a model object\n", - "\n", - "Additionally, you'll need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data. \n", + ")\n", "\n", - "Simply intialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82d74165", - "metadata": {}, - "outputs": [], - "source": [ + "# Initialize a model object\n", "vm_model = vm.init_model(\n", " model,\n", " input_id=\"model\",\n", " __log=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "8c5a2b7c", - "metadata": {}, - "source": [ - "#### Assign predictions to the datasets\n", - "\n", - "We can now use the `assign_predictions()` method from the Dataset object to link existing predictions to any model.\n", + ")\n", "\n", - "If no prediction values are passed, the method will compute predictions automatically:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cee7ab79", - "metadata": {}, - "outputs": [], - "source": [ + "# Assign predictions to the datasets\n", "vm_train_ds.assign_predictions(\n", " model=vm_model,\n", ")\n", @@ -277,20 +203,6 @@ "source": [ "### Using `RawData` in custom tests" ] - }, - { - "cell_type": "markdown", - "id": "c3e825ea", - "metadata": {}, - "source": [ - "## Conclusion\n", - "\n", - "ValidMind's `RawData` feature allows you to customize the output of tests, including custom tests. \n", - "\n", - "This notebook has demonstrated how to use the `RawData` feature of ValidMind tests to customize the output of tests. It has also shown how to create custom tests that return `RawData` objects and use them in the same way.\n", - "\n", - "This feature is a powerful tool for creating custom tests and post-processing functions that can be used to generate a wide variety of outputs from ValidMind tests." - ] } ], "metadata": { From 0fc7e4b15b24414231d1d47c335730f9bc5ded62 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Tue, 4 Feb 2025 14:26:29 -0800 Subject: [PATCH 4/8] Cleaning up 1st example --- .../understand_utilize_rawdata.ipynb | 135 +++++++++++++++++- 1 file changed, 132 insertions(+), 3 deletions(-) diff --git a/notebooks/code_sharing/understand_utilize_rawdata.ipynb b/notebooks/code_sharing/understand_utilize_rawdata.ipynb index 2c6a9c047..ad44ffd88 100644 --- a/notebooks/code_sharing/understand_utilize_rawdata.ipynb +++ b/notebooks/code_sharing/understand_utilize_rawdata.ipynb @@ -50,7 +50,7 @@ "# Initialize the ValidMind Library\n", "import validmind as vm\n", "\n", - "# Import the `xgboost` library with an allias\n", + "# Import the `xgboost` library with an alias\n", "import xgboost as xgb\n" ] }, @@ -169,7 +169,12 @@ "source": [ "## Examples\n", "\n", - "Once you're set up, you can then run the following examples:" + "Once you're set up, you can then run the following examples:\n", + "\n", + "- Using `RawData` from the ROC Curve Test\n", + "- Pearson Correlation Matrix\n", + "- Precision-Recall Curve\n", + "- Using `RawData` in custom tests" ] }, { @@ -177,7 +182,131 @@ "id": "18808f71", "metadata": {}, "source": [ - "### Using `RawData` from the ROC Curve Test" + "### Using `RawData` from the ROC Curve Test\n", + "\n", + "In this example, we run the [ROC Curve](https://docs.validmind.ai/tests/model_validation/sklearn/ROCCurve.html) test, inspect its `RawData` output, and then create a custom ROC curve using the raw data values.\n", + "\n", + "First, let's run the default ROC Curve test for comparsion with later iterations:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58a3a779", + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.tests import run_test\n", + "\n", + "# Run the ROC Curve test normally\n", + "result_roc = run_test(\n", + " \"validmind.model_validation.sklearn.ROCCurve\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " generate_description=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "66c44fe0", + "metadata": {}, + "source": [ + "Now let's assume we want to create a custom version of the above figure. First, let's inspect the raw data that this test produces so we can see what we have to work with.\n", + "\n", + "`RawData` objects have a `inspect()` method that will pretty print the attributes of the object to be able to quickly see the data and its types:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "513ce01e", + "metadata": {}, + "outputs": [], + "source": [ + "# Inspect the RawData output from the ROC test\n", + "print(\"RawData from ROC Curve Test:\")\n", + "result_roc.raw_data.inspect()" + ] + }, + { + "cell_type": "markdown", + "id": "586f3a12", + "metadata": {}, + "source": [ + "As we can see, the ROC Curve returns a `RawData` object with the following attributes:\n", + "- **`fpr`:** A list of false positive rates\n", + "- **`tpr`:** A list of true positive rates\n", + "- **`auc`:** The area under the curve\n", + "\n", + "This should be enough to create our own custom ROC curve via a post-processing function without having to create a whole new test from scratch and without having to recompute any of the data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "613778d2", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "from validmind.vm_models.result import TestResult\n", + "\n", + "\n", + "def custom_roc_curve(result: TestResult):\n", + " # Extract raw data from the test result\n", + " fpr = result.raw_data.fpr\n", + " tpr = result.raw_data.tpr\n", + " auc = result.raw_data.auc\n", + "\n", + " # Create a custom ROC curve plot\n", + " fig = plt.figure()\n", + " plt.plot(fpr, tpr, label=f\"Custom ROC (AUC = {auc:.2f})\", color=\"blue\")\n", + " plt.plot([0, 1], [0, 1], linestyle=\"--\", color=\"gray\", label=\"Random Guess\")\n", + " plt.xlabel(\"False Positive Rate\")\n", + " plt.ylabel(\"True Positive Rate\")\n", + " plt.title(\"Custom ROC Curve from RawData\")\n", + " plt.legend()\n", + "\n", + " # close the plot to avoid it automatically being shown in the notebook\n", + " plt.close()\n", + "\n", + " # remove existing figure\n", + " result.remove_figure(0)\n", + "\n", + " # add new figure\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "# test it on the existing result\n", + "modified_result = custom_roc_curve(result_roc)\n", + "\n", + "# show the modified result\n", + "modified_result.show()" + ] + }, + { + "cell_type": "markdown", + "id": "794d026c", + "metadata": {}, + "source": [ + "Now that we have created a post-processing function and verified that it works on our existing test result, we can use it directly in `run_test()` from now on:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d98b5b2", + "metadata": {}, + "outputs": [], + "source": [ + "result = run_test(\n", + " \"validmind.model_validation.sklearn.ROCCurve\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " post_process_fn=custom_roc_curve,\n", + " generate_description=False,\n", + ")" ] }, { From 4ab42b3481f3966e596cf46c180a88c3e5fc0114 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Tue, 4 Feb 2025 14:33:49 -0800 Subject: [PATCH 5/8] Setup for custom tests --- .../understand_utilize_rawdata.ipynb | 185 +++++++++++++++++- 1 file changed, 179 insertions(+), 6 deletions(-) diff --git a/notebooks/code_sharing/understand_utilize_rawdata.ipynb b/notebooks/code_sharing/understand_utilize_rawdata.ipynb index ad44ffd88..526cf33b1 100644 --- a/notebooks/code_sharing/understand_utilize_rawdata.ipynb +++ b/notebooks/code_sharing/understand_utilize_rawdata.ipynb @@ -24,7 +24,8 @@ "\n", "Before we can run our examples, we'll need to set the stage to enable running tests with the ValidMind Library. Since the focus of this notebook is on the `RawData` object, this section will merely summarize the steps instead of going into greater detail. \n", "\n", - "To learn more about running tests with ValidMind: **[Run tests and test suites](https://docs.validmind.ai/developer/model-testing/testing-overview.html)**" + "\n", + "**To learn more about running tests with ValidMind:** [Run tests and test suites](https://docs.validmind.ai/developer/model-testing/testing-overview.html)" ] }, { @@ -169,7 +170,7 @@ "source": [ "## Examples\n", "\n", - "Once you're set up, you can then run the following examples:\n", + "Once you're set up to run tests, you can then try out the following examples:\n", "\n", "- Using `RawData` from the ROC Curve Test\n", "- Pearson Correlation Matrix\n", @@ -184,7 +185,7 @@ "source": [ "### Using `RawData` from the ROC Curve Test\n", "\n", - "In this example, we run the [ROC Curve](https://docs.validmind.ai/tests/model_validation/sklearn/ROCCurve.html) test, inspect its `RawData` output, and then create a custom ROC curve using the raw data values.\n", + "In this introductory example, we run the [ROC Curve](https://docs.validmind.ai/tests/model_validation/sklearn/ROCCurve.html) test, inspect its `RawData` output, and then create a custom ROC curve using the raw data values.\n", "\n", "First, let's run the default ROC Curve test for comparsion with later iterations:" ] @@ -314,7 +315,47 @@ "id": "ed1b2ff9", "metadata": {}, "source": [ - "### Pearson Correlation Matrix" + "### Pearson Correlation Matrix\n", + "\n", + "In this next example, try commenting out the `post_process_fn` argument in the following cell and see what happens between different runs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d131cc49", + "metadata": {}, + "outputs": [], + "source": [ + "import plotly.graph_objects as go\n", + "\n", + "\n", + "def custom_heatmap(result: TestResult):\n", + " corr_matrix = result.raw_data.correlation_matrix\n", + "\n", + " heatmap = go.Heatmap(\n", + " z=corr_matrix.values,\n", + " x=list(corr_matrix.columns),\n", + " y=list(corr_matrix.index),\n", + " colorscale=\"Viridis\",\n", + " )\n", + " fig = go.Figure(data=[heatmap])\n", + " fig.update_layout(title=\"Custom Heatmap from RawData\")\n", + "\n", + " plt.close()\n", + "\n", + " result.remove_figure(0)\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "\n", + "result_corr = run_test(\n", + " \"validmind.data_validation.PearsonCorrelationMatrix\",\n", + " inputs={\"dataset\": vm_test_ds},\n", + " generate_description=False,\n", + " post_process_fn=custom_heatmap,\n", + ")" ] }, { @@ -322,7 +363,41 @@ "id": "d4006753", "metadata": {}, "source": [ - "### Precision-Recall Curve" + "### Precision-Recall Curve\n", + "\n", + "Then, let's try the same thing with the [Precision-Recall Curve](https://docs.validmind.ai/tests/model_validation/sklearn/PrecisionRecallCurve.html) test:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed1c16df", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_pr_curve(result: TestResult):\n", + " precision = result.raw_data.precision\n", + " recall = result.raw_data.recall\n", + "\n", + " fig = plt.figure()\n", + " plt.plot(recall, precision, label=\"Precision-Recall Curve\")\n", + " plt.xlabel(\"Recall\")\n", + " plt.ylabel(\"Precision\")\n", + " plt.title(\"Custom Precision-Recall Curve from RawData\")\n", + " plt.legend()\n", + "\n", + " plt.close()\n", + " result.remove_figure(0)\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "result_pr = run_test(\n", + " \"validmind.model_validation.sklearn.PrecisionRecallCurve\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " generate_description=False,\n", + " post_process_fn=custom_pr_curve,\n", + ")" ] }, { @@ -330,7 +405,105 @@ "id": "eaca281f", "metadata": {}, "source": [ - "### Using `RawData` in custom tests" + "### Using `RawData` in custom tests\n", + "\n", + "These examples demonstrate some very simple ways to use the `RawData` feature of ValidMind tests. The majority of ValidMind-developed tests return some form of raw data that can be used to customize the output of the test, but you can also create your own tests that return `RawData` objects and use them in the same way.\n", + "\n", + "Let's take a look at how this can be done in custom tests." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc6a389f", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from validmind import test, RawData\n", + "from validmind.vm_models import VMDataset, VMModel\n", + "\n", + "\n", + "@test(\"custom.MyCustomTest\")\n", + "def MyCustomTest(dataset: VMDataset, model: VMModel) -> tuple[go.Figure, RawData]:\n", + " \"\"\"Custom test that produces a figure and a RawData object\"\"\"\n", + " # pretend we are using the dataset and model to compute some data\n", + " # ...\n", + "\n", + " # create some fake data that will be used to generate a figure\n", + " data = pd.DataFrame({\"x\": [10, 20, 30, 40, 50], \"y\": [10, 20, 30, 40, 50]})\n", + "\n", + " # create the figure (scatter plot)\n", + " fig = go.Figure(data=go.Scatter(x=data[\"x\"], y=data[\"y\"]))\n", + "\n", + " # now let's create a RawData object that holds the \"computed\" data\n", + " raw_data = RawData(scatter_data_df=data)\n", + "\n", + " # finally, return both the figure and the raw data\n", + " return fig, raw_data\n", + "\n", + "\n", + "my_result = run_test(\n", + " \"custom.MyCustomTest\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " generate_description=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "854c219c", + "metadata": {}, + "source": [ + "We can see that the test result shows the figure. But since we returned a `RawData` object, we can also inspect the contents and see how we could use it to customize or regenerate the figure in the post-processing function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1cb661d1", + "metadata": {}, + "outputs": [], + "source": [ + "my_result.raw_data.inspect()" + ] + }, + { + "cell_type": "markdown", + "id": "55ad4acd", + "metadata": {}, + "source": [ + "We can see that we get a nicely-formatted preview of the dataframe we stored in the raw data object. Let's go ahead and use it to re-plot our data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1242083", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_plot(result: TestResult):\n", + " data = result.raw_data.scatter_data_df\n", + "\n", + " # use something other than a scatter plot\n", + " fig = go.Figure(data=go.Bar(x=data[\"x\"], y=data[\"y\"]))\n", + " fig.update_layout(title=\"Custom Bar Chart from RawData\")\n", + " fig.update_xaxes(title=\"X Axis\")\n", + " fig.update_yaxes(title=\"Y Axis\")\n", + "\n", + " result.remove_figure(0)\n", + " result.add_figure(fig)\n", + "\n", + " return result\n", + "\n", + "result = run_test(\n", + " \"custom.MyCustomTest\",\n", + " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", + " post_process_fn=custom_plot,\n", + " generate_description=False,\n", + ")" ] } ], From 8cfebe7394943f58a1311d88a1114a403631348d Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Tue, 4 Feb 2025 14:40:27 -0800 Subject: [PATCH 6/8] Adding ToC --- .../understand_utilize_rawdata.ipynb | 89 +++++++++++++------ 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/notebooks/code_sharing/understand_utilize_rawdata.ipynb b/notebooks/code_sharing/understand_utilize_rawdata.ipynb index 526cf33b1..4f920ef37 100644 --- a/notebooks/code_sharing/understand_utilize_rawdata.ipynb +++ b/notebooks/code_sharing/understand_utilize_rawdata.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "4f9a3237", + "id": "c18ba8a2", "metadata": {}, "source": [ "# Understand and utilize `RawData` in ValidMind tests\n", @@ -17,9 +17,39 @@ }, { "cell_type": "markdown", - "id": "7d0ceab4", + "id": "5b5b248c", "metadata": {}, "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [Setup](#toc1_) \n", + " - [Installation and intialization](#toc1_1_) \n", + " - [Load the sample dataset](#toc1_2_) \n", + " - [Initialize the ValidMind objects](#toc1_3_) \n", + "- [`RawData` usage examples](#toc2_) \n", + " - [Using `RawData` from the ROC Curve Test](#toc2_1_) \n", + " - [Pearson Correlation Matrix](#toc2_2_) \n", + " - [Precision-Recall Curve](#toc2_3_) \n", + " - [Using `RawData` in custom tests](#toc2_4_) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "6dd79a98", + "metadata": {}, + "source": [ + "\n", + "\n", "## Setup\n", "\n", "Before we can run our examples, we'll need to set the stage to enable running tests with the ValidMind Library. Since the focus of this notebook is on the `RawData` object, this section will merely summarize the steps instead of going into greater detail. \n", @@ -30,9 +60,10 @@ }, { "cell_type": "markdown", - "id": "install-library-eeed10fa-6015-4d97-bcaa-32098df7f764", "metadata": {}, "source": [ + "\n", + "\n", "### Installation and intialization\n", "\n", "First, let's make sure that the ValidMind Library is installed and ready to go, and our Python environment set up for data analysis:" @@ -41,7 +72,7 @@ { "cell_type": "code", "execution_count": null, - "id": "install-python-b56f71e6-6faf-4125-bb78-203a240f65b3", + "id": "04eb084e", "metadata": {}, "outputs": [], "source": [ @@ -57,9 +88,10 @@ }, { "cell_type": "markdown", - "id": "67a2992e", "metadata": {}, "source": [ + "\n", + "\n", "### Load the sample dataset\n", "\n", "Then, we'll import a sample ValidMind dataset and preprocess it:" @@ -68,7 +100,7 @@ { "cell_type": "code", "execution_count": null, - "id": "dc19d4f8", + "id": "50d72eba", "metadata": {}, "outputs": [], "source": [ @@ -102,9 +134,10 @@ }, { "cell_type": "markdown", - "id": "7f904540", "metadata": {}, "source": [ + "\n", + "\n", "### Initialize the ValidMind objects" ] }, @@ -119,7 +152,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6c072304", + "id": "b2310bc4", "metadata": {}, "outputs": [], "source": [ @@ -165,24 +198,27 @@ }, { "cell_type": "markdown", - "id": "e5b08241", + "id": "25ec99fc", "metadata": {}, "source": [ - "## Examples\n", + "\n", + "\n", + "## `RawData` usage examples\n", "\n", "Once you're set up to run tests, you can then try out the following examples:\n", "\n", - "- Using `RawData` from the ROC Curve Test\n", - "- Pearson Correlation Matrix\n", - "- Precision-Recall Curve\n", - "- Using `RawData` in custom tests" + " - [Using `RawData` from the ROC Curve Test](#toc2_1_) \n", + " - [Pearson Correlation Matrix](#toc2_2_) \n", + " - [Precision-Recall Curve](#toc2_3_) \n", + " - [Using `RawData` in custom tests](#toc2_4_) " ] }, { "cell_type": "markdown", - "id": "18808f71", "metadata": {}, "source": [ + "\n", + "\n", "### Using `RawData` from the ROC Curve Test\n", "\n", "In this introductory example, we run the [ROC Curve](https://docs.validmind.ai/tests/model_validation/sklearn/ROCCurve.html) test, inspect its `RawData` output, and then create a custom ROC curve using the raw data values.\n", @@ -298,7 +334,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7d98b5b2", + "id": "7c7566f3", "metadata": {}, "outputs": [], "source": [ @@ -312,9 +348,10 @@ }, { "cell_type": "markdown", - "id": "ed1b2ff9", "metadata": {}, "source": [ + "\n", + "\n", "### Pearson Correlation Matrix\n", "\n", "In this next example, try commenting out the `post_process_fn` argument in the following cell and see what happens between different runs:" @@ -323,7 +360,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d131cc49", + "id": "c57fb01b", "metadata": {}, "outputs": [], "source": [ @@ -360,9 +397,10 @@ }, { "cell_type": "markdown", - "id": "d4006753", "metadata": {}, "source": [ + "\n", + "\n", "### Precision-Recall Curve\n", "\n", "Then, let's try the same thing with the [Precision-Recall Curve](https://docs.validmind.ai/tests/model_validation/sklearn/PrecisionRecallCurve.html) test:" @@ -371,7 +409,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ed1c16df", + "id": "d16c5209", "metadata": {}, "outputs": [], "source": [ @@ -402,14 +440,15 @@ }, { "cell_type": "markdown", - "id": "eaca281f", "metadata": {}, "source": [ + "\n", + "\n", "### Using `RawData` in custom tests\n", "\n", "These examples demonstrate some very simple ways to use the `RawData` feature of ValidMind tests. The majority of ValidMind-developed tests return some form of raw data that can be used to customize the output of the test, but you can also create your own tests that return `RawData` objects and use them in the same way.\n", "\n", - "Let's take a look at how this can be done in custom tests." + "Let's take a look at how this can be done in custom tests. To start, define and run your custom test:" ] }, { @@ -456,7 +495,7 @@ "id": "854c219c", "metadata": {}, "source": [ - "We can see that the test result shows the figure. But since we returned a `RawData` object, we can also inspect the contents and see how we could use it to customize or regenerate the figure in the post-processing function." + "We can see that the test result shows the figure. But since we returned a `RawData` object, we can also inspect the contents and see how we could use it to customize or regenerate the figure in the post-processing function:" ] }, { @@ -474,7 +513,7 @@ "id": "55ad4acd", "metadata": {}, "source": [ - "We can see that we get a nicely-formatted preview of the dataframe we stored in the raw data object. Let's go ahead and use it to re-plot our data." + "We can see that we get a nicely-formatted preview of the dataframe we stored in the raw data object. Let's go ahead and use it to re-plot our data:" ] }, { @@ -515,7 +554,7 @@ }, "language_info": { "name": "python", - "version": "3.10" + "version": "3.10.13" } }, "nbformat": 4, From f42b6e1f234100c012b9a98e8270ff39cc0945f4 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Tue, 4 Feb 2025 14:44:40 -0800 Subject: [PATCH 7/8] Final touches --- notebooks/code_sharing/understand_utilize_rawdata.ipynb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/notebooks/code_sharing/understand_utilize_rawdata.ipynb b/notebooks/code_sharing/understand_utilize_rawdata.ipynb index 4f920ef37..5aba5bebd 100644 --- a/notebooks/code_sharing/understand_utilize_rawdata.ipynb +++ b/notebooks/code_sharing/understand_utilize_rawdata.ipynb @@ -391,6 +391,7 @@ " \"validmind.data_validation.PearsonCorrelationMatrix\",\n", " inputs={\"dataset\": vm_test_ds},\n", " generate_description=False,\n", + " # COMMENT OUT `post_process_fn`\n", " post_process_fn=custom_heatmap,\n", ")" ] @@ -434,6 +435,7 @@ " \"validmind.model_validation.sklearn.PrecisionRecallCurve\",\n", " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", " generate_description=False,\n", + " # COMMENT OUT `post_process_fn`\n", " post_process_fn=custom_pr_curve,\n", ")" ] From 571e488178c7d8c5a14cd7bd309225f4e8272fb1 Mon Sep 17 00:00:00 2001 From: Beck <164545837+validbeck@users.noreply.github.com> Date: Tue, 4 Feb 2025 14:47:02 -0800 Subject: [PATCH 8/8] Merged notebooks & moved to how_to --- notebooks/code_sharing/rawdata_usage.ipynb | 437 ------------------ .../understand_utilize_rawdata.ipynb | 7 + 2 files changed, 7 insertions(+), 437 deletions(-) delete mode 100644 notebooks/code_sharing/rawdata_usage.ipynb rename notebooks/{code_sharing => how_to}/understand_utilize_rawdata.ipynb (99%) diff --git a/notebooks/code_sharing/rawdata_usage.ipynb b/notebooks/code_sharing/rawdata_usage.ipynb deleted file mode 100644 index 1d66c9eef..000000000 --- a/notebooks/code_sharing/rawdata_usage.ipynb +++ /dev/null @@ -1,437 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Understanding and Utilizing RawData in ValidMind Tests" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In ValidMind, test functions can return a special object called **RawData**. This object holds, as the name suggests, intermediate or unprocessed data that is produced somewhere in the test logic but is not returned as part of the test's visible output (tables or figures). This data can be useful when running post-processing functions with tests to recompute tabular outputs, redraw figure or even create new outputs entirely. In this notebook, we demonstrate how to access, inspect, and utilize RawData from ValidMind tests with a couple of examples." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import xgboost as xgb\n", - "import validmind as vm\n", - "from validmind.datasets.classification import customer_churn\n", - "\n", - "raw_df = customer_churn.load_data()\n", - "\n", - "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)\n", - "\n", - "x_train = train_df.drop(customer_churn.target_column, axis=1)\n", - "y_train = train_df[customer_churn.target_column]\n", - "x_val = validation_df.drop(customer_churn.target_column, axis=1)\n", - "y_val = validation_df[customer_churn.target_column]\n", - "\n", - "model = xgb.XGBClassifier(early_stopping_rounds=10)\n", - "model.set_params(\n", - " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", - ")\n", - "model.fit(\n", - " x_train,\n", - " y_train,\n", - " eval_set=[(x_val, y_val)],\n", - " verbose=False,\n", - ")\n", - "\n", - "vm_raw_dataset = vm.init_dataset(\n", - " dataset=raw_df,\n", - " input_id=\"raw_dataset\",\n", - " target_column=customer_churn.target_column,\n", - " class_labels=customer_churn.class_labels,\n", - " __log=False,\n", - ")\n", - "\n", - "vm_train_ds = vm.init_dataset(\n", - " dataset=train_df,\n", - " input_id=\"train_dataset\",\n", - " target_column=customer_churn.target_column,\n", - " __log=False,\n", - ")\n", - "\n", - "vm_test_ds = vm.init_dataset(\n", - " dataset=test_df,\n", - " input_id=\"test_dataset\",\n", - " target_column=customer_churn.target_column,\n", - " __log=False,\n", - ")\n", - "\n", - "vm_model = vm.init_model(\n", - " model,\n", - " input_id=\"model\",\n", - " __log=False,\n", - ")\n", - "\n", - "vm_train_ds.assign_predictions(\n", - " model=vm_model,\n", - ")\n", - "\n", - "vm_test_ds.assign_predictions(\n", - " model=vm_model,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example 1: Using RawData from the ROC Curve Test\n", - "\n", - "In this example, we run the ROC Curve test, inspect its RawData output, and then create a custom ROC curve using the raw data values." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from validmind.tests import run_test\n", - "\n", - "# Run the ROC Curve test normally\n", - "result_roc = run_test(\n", - " \"validmind.model_validation.sklearn.ROCCurve\",\n", - " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", - " generate_description=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let's assume we want to create a custom version of the above figure. First, let's inspect the raw data that this test produces so we can see what we have to work with.\n", - "\n", - "`RawData` objects have a `inspect()` method that will pretty print the attributes of the object to be able to quickly see the data and its types." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Inspect the RawData output from the ROC test\n", - "print(\"RawData from ROC Curve Test:\")\n", - "result_roc.raw_data.inspect()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As we can see, the ROC Curve returns a `RawData` object with the following attributes:\n", - "- `fpr`: A list of false positive rates\n", - "- `tpr`: A list of true positive rates\n", - "- `auc`: The area under the curve\n", - "\n", - "This should be enough to create our own custom ROC curve via a post-processing function without having to create a whole new test from scratch and without having to recompute any of the data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "from validmind.vm_models.result import TestResult\n", - "\n", - "\n", - "def custom_roc_curve(result: TestResult):\n", - " # Extract raw data from the test result\n", - " fpr = result.raw_data.fpr\n", - " tpr = result.raw_data.tpr\n", - " auc = result.raw_data.auc\n", - "\n", - " # Create a custom ROC curve plot\n", - " fig = plt.figure()\n", - " plt.plot(fpr, tpr, label=f\"Custom ROC (AUC = {auc:.2f})\", color=\"blue\")\n", - " plt.plot([0, 1], [0, 1], linestyle=\"--\", color=\"gray\", label=\"Random Guess\")\n", - " plt.xlabel(\"False Positive Rate\")\n", - " plt.ylabel(\"True Positive Rate\")\n", - " plt.title(\"Custom ROC Curve from RawData\")\n", - " plt.legend()\n", - "\n", - " # close the plot to avoid it automatically being shown in the notebook\n", - " plt.close()\n", - "\n", - " # remove existing figure\n", - " result.remove_figure(0)\n", - "\n", - " # add new figure\n", - " result.add_figure(fig)\n", - "\n", - " return result\n", - "\n", - "# test it on the existing result\n", - "modified_result = custom_roc_curve(result_roc)\n", - "\n", - "# show the modified result\n", - "modified_result.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that we have created a post-processing function and verified that it works on our existing test result, we can use it directly in `run_test()` from now on:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "result = run_test(\n", - " \"validmind.model_validation.sklearn.ROCCurve\",\n", - " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", - " post_process_fn=custom_roc_curve,\n", - " generate_description=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## More Examples\n", - "\n", - "### Pearson Correlation Matrix" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Try commenting out the `post_process_fn` argument in the following cell and see what happens between different runs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import plotly.graph_objects as go\n", - "\n", - "\n", - "def custom_heatmap(result: TestResult):\n", - " corr_matrix = result.raw_data.correlation_matrix\n", - "\n", - " heatmap = go.Heatmap(\n", - " z=corr_matrix.values,\n", - " x=list(corr_matrix.columns),\n", - " y=list(corr_matrix.index),\n", - " colorscale=\"Viridis\",\n", - " )\n", - " fig = go.Figure(data=[heatmap])\n", - " fig.update_layout(title=\"Custom Heatmap from RawData\")\n", - "\n", - " plt.close()\n", - "\n", - " result.remove_figure(0)\n", - " result.add_figure(fig)\n", - "\n", - " return result\n", - "\n", - "\n", - "result_corr = run_test(\n", - " \"validmind.data_validation.PearsonCorrelationMatrix\",\n", - " inputs={\"dataset\": vm_test_ds},\n", - " generate_description=False,\n", - " post_process_fn=custom_heatmap,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Precision-Recall Curve" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's try the same thing with the Precision-Recall Curve test." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def custom_pr_curve(result: TestResult):\n", - " precision = result.raw_data.precision\n", - " recall = result.raw_data.recall\n", - "\n", - " fig = plt.figure()\n", - " plt.plot(recall, precision, label=\"Precision-Recall Curve\")\n", - " plt.xlabel(\"Recall\")\n", - " plt.ylabel(\"Precision\")\n", - " plt.title(\"Custom Precision-Recall Curve from RawData\")\n", - " plt.legend()\n", - "\n", - " plt.close()\n", - " result.remove_figure(0)\n", - " result.add_figure(fig)\n", - "\n", - " return result\n", - "\n", - "result_pr = run_test(\n", - " \"validmind.model_validation.sklearn.PrecisionRecallCurve\",\n", - " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", - " generate_description=False,\n", - " post_process_fn=custom_pr_curve,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using RawData in Custom Tests\n", - "\n", - "These examples demonstrate some very simple ways to use the `RawData` feature of ValidMind tests. The majority of ValidMind-developed tests return some form of raw data that can be used to customize the output of the test. But you can also create your own tests that return `RawData` objects and use them in the same way.\n", - "\n", - "Let's take a look at how this can be done in custom tests." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "from validmind import test, RawData\n", - "from validmind.vm_models import VMDataset, VMModel\n", - "\n", - "\n", - "@test(\"custom.MyCustomTest\")\n", - "def MyCustomTest(dataset: VMDataset, model: VMModel) -> tuple[go.Figure, RawData]:\n", - " \"\"\"Custom test that produces a figure and a RawData object\"\"\"\n", - " # pretend we are using the dataset and model to compute some data\n", - " # ...\n", - "\n", - " # create some fake data that will be used to generate a figure\n", - " data = pd.DataFrame({\"x\": [10, 20, 30, 40, 50], \"y\": [10, 20, 30, 40, 50]})\n", - "\n", - " # create the figure (scatter plot)\n", - " fig = go.Figure(data=go.Scatter(x=data[\"x\"], y=data[\"y\"]))\n", - "\n", - " # now let's create a RawData object that holds the \"computed\" data\n", - " raw_data = RawData(scatter_data_df=data)\n", - "\n", - " # finally, return both the figure and the raw data\n", - " return fig, raw_data\n", - "\n", - "\n", - "my_result = run_test(\n", - " \"custom.MyCustomTest\",\n", - " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", - " generate_description=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see that the test result shows the figure. But since we returned a `RawData` object, we can also inspect the contents and see how we could use it to customize or regenerate the figure in the post-processing function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "my_result.raw_data.inspect()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see that we get a nicely-formatted preview of the dataframe we stored in the raw data object. Let's go ahead and use it to re-plot our data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def custom_plot(result: TestResult):\n", - " data = result.raw_data.scatter_data_df\n", - "\n", - " # use something other than a scatter plot\n", - " fig = go.Figure(data=go.Bar(x=data[\"x\"], y=data[\"y\"]))\n", - " fig.update_layout(title=\"Custom Bar Chart from RawData\")\n", - " fig.update_xaxes(title=\"X Axis\")\n", - " fig.update_yaxes(title=\"Y Axis\")\n", - "\n", - " result.remove_figure(0)\n", - " result.add_figure(fig)\n", - "\n", - " return result\n", - "\n", - "result = run_test(\n", - " \"custom.MyCustomTest\",\n", - " inputs={\"dataset\": vm_test_ds, \"model\": vm_model},\n", - " post_process_fn=custom_plot,\n", - " generate_description=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Conclusion\n", - "\n", - "This notebook has demonstrated how to use the `RawData` feature of ValidMind tests to customize the output of tests. It has also shown how to create custom tests that return `RawData` objects and use them in the same way.\n", - "\n", - "This feature is a powerful tool for creating custom tests and post-processing functions that can be used to generate a wide variety of outputs from ValidMind tests." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/code_sharing/understand_utilize_rawdata.ipynb b/notebooks/how_to/understand_utilize_rawdata.ipynb similarity index 99% rename from notebooks/code_sharing/understand_utilize_rawdata.ipynb rename to notebooks/how_to/understand_utilize_rawdata.ipynb index 5aba5bebd..7354eae85 100644 --- a/notebooks/code_sharing/understand_utilize_rawdata.ipynb +++ b/notebooks/how_to/understand_utilize_rawdata.ipynb @@ -60,6 +60,7 @@ }, { "cell_type": "markdown", + "id": "5b6d8d15", "metadata": {}, "source": [ "\n", @@ -88,6 +89,7 @@ }, { "cell_type": "markdown", + "id": "5e6aa2cb", "metadata": {}, "source": [ "\n", @@ -134,6 +136,7 @@ }, { "cell_type": "markdown", + "id": "e3895d35", "metadata": {}, "source": [ "\n", @@ -215,6 +218,7 @@ }, { "cell_type": "markdown", + "id": "33d79841", "metadata": {}, "source": [ "\n", @@ -348,6 +352,7 @@ }, { "cell_type": "markdown", + "id": "1d0b94aa", "metadata": {}, "source": [ "\n", @@ -398,6 +403,7 @@ }, { "cell_type": "markdown", + "id": "0a7cbbc6", "metadata": {}, "source": [ "\n", @@ -442,6 +448,7 @@ }, { "cell_type": "markdown", + "id": "e25391a4", "metadata": {}, "source": [ "\n",