diff --git a/.gitignore b/.gitignore index 9581320a9..b29e2dfe1 100644 --- a/.gitignore +++ b/.gitignore @@ -226,4 +226,3 @@ my_tests/ # Quarto docs docs/validmind.json *.html -*.qmd diff --git a/docs/_sidebar.yml b/docs/_sidebar.yml index 9683b355b..aa9e2b0d0 100644 --- a/docs/_sidebar.yml +++ b/docs/_sidebar.yml @@ -10,7 +10,7 @@ website: - text: "---" - text: "Python API" # Root level items from validmind.qmd - - text: "`2.9.5`" + - text: "`2.10.0`" file: validmind/validmind.qmd#version__ - text: "init" file: validmind/validmind.qmd#init diff --git a/docs/validmind.qmd b/docs/validmind.qmd index 3fa4c1e74..e6a5fca30 100644 --- a/docs/validmind.qmd +++ b/docs/validmind.qmd @@ -44,7 +44,7 @@ After you have pasted the code snippet into your development source code and exe ::: {.signature} -2.9.5 +2.10.0 ::: diff --git a/docs/validmind/datasets/llm.qmd b/docs/validmind/datasets/llm.qmd new file mode 100644 index 000000000..f36936c4c --- /dev/null +++ b/docs/validmind/datasets/llm.qmd @@ -0,0 +1,250 @@ +--- +title: "[validmind](/validmind/validmind.qmd).llm" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +Entrypoint for LLM datasets. + +- [rag](llm/rag.qmd) + + + +## LLMAgentDataset + + + +::: {.signature} + +classLLMAgentDataset(VMDataset): + +::: + + + +LLM Agent Dataset for DeepEval integration with ValidMind. + +This dataset class allows you to use all DeepEval tests and metrics within the ValidMind evaluation framework. It stores LLM interaction data in a format compatible with both frameworks. + +**Arguments** + +- `test_cases (List[LLMTestCase])`: List of DeepEval test cases +- `goldens (List[Golden])`: List of DeepEval golden templates +- `deepeval_dataset (EvaluationDataset)`: DeepEval dataset instance + +### LLMAgentDataset + + + +::: {.signature} + +LLMAgentDataset(input_id:str=None,test_cases:Optional\[List\]=None,goldens:Optional\[List\]=None,deepeval_dataset:Optional\[Any\]=None,\*\*kwargs) + +::: + + + +Initialize LLMAgentDataset. + +**Arguments** + +- `input_id`: Identifier for the dataset +- `test_cases`: List of DeepEval LLMTestCase objects +- `goldens`: List of DeepEval Golden objects +- `deepeval_dataset`: DeepEval EvaluationDataset instance +- `**kwargs`: Additional arguments passed to VMDataset + +### add_golden + + + +::: {.signature} + +defadd_golden(self,golden): + +::: + + + +Add a DeepEval golden to the dataset. + +**Arguments** + +- `golden`: DeepEval Golden instance + +### add_test_case + + + +::: {.signature} + +defadd_test_case(self,test_case): + +::: + + + +Add a DeepEval test case to the dataset. + +**Arguments** + +- `test_case`: DeepEval LLMTestCase instance + +### convert_goldens_to_test_cases + + + +::: {.signature} + +defconvert_goldens_to_test_cases(self,llm_app_function): + +::: + + + +Convert goldens to test cases by generating actual outputs. + +**Arguments** + +- `llm_app_function`: Function that takes input and returns LLM output + +### evaluate_with_deepeval + + + +::: {.signature} + +defevaluate_with_deepeval(self,metrics:List,\*\*kwargs)Dict\[str, Any\]: + +::: + + + +Evaluate the dataset using DeepEval metrics. + +**Arguments** + +- `metrics`: List of DeepEval metric instances +- `**kwargs`: Additional arguments passed to deepeval.evaluate() + +**Returns** + +- Evaluation results dictionary + +### from_deepeval_dataset + + + +::: {.signature} + +@classmethod + +deffrom_deepeval_dataset(cls,deepeval_dataset,input_id:str='llm_agent_dataset',\*\*kwargs)validmind.vm_models.LLMAgentDataset: + +::: + + + +Create LLMAgentDataset from DeepEval EvaluationDataset. + +**Arguments** + +- `deepeval_dataset`: DeepEval EvaluationDataset instance +- `input_id`: Dataset identifier +- `**kwargs`: Additional arguments + +**Returns** + +- LLMAgentDataset instance + +### from_goldens + + + +::: {.signature} + +@classmethod + +deffrom_goldens(cls,goldens:List,input_id:str='llm_agent_dataset',\*\*kwargs)validmind.vm_models.LLMAgentDataset: + +::: + + + +Create LLMAgentDataset from DeepEval goldens. + +**Arguments** + +- `goldens`: List of DeepEval Golden objects +- `input_id`: Dataset identifier +- `**kwargs`: Additional arguments + +**Returns** + +- LLMAgentDataset instance + +### from_test_cases + + + +::: {.signature} + +@classmethod + +deffrom_test_cases(cls,test_cases:List,input_id:str='llm_agent_dataset',\*\*kwargs)validmind.vm_models.LLMAgentDataset: + +::: + + + +Create LLMAgentDataset from DeepEval test cases. + +**Arguments** + +- `test_cases`: List of DeepEval LLMTestCase objects +- `input_id`: Dataset identifier +- `**kwargs`: Additional arguments + +**Returns** + +- LLMAgentDataset instance + +### get_deepeval_dataset + + + +::: {.signature} + +defget_deepeval_dataset(self): + +::: + + + +Get or create a DeepEval EvaluationDataset instance. + +**Returns** + +- DeepEval EvaluationDataset instance + +### to_deepeval_test_cases + + + +::: {.signature} + +defto_deepeval_test_cases(self)List: + +::: + + + +Convert dataset rows back to DeepEval test cases. + +**Returns** + +- List of DeepEval LLMTestCase objects diff --git a/docs/validmind/datasets/llm/rag.qmd b/docs/validmind/datasets/llm/rag.qmd new file mode 100644 index 000000000..533f760a4 --- /dev/null +++ b/docs/validmind/datasets/llm/rag.qmd @@ -0,0 +1,13 @@ +--- +title: "[validmind](/validmind/validmind.qmd).rag" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +Entrypoint for classification datasets. + +- [rfp](rag/rfp.qmd) diff --git a/docs/validmind/datasets/llm/rag/rfp.qmd b/docs/validmind/datasets/llm/rag/rfp.qmd new file mode 100644 index 000000000..98337190e --- /dev/null +++ b/docs/validmind/datasets/llm/rag/rfp.qmd @@ -0,0 +1,31 @@ +--- +title: "[validmind](/validmind/validmind.qmd).rfp" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +## load_data + + + +::: {.signature} + +defload_data(full_dataset=False): + +::: + + + +## preprocess + + + +::: {.signature} + +defpreprocess(df): + +::: diff --git a/docs/validmind/scorer.qmd b/docs/validmind/scorer.qmd new file mode 100644 index 000000000..819edd0dc --- /dev/null +++ b/docs/validmind/scorer.qmd @@ -0,0 +1,110 @@ +--- +title: "[validmind](/validmind/validmind.qmd).scorer" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + +## scorer + + + +::: {.signature} + +defscorer(func_or_id:Union\[Callable\[..., Any\], str, None\]=None)Callable\[\[validmind.vm_models.F\], validmind.vm_models.F\]: + +::: + + + +Decorator for creating and registering custom scorers + +This decorator registers the function it wraps as a scorer function within ValidMind under the provided ID. Once decorated, the function can be run using the `validmind.scorer.run_scorer` function. + +The scorer ID can be provided in three ways: + +1. Explicit ID: `@scorer("validmind.scorer.classification.BrierScore")` +1. Auto-generated from path: `@scorer()` - automatically generates ID from file path +1. Function name only: `@scorer` - uses function name with validmind.scorer prefix + +The function can take two different types of arguments: + +- Inputs: ValidMind model or dataset (or list of models/datasets). These arguments must use the following names: `model`, `models`, `dataset`, `datasets`. +- Parameters: Any additional keyword arguments of any type (must have a default value) that can have any name. + +The function should return one of the following types: + +- Table: Either a list of dictionaries or a pandas DataFrame +- Plot: Either a matplotlib figure or a plotly figure +- Scalar: A single number (int or float) +- Boolean: A single boolean value indicating whether the test passed or failed +- List: A list of values (for row-level metrics) or a list of dictionaries with consistent keys +- Any other type: The output will be stored as raw data for use by calling code + +When returning a list of dictionaries: + +- All dictionaries must have the same keys +- The list length must match the number of rows in the dataset +- Each dictionary key will become a separate column when using assign_scores +- Column naming follows the pattern: {model_id}_{metric_name}_{dict_key} + +Note: Scorer outputs are not logged to the backend and are intended for use by other parts of the system (e.g., assign_scores method). + +The function may also include a docstring. This docstring will be used and logged as the scorer's description. + +**Arguments** + +- `func_or_id (Union[Callable[..., Any], str, None], optional)`: Either the function to decorate or the scorer ID. If None or empty string, the ID is auto-generated from the file path. Defaults to None. + +**Returns** + +- The decorated function. + + + +## list_scorers + + + +::: {.signature} + +deflist_scorers(\*\*kwargs): + +::: + + + +List all scorers + + + +## describe_scorer + + + +::: {.signature} + +defdescribe_scorer(scorer_id:str,\*\*kwargs): + +::: + + + +Describe a scorer + + + +## run_scorer + + + +::: {.signature} + +defrun_scorer(scorer_id:str,\*\*kwargs): + +::: + + + +Run a scorer diff --git a/docs/validmind/tests/plots.qmd b/docs/validmind/tests/plots.qmd new file mode 100644 index 000000000..7d55dedf8 --- /dev/null +++ b/docs/validmind/tests/plots.qmd @@ -0,0 +1,12 @@ +--- +title: "[validmind](/validmind/validmind.qmd).plots" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + +- [BoxPlot](plots/BoxPlot.qmd) +- [CorrelationHeatmap](plots/CorrelationHeatmap.qmd) +- [HistogramPlot](plots/HistogramPlot.qmd) +- [ViolinPlot](plots/ViolinPlot.qmd) diff --git a/docs/validmind/tests/plots/BoxPlot.qmd b/docs/validmind/tests/plots/BoxPlot.qmd new file mode 100644 index 000000000..6054feb95 --- /dev/null +++ b/docs/validmind/tests/plots/BoxPlot.qmd @@ -0,0 +1,63 @@ +--- +title: "[validmind](/validmind/validmind.qmd).BoxPlot" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +## BoxPlot + + + +::: {.signature} + +@tags('tabular_data', 'visualization', 'data_quality') + +@tasks('classification', 'regression', 'clustering') + +defBoxPlot(dataset:validmind.vm_models.VMDataset,columns:Optional\[List\[str\]\]=None,group_by:Optional\[str\]=None,width:int=1200,height:int=600,colors:Optional\[List\[str\]\]=None,show_outliers:bool=True,title_prefix:str='Box Plot of')go.Figure: + +::: + + + +Generates customizable box plots for numerical features in a dataset with optional grouping using Plotly. + +### Purpose + +This test provides a flexible way to visualize the distribution of numerical features through interactive box plots, with optional grouping by categorical variables. Box plots are effective for identifying outliers, comparing distributions across groups, and understanding the spread and central tendency of the data. + +### Test Mechanism + +The test creates interactive box plots for specified numerical columns (or all numerical columns if none specified). It supports various customization options including: + +- Grouping by categorical variables +- Customizable colors and styling +- Outlier display options +- Interactive hover information +- Zoom and pan capabilities + +### Signs of High Risk + +- Presence of many outliers indicating data quality issues +- Highly skewed distributions +- Large differences in variance across groups +- Unexpected patterns in grouped data + +### Strengths + +- Clear visualization of distribution statistics (median, quartiles, outliers) +- Interactive Plotly plots with hover information and zoom capabilities +- Effective for comparing distributions across groups +- Handles missing values appropriately +- Highly customizable appearance + +### Limitations + +- Limited to numerical features only +- May not be suitable for continuous variables with many unique values +- Visual interpretation may be subjective +- Less effective with very large datasets diff --git a/docs/validmind/tests/plots/CorrelationHeatmap.qmd b/docs/validmind/tests/plots/CorrelationHeatmap.qmd new file mode 100644 index 000000000..410a51410 --- /dev/null +++ b/docs/validmind/tests/plots/CorrelationHeatmap.qmd @@ -0,0 +1,64 @@ +--- +title: "[validmind](/validmind/validmind.qmd).CorrelationHeatmap" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +## CorrelationHeatmap + + + +::: {.signature} + +@tags('tabular_data', 'visualization', 'correlation') + +@tasks('classification', 'regression', 'clustering') + +defCorrelationHeatmap(dataset:validmind.vm_models.VMDataset,columns:Optional\[List\[str\]\]=None,method:str='pearson',show_values:bool=True,colorscale:str='RdBu',width:int=800,height:int=600,mask_upper:bool=False,threshold:Optional\[float\]=None,title:str='Correlation Heatmap')go.Figure: + +::: + + + +Generates customizable correlation heatmap plots for numerical features in a dataset using Plotly. + +### Purpose + +This test provides a flexible way to visualize correlations between numerical features in a dataset using interactive Plotly heatmaps. It supports different correlation methods and extensive customization options for the heatmap appearance, making it suitable for exploring feature relationships in data analysis. + +### Test Mechanism + +The test computes correlation coefficients between specified numerical columns (or all numerical columns if none specified) using the specified method. It then creates an interactive heatmap visualization with customizable appearance options including: + +- Different correlation methods (pearson, spearman, kendall) +- Color schemes and annotations +- Masking options for upper triangle +- Threshold filtering for significant correlations +- Interactive hover information + +### Signs of High Risk + +- Very high correlations (>0.9) between features indicating multicollinearity +- Unexpected correlation patterns that contradict domain knowledge +- Features with no correlation to any other variables +- Strong correlations with the target variable that might indicate data leakage + +### Strengths + +- Supports multiple correlation methods +- Interactive Plotly plots with hover information and zoom capabilities +- Highly customizable visualization options +- Can handle missing values appropriately +- Provides clear visual representation of feature relationships +- Optional thresholding to focus on significant correlations + +### Limitations + +- Limited to numerical features only +- Cannot capture non-linear relationships effectively +- May be difficult to interpret with many features +- Correlation does not imply causation diff --git a/docs/validmind/tests/plots/HistogramPlot.qmd b/docs/validmind/tests/plots/HistogramPlot.qmd new file mode 100644 index 000000000..f9610ff5d --- /dev/null +++ b/docs/validmind/tests/plots/HistogramPlot.qmd @@ -0,0 +1,65 @@ +--- +title: "[validmind](/validmind/validmind.qmd).HistogramPlot" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +## HistogramPlot + + + +::: {.signature} + +@tags('tabular_data', 'visualization', 'data_quality') + +@tasks('classification', 'regression', 'clustering') + +defHistogramPlot(dataset:validmind.vm_models.VMDataset,columns:Optional\[List\[str\]\]=None,bins:Union\[int, str, List\]=30,color:str='steelblue',opacity:float=0.7,show_kde:bool=True,normalize:bool=False,log_scale:bool=False,title_prefix:str='Histogram of',width:int=1200,height:int=800,n_cols:int=2,vertical_spacing:float=0.15,horizontal_spacing:float=0.1)go.Figure: + +::: + + + +Generates customizable histogram plots for numerical features in a dataset using Plotly. + +### Purpose + +This test provides a flexible way to visualize the distribution of numerical features in a dataset. It allows for extensive customization of the histogram appearance and behavior through parameters, making it suitable for various exploratory data analysis tasks. + +### Test Mechanism + +The test creates histogram plots for specified numerical columns (or all numerical columns if none specified). It supports various customization options including: + +- Number of bins or bin edges +- Color and opacity +- Kernel density estimation overlay +- Logarithmic scaling +- Normalization options +- Configurable subplot layout (columns and spacing) + +### Signs of High Risk + +- Highly skewed distributions that may indicate data quality issues +- Unexpected bimodal or multimodal distributions +- Presence of extreme outliers +- Empty or sparse distributions + +### Strengths + +- Highly customizable visualization options +- Interactive Plotly plots with zoom, pan, and hover capabilities +- Supports both single and multiple column analysis +- Provides insights into data distribution patterns +- Can handle different data types and scales +- Configurable subplot layout for better visualization + +### Limitations + +- Limited to numerical features only +- Visual interpretation may be subjective +- May not be suitable for high-dimensional datasets +- Performance may degrade with very large datasets diff --git a/docs/validmind/tests/plots/ViolinPlot.qmd b/docs/validmind/tests/plots/ViolinPlot.qmd new file mode 100644 index 000000000..892703424 --- /dev/null +++ b/docs/validmind/tests/plots/ViolinPlot.qmd @@ -0,0 +1,56 @@ +--- +title: "[validmind](/validmind/validmind.qmd).ViolinPlot" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +## ViolinPlot + + + +::: {.signature} + +@tags('tabular_data', 'visualization', 'distribution') + +@tasks('classification', 'regression', 'clustering') + +defViolinPlot(dataset:validmind.vm_models.VMDataset,columns:Optional\[List\[str\]\]=None,group_by:Optional\[str\]=None,width:int=800,height:int=600)validmind.vm_models.px.validmind.vm_models.violin: + +::: + + + +Generates interactive violin plots for numerical features using Plotly. + +### Purpose + +This test creates violin plots to visualize the distribution of numerical features, showing both the probability density and summary statistics. Violin plots combine aspects of box plots and kernel density estimation for rich distribution visualization. + +### Test Mechanism + +The test creates violin plots for specified numerical columns, with optional grouping by categorical variables. Each violin shows the distribution shape, quartiles, and median values. + +### Signs of High Risk + +- Multimodal distributions that might indicate mixed populations +- Highly skewed distributions suggesting data quality issues +- Large differences in distribution shapes across groups +- Unusual distribution patterns that contradict domain expectations + +### Strengths + +- Shows detailed distribution shape information +- Interactive Plotly visualization with hover details +- Effective for comparing distributions across groups +- Combines density estimation with quartile information + +### Limitations + +- Limited to numerical features only +- Requires sufficient data points for meaningful density estimation +- May not be suitable for discrete variables +- Can be misleading with very small sample sizes diff --git a/docs/validmind/tests/stats.qmd b/docs/validmind/tests/stats.qmd new file mode 100644 index 000000000..1a0c2b165 --- /dev/null +++ b/docs/validmind/tests/stats.qmd @@ -0,0 +1,12 @@ +--- +title: "[validmind](/validmind/validmind.qmd).stats" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + +- [CorrelationAnalysis](stats/CorrelationAnalysis.qmd) +- [DescriptiveStats](stats/DescriptiveStats.qmd) +- [NormalityTests](stats/NormalityTests.qmd) +- [OutlierDetection](stats/OutlierDetection.qmd) diff --git a/docs/validmind/tests/stats/CorrelationAnalysis.qmd b/docs/validmind/tests/stats/CorrelationAnalysis.qmd new file mode 100644 index 000000000..44fb90108 --- /dev/null +++ b/docs/validmind/tests/stats/CorrelationAnalysis.qmd @@ -0,0 +1,62 @@ +--- +title: "[validmind](/validmind/validmind.qmd).CorrelationAnalysis" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +## CorrelationAnalysis + + + +::: {.signature} + +@tags('tabular_data', 'statistics', 'correlation') + +@tasks('classification', 'regression', 'clustering') + +defCorrelationAnalysis(dataset:validmind.vm_models.VMDataset,columns:Optional\[List\[str\]\]=None,method:str='pearson',significance_level:float=0.05,min_correlation:float=0.1)Dict\[str, Any\]: + +::: + + + +Performs comprehensive correlation analysis with significance testing for numerical features. + +### Purpose + +This test conducts detailed correlation analysis between numerical features, including correlation coefficients, significance testing, and identification of significant relationships. It helps identify multicollinearity, feature relationships, and potential redundancies in the dataset. + +### Test Mechanism + +The test computes correlation coefficients using the specified method and performs statistical significance testing for each correlation pair. It provides: + +- Correlation matrix with significance indicators +- List of significant correlations above threshold +- Summary statistics about correlation patterns +- Identification of highly correlated feature pairs + +### Signs of High Risk + +- Very high correlations (>0.9) indicating potential multicollinearity +- Many significant correlations suggesting complex feature interactions +- Features with no significant correlations to others (potential isolation) +- Unexpected correlation patterns contradicting domain knowledge + +### Strengths + +- Provides statistical significance testing for correlations +- Supports multiple correlation methods (Pearson, Spearman, Kendall) +- Identifies potentially problematic high correlations +- Filters results by minimum correlation threshold +- Comprehensive summary of correlation patterns + +### Limitations + +- Limited to numerical features only +- Cannot detect non-linear relationships (except with Spearman) +- Significance testing assumes certain distributional properties +- Correlation does not imply causation diff --git a/docs/validmind/tests/stats/DescriptiveStats.qmd b/docs/validmind/tests/stats/DescriptiveStats.qmd new file mode 100644 index 000000000..ae29e79f9 --- /dev/null +++ b/docs/validmind/tests/stats/DescriptiveStats.qmd @@ -0,0 +1,64 @@ +--- +title: "[validmind](/validmind/validmind.qmd).DescriptiveStats" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +## DescriptiveStats + + + +::: {.signature} + +@tags('tabular_data', 'statistics', 'data_quality') + +@tasks('classification', 'regression', 'clustering') + +defDescriptiveStats(dataset:validmind.vm_models.VMDataset,columns:Optional\[List\[str\]\]=None,include_advanced:bool=True,confidence_level:float=0.95)Dict\[str, Any\]: + +::: + + + +Provides comprehensive descriptive statistics for numerical features in a dataset. + +### Purpose + +This test generates detailed descriptive statistics for numerical features, including basic statistics, distribution measures, confidence intervals, and normality tests. It provides a comprehensive overview of data characteristics essential for understanding data quality and distribution properties. + +### Test Mechanism + +The test computes various statistical measures for each numerical column: + +- Basic statistics: count, mean, median, std, min, max, quartiles +- Distribution measures: skewness, kurtosis, coefficient of variation +- Confidence intervals for the mean +- Normality tests (Shapiro-Wilk for small samples, Anderson-Darling for larger) +- Missing value analysis + +### Signs of High Risk + +- High skewness or kurtosis indicating non-normal distributions +- Large coefficients of variation suggesting high data variability +- Significant results in normality tests when normality is expected +- High percentage of missing values +- Extreme outliers based on IQR analysis + +### Strengths + +- Comprehensive statistical analysis in a single test +- Includes advanced statistical measures beyond basic descriptives +- Provides confidence intervals for uncertainty quantification +- Handles missing values appropriately +- Suitable for both exploratory and confirmatory analysis + +### Limitations + +- Limited to numerical features only +- Normality tests may not be meaningful for all data types +- Large datasets may make some tests computationally expensive +- Interpretation requires statistical knowledge diff --git a/docs/validmind/tests/stats/NormalityTests.qmd b/docs/validmind/tests/stats/NormalityTests.qmd new file mode 100644 index 000000000..67d6d04e1 --- /dev/null +++ b/docs/validmind/tests/stats/NormalityTests.qmd @@ -0,0 +1,57 @@ +--- +title: "[validmind](/validmind/validmind.qmd).NormalityTests" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +## NormalityTests + + + +::: {.signature} + +@tags('tabular_data', 'statistics', 'normality') + +@tasks('classification', 'regression', 'clustering') + +defNormalityTests(dataset:validmind.vm_models.VMDataset,columns:Optional\[List\[str\]\]=None,alpha:float=0.05,tests:List\[str\]=\['shapiro', 'anderson', 'kstest'\])Dict\[str, Any\]: + +::: + + + +Performs multiple normality tests on numerical features to assess distribution normality. + +### Purpose + +This test evaluates whether numerical features follow a normal distribution using various statistical tests. Understanding distribution normality is crucial for selecting appropriate statistical methods and model assumptions. + +### Test Mechanism + +The test applies multiple normality tests: + +- Shapiro-Wilk test: Best for small to medium samples +- Anderson-Darling test: More sensitive to deviations in tails +- Kolmogorov-Smirnov test: General goodness-of-fit test + +### Signs of High Risk + +- Multiple normality tests failing consistently +- Very low p-values indicating strong evidence against normality +- Conflicting results between different normality tests + +### Strengths + +- Multiple statistical tests for robust assessment +- Clear pass/fail indicators for each test +- Suitable for different sample sizes + +### Limitations + +- Limited to numerical features only +- Some tests sensitive to sample size +- Perfect normality is rare in real data diff --git a/docs/validmind/tests/stats/OutlierDetection.qmd b/docs/validmind/tests/stats/OutlierDetection.qmd new file mode 100644 index 000000000..b6c116708 --- /dev/null +++ b/docs/validmind/tests/stats/OutlierDetection.qmd @@ -0,0 +1,57 @@ +--- +title: "[validmind](/validmind/validmind.qmd).OutlierDetection" +sidebar: validmind-reference +toc-depth: 4 +toc-expand: 4 +# module.qmd.jinja2 +--- + + + +## OutlierDetection + + + +::: {.signature} + +@tags('tabular_data', 'statistics', 'outliers') + +@tasks('classification', 'regression', 'clustering') + +defOutlierDetection(dataset:validmind.vm_models.VMDataset,columns:Optional\[List\[str\]\]=None,methods:List\[str\]=\['iqr', 'zscore', 'isolation_forest'\],iqr_threshold:float=1.5,zscore_threshold:float=3.0,contamination:float=0.1)Dict\[str, Any\]: + +::: + + + +Detects outliers in numerical features using multiple statistical methods. + +### Purpose + +This test identifies outliers in numerical features using various statistical methods including IQR, Z-score, and Isolation Forest. It provides comprehensive outlier detection to help identify data quality issues and potential anomalies. + +### Test Mechanism + +The test applies multiple outlier detection methods: + +- IQR method: Values beyond Q1 - 1.5*IQR or Q3 + 1.5*IQR +- Z-score method: Values with |z-score| > threshold +- Isolation Forest: ML-based anomaly detection + +### Signs of High Risk + +- High percentage of outliers indicating data quality issues +- Inconsistent outlier detection across methods +- Extreme outliers that significantly deviate from normal patterns + +### Strengths + +- Multiple detection methods for robust outlier identification +- Customizable thresholds for different sensitivity levels +- Clear summary of outlier patterns across features + +### Limitations + +- Limited to numerical features only +- Some methods assume normal distributions +- Threshold selection can be subjective diff --git a/docs/validmind/version.qmd b/docs/validmind/version.qmd index 714ed3af5..d87cb2fbf 100644 --- a/docs/validmind/version.qmd +++ b/docs/validmind/version.qmd @@ -9,6 +9,6 @@ sidebar: validmind-reference ::: {.signature} -2.9.5 +2.10.0 :::