From ede7d0ed8d2fd23005ca19f09a96a87f49506b4e Mon Sep 17 00:00:00 2001 From: John Walz Date: Mon, 4 Aug 2025 12:01:38 -0400 Subject: [PATCH 1/3] chore: remove use of genai route --- validmind/ai/utils.py | 25 +- validmind/api_client.py | 15 -- validmind/tests/__types__.py | 214 +----------------- .../sklearn/FeatureImportance.py | 6 +- .../sklearn/HyperParametersTuning.py | 4 +- .../sklearn/RobustnessDiagnosis.py | 6 +- 6 files changed, 11 insertions(+), 259 deletions(-) diff --git a/validmind/ai/utils.py b/validmind/ai/utils.py index 00a5c13c2..478176c39 100644 --- a/validmind/ai/utils.py +++ b/validmind/ai/utils.py @@ -83,28 +83,9 @@ def get_client_and_model(): logger.debug(f"Using Azure OpenAI {__model} for generating descriptions") else: - try: - # TODO: fix circular import - from ..api_client import get_ai_key, get_api_host - - response = get_ai_key() - __client = Client( - base_url=( - # TODO: improve this to be a bit more dynamic - "http://localhost:4000/genai" - if "localhost" in get_api_host() - else urljoin(get_api_host(), "/genai") - ), - api_key=response["key"], - ) - __model = "gpt-4o" # TODO: backend should tell us which model to use - logger.debug(f"Using ValidMind {__model} for generating descriptions") - except Exception as e: - logger.debug(f"Failed to get API key: {e}") - raise ValueError( - "OPENAI_API_KEY, AZURE_OPENAI_KEY must be set, or your account " - "must be setup to use ValidMind's LLM in order to use LLM features" - ) + raise ValueError( + "OPENAI_API_KEY, AZURE_OPENAI_KEY must be setup to use LLM features" + ) return __client, __model diff --git a/validmind/api_client.py b/validmind/api_client.py index 51e03d547..7bee2290b 100644 --- a/validmind/api_client.py +++ b/validmind/api_client.py @@ -525,21 +525,6 @@ def log_metric( ) -def get_ai_key() -> Dict[str, Any]: - """Calls the API to get an API key for our LLM proxy.""" - r = requests.get( - url=_get_url("ai/key"), - headers=_get_api_headers(), - ) - - if r.status_code != 200: - # TODO: improve error handling when there's no Open AI API or AI key available - # logger.error("Could not get AI key from ValidMind API") - raise_api_error(r.text) - - return r.json() - - def generate_test_result_description(test_result_data: Dict[str, Any]) -> str: r = requests.post( url=_get_url("ai/generate/test_result_description"), diff --git a/validmind/tests/__types__.py b/validmind/tests/__types__.py index 43346c41d..88575ebe2 100644 --- a/validmind/tests/__types__.py +++ b/validmind/tests/__types__.py @@ -1,215 +1,3 @@ -# Copyright © 2023-2024 ValidMind Inc. All rights reserved. -# See the LICENSE file in the root of this repository for details. -# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial - -"""Literal types for test IDs. - -This module is auto-generated by running `make generate-test-id-types`. -Should not be modified manually. -""" - from typing import Literal, Union -TestID = Union[ - Literal[ - "validmind.data_validation.ACFandPACFPlot", - "validmind.data_validation.ADF", - "validmind.data_validation.AutoAR", - "validmind.data_validation.AutoMA", - "validmind.data_validation.AutoStationarity", - "validmind.data_validation.BivariateScatterPlots", - "validmind.data_validation.BoxPierce", - "validmind.data_validation.ChiSquaredFeaturesTable", - "validmind.data_validation.ClassImbalance", - "validmind.data_validation.DatasetDescription", - "validmind.data_validation.DatasetSplit", - "validmind.data_validation.DescriptiveStatistics", - "validmind.data_validation.DickeyFullerGLS", - "validmind.data_validation.Duplicates", - "validmind.data_validation.EngleGrangerCoint", - "validmind.data_validation.FeatureTargetCorrelationPlot", - "validmind.data_validation.HighCardinality", - "validmind.data_validation.HighPearsonCorrelation", - "validmind.data_validation.IQROutliersBarPlot", - "validmind.data_validation.IQROutliersTable", - "validmind.data_validation.IsolationForestOutliers", - "validmind.data_validation.JarqueBera", - "validmind.data_validation.KPSS", - "validmind.data_validation.LJungBox", - "validmind.data_validation.LaggedCorrelationHeatmap", - "validmind.data_validation.MissingValues", - "validmind.data_validation.MissingValuesBarPlot", - "validmind.data_validation.MutualInformation", - "validmind.data_validation.PearsonCorrelationMatrix", - "validmind.data_validation.PhillipsPerronArch", - "validmind.data_validation.ProtectedClassesCombination", - "validmind.data_validation.ProtectedClassesDescription", - "validmind.data_validation.ProtectedClassesDisparity", - "validmind.data_validation.ProtectedClassesThresholdOptimizer", - "validmind.data_validation.RollingStatsPlot", - "validmind.data_validation.RunsTest", - "validmind.data_validation.ScatterPlot", - "validmind.data_validation.ScoreBandDefaultRates", - "validmind.data_validation.SeasonalDecompose", - "validmind.data_validation.ShapiroWilk", - "validmind.data_validation.Skewness", - "validmind.data_validation.SpreadPlot", - "validmind.data_validation.TabularCategoricalBarPlots", - "validmind.data_validation.TabularDateTimeHistograms", - "validmind.data_validation.TabularDescriptionTables", - "validmind.data_validation.TabularNumericalHistograms", - "validmind.data_validation.TargetRateBarPlots", - "validmind.data_validation.TimeSeriesDescription", - "validmind.data_validation.TimeSeriesDescriptiveStatistics", - "validmind.data_validation.TimeSeriesFrequency", - "validmind.data_validation.TimeSeriesHistogram", - "validmind.data_validation.TimeSeriesLinePlot", - "validmind.data_validation.TimeSeriesMissingValues", - "validmind.data_validation.TimeSeriesOutliers", - "validmind.data_validation.TooManyZeroValues", - "validmind.data_validation.UniqueRows", - "validmind.data_validation.WOEBinPlots", - "validmind.data_validation.WOEBinTable", - "validmind.data_validation.ZivotAndrewsArch", - "validmind.data_validation.nlp.CommonWords", - "validmind.data_validation.nlp.Hashtags", - "validmind.data_validation.nlp.LanguageDetection", - "validmind.data_validation.nlp.Mentions", - "validmind.data_validation.nlp.PolarityAndSubjectivity", - "validmind.data_validation.nlp.Punctuations", - "validmind.data_validation.nlp.Sentiment", - "validmind.data_validation.nlp.StopWords", - "validmind.data_validation.nlp.TextDescription", - "validmind.data_validation.nlp.Toxicity", - "validmind.model_validation.BertScore", - "validmind.model_validation.BleuScore", - "validmind.model_validation.ClusterSizeDistribution", - "validmind.model_validation.ContextualRecall", - "validmind.model_validation.FeaturesAUC", - "validmind.model_validation.MeteorScore", - "validmind.model_validation.ModelMetadata", - "validmind.model_validation.ModelPredictionResiduals", - "validmind.model_validation.RegardScore", - "validmind.model_validation.RegressionResidualsPlot", - "validmind.model_validation.RougeScore", - "validmind.model_validation.TimeSeriesPredictionWithCI", - "validmind.model_validation.TimeSeriesPredictionsPlot", - "validmind.model_validation.TimeSeriesR2SquareBySegments", - "validmind.model_validation.TokenDisparity", - "validmind.model_validation.ToxicityScore", - "validmind.model_validation.embeddings.ClusterDistribution", - "validmind.model_validation.embeddings.CosineSimilarityComparison", - "validmind.model_validation.embeddings.CosineSimilarityDistribution", - "validmind.model_validation.embeddings.CosineSimilarityHeatmap", - "validmind.model_validation.embeddings.DescriptiveAnalytics", - "validmind.model_validation.embeddings.EmbeddingsVisualization2D", - "validmind.model_validation.embeddings.EuclideanDistanceComparison", - "validmind.model_validation.embeddings.EuclideanDistanceHeatmap", - "validmind.model_validation.embeddings.PCAComponentsPairwisePlots", - "validmind.model_validation.embeddings.StabilityAnalysisKeyword", - "validmind.model_validation.embeddings.StabilityAnalysisRandomNoise", - "validmind.model_validation.embeddings.StabilityAnalysisSynonyms", - "validmind.model_validation.embeddings.StabilityAnalysisTranslation", - "validmind.model_validation.embeddings.TSNEComponentsPairwisePlots", - "validmind.model_validation.ragas.AnswerCorrectness", - "validmind.model_validation.ragas.AspectCritic", - "validmind.model_validation.ragas.ContextEntityRecall", - "validmind.model_validation.ragas.ContextPrecision", - "validmind.model_validation.ragas.ContextPrecisionWithoutReference", - "validmind.model_validation.ragas.ContextRecall", - "validmind.model_validation.ragas.Faithfulness", - "validmind.model_validation.ragas.NoiseSensitivity", - "validmind.model_validation.ragas.ResponseRelevancy", - "validmind.model_validation.ragas.SemanticSimilarity", - "validmind.model_validation.sklearn.AdjustedMutualInformation", - "validmind.model_validation.sklearn.AdjustedRandIndex", - "validmind.model_validation.sklearn.CalibrationCurve", - "validmind.model_validation.sklearn.ClassifierPerformance", - "validmind.model_validation.sklearn.ClassifierThresholdOptimization", - "validmind.model_validation.sklearn.ClusterCosineSimilarity", - "validmind.model_validation.sklearn.ClusterPerformanceMetrics", - "validmind.model_validation.sklearn.CompletenessScore", - "validmind.model_validation.sklearn.ConfusionMatrix", - "validmind.model_validation.sklearn.FeatureImportance", - "validmind.model_validation.sklearn.FowlkesMallowsScore", - "validmind.model_validation.sklearn.HomogeneityScore", - "validmind.model_validation.sklearn.HyperParametersTuning", - "validmind.model_validation.sklearn.KMeansClustersOptimization", - "validmind.model_validation.sklearn.MinimumAccuracy", - "validmind.model_validation.sklearn.MinimumF1Score", - "validmind.model_validation.sklearn.MinimumROCAUCScore", - "validmind.model_validation.sklearn.ModelParameters", - "validmind.model_validation.sklearn.ModelsPerformanceComparison", - "validmind.model_validation.sklearn.OverfitDiagnosis", - "validmind.model_validation.sklearn.PermutationFeatureImportance", - "validmind.model_validation.sklearn.PopulationStabilityIndex", - "validmind.model_validation.sklearn.PrecisionRecallCurve", - "validmind.model_validation.sklearn.ROCCurve", - "validmind.model_validation.sklearn.RegressionErrors", - "validmind.model_validation.sklearn.RegressionErrorsComparison", - "validmind.model_validation.sklearn.RegressionPerformance", - "validmind.model_validation.sklearn.RegressionR2Square", - "validmind.model_validation.sklearn.RegressionR2SquareComparison", - "validmind.model_validation.sklearn.RobustnessDiagnosis", - "validmind.model_validation.sklearn.SHAPGlobalImportance", - "validmind.model_validation.sklearn.ScoreProbabilityAlignment", - "validmind.model_validation.sklearn.SilhouettePlot", - "validmind.model_validation.sklearn.TrainingTestDegradation", - "validmind.model_validation.sklearn.VMeasure", - "validmind.model_validation.sklearn.WeakspotsDiagnosis", - "validmind.model_validation.statsmodels.AutoARIMA", - "validmind.model_validation.statsmodels.CumulativePredictionProbabilities", - "validmind.model_validation.statsmodels.DurbinWatsonTest", - "validmind.model_validation.statsmodels.GINITable", - "validmind.model_validation.statsmodels.KolmogorovSmirnov", - "validmind.model_validation.statsmodels.Lilliefors", - "validmind.model_validation.statsmodels.PredictionProbabilitiesHistogram", - "validmind.model_validation.statsmodels.RegressionCoeffs", - "validmind.model_validation.statsmodels.RegressionFeatureSignificance", - "validmind.model_validation.statsmodels.RegressionModelForecastPlot", - "validmind.model_validation.statsmodels.RegressionModelForecastPlotLevels", - "validmind.model_validation.statsmodels.RegressionModelSensitivityPlot", - "validmind.model_validation.statsmodels.RegressionModelSummary", - "validmind.model_validation.statsmodels.RegressionPermutationFeatureImportance", - "validmind.model_validation.statsmodels.ScorecardHistogram", - "validmind.ongoing_monitoring.CalibrationCurveDrift", - "validmind.ongoing_monitoring.ClassDiscriminationDrift", - "validmind.ongoing_monitoring.ClassImbalanceDrift", - "validmind.ongoing_monitoring.ClassificationAccuracyDrift", - "validmind.ongoing_monitoring.ConfusionMatrixDrift", - "validmind.ongoing_monitoring.CumulativePredictionProbabilitiesDrift", - "validmind.ongoing_monitoring.FeatureDrift", - "validmind.ongoing_monitoring.PredictionAcrossEachFeature", - "validmind.ongoing_monitoring.PredictionCorrelation", - "validmind.ongoing_monitoring.PredictionProbabilitiesHistogramDrift", - "validmind.ongoing_monitoring.PredictionQuantilesAcrossFeatures", - "validmind.ongoing_monitoring.ROCCurveDrift", - "validmind.ongoing_monitoring.ScoreBandsDrift", - "validmind.ongoing_monitoring.ScorecardHistogramDrift", - "validmind.ongoing_monitoring.TargetPredictionDistributionPlot", - "validmind.prompt_validation.Bias", - "validmind.prompt_validation.Clarity", - "validmind.prompt_validation.Conciseness", - "validmind.prompt_validation.Delimitation", - "validmind.prompt_validation.NegativeInstruction", - "validmind.prompt_validation.Robustness", - "validmind.prompt_validation.Specificity", - "validmind.unit_metrics.classification.Accuracy", - "validmind.unit_metrics.classification.F1", - "validmind.unit_metrics.classification.Precision", - "validmind.unit_metrics.classification.ROC_AUC", - "validmind.unit_metrics.classification.Recall", - "validmind.unit_metrics.regression.AdjustedRSquaredScore", - "validmind.unit_metrics.regression.GiniCoefficient", - "validmind.unit_metrics.regression.HuberLoss", - "validmind.unit_metrics.regression.KolmogorovSmirnovStatistic", - "validmind.unit_metrics.regression.MeanAbsoluteError", - "validmind.unit_metrics.regression.MeanAbsolutePercentageError", - "validmind.unit_metrics.regression.MeanBiasDeviation", - "validmind.unit_metrics.regression.MeanSquaredError", - "validmind.unit_metrics.regression.QuantileLoss", - "validmind.unit_metrics.regression.RSquaredScore", - "validmind.unit_metrics.regression.RootMeanSquaredError", - ], - str, -] +TestID = Union[Literal["test_id"], str] diff --git a/validmind/tests/model_validation/sklearn/FeatureImportance.py b/validmind/tests/model_validation/sklearn/FeatureImportance.py index 4dff302f4..3e8aafcc5 100644 --- a/validmind/tests/model_validation/sklearn/FeatureImportance.py +++ b/validmind/tests/model_validation/sklearn/FeatureImportance.py @@ -76,9 +76,9 @@ def FeatureImportance( for i in range(num_features): if i < len(top_features): - result[ - f"Feature {i + 1}" - ] = f"[{top_features[i][0]}; {top_features[i][1]:.4f}]" + result[f"Feature {i + 1}"] = ( + f"[{top_features[i][0]}; {top_features[i][1]:.4f}]" + ) else: result[f"Feature {i + 1}"] = None diff --git a/validmind/tests/model_validation/sklearn/HyperParametersTuning.py b/validmind/tests/model_validation/sklearn/HyperParametersTuning.py index 0b93decd9..201b6003a 100644 --- a/validmind/tests/model_validation/sklearn/HyperParametersTuning.py +++ b/validmind/tests/model_validation/sklearn/HyperParametersTuning.py @@ -23,9 +23,7 @@ def _get_metrics(scoring): return ( scoring if isinstance(scoring, list) - else list(scoring.keys()) - if isinstance(scoring, dict) - else [scoring] + else list(scoring.keys()) if isinstance(scoring, dict) else [scoring] ) diff --git a/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py b/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py index b688d68a1..8bfc1019d 100644 --- a/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +++ b/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py @@ -323,9 +323,9 @@ def RobustnessDiagnosis( model=model.input_id, ) # rename perturbation size for baseline - results_df.loc[ - results_df["Perturbation Size"] == 0.0, "Perturbation Size" - ] = "Baseline (0.0)" + results_df.loc[results_df["Perturbation Size"] == 0.0, "Perturbation Size"] = ( + "Baseline (0.0)" + ) return ( results_df, From 3d34826e62f453eba6622bbd407fad836d4e9aad Mon Sep 17 00:00:00 2001 From: John Walz Date: Tue, 5 Aug 2025 13:01:09 -0400 Subject: [PATCH 2/3] chore: add copyright and license information to __types__.py --- validmind/tests/__types__.py | 214 +++++++++++++++++- .../sklearn/FeatureImportance.py | 6 +- .../sklearn/HyperParametersTuning.py | 4 +- .../sklearn/RobustnessDiagnosis.py | 6 +- 4 files changed, 222 insertions(+), 8 deletions(-) diff --git a/validmind/tests/__types__.py b/validmind/tests/__types__.py index 88575ebe2..43346c41d 100644 --- a/validmind/tests/__types__.py +++ b/validmind/tests/__types__.py @@ -1,3 +1,215 @@ +# Copyright © 2023-2024 ValidMind Inc. All rights reserved. +# See the LICENSE file in the root of this repository for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial + +"""Literal types for test IDs. + +This module is auto-generated by running `make generate-test-id-types`. +Should not be modified manually. +""" + from typing import Literal, Union -TestID = Union[Literal["test_id"], str] +TestID = Union[ + Literal[ + "validmind.data_validation.ACFandPACFPlot", + "validmind.data_validation.ADF", + "validmind.data_validation.AutoAR", + "validmind.data_validation.AutoMA", + "validmind.data_validation.AutoStationarity", + "validmind.data_validation.BivariateScatterPlots", + "validmind.data_validation.BoxPierce", + "validmind.data_validation.ChiSquaredFeaturesTable", + "validmind.data_validation.ClassImbalance", + "validmind.data_validation.DatasetDescription", + "validmind.data_validation.DatasetSplit", + "validmind.data_validation.DescriptiveStatistics", + "validmind.data_validation.DickeyFullerGLS", + "validmind.data_validation.Duplicates", + "validmind.data_validation.EngleGrangerCoint", + "validmind.data_validation.FeatureTargetCorrelationPlot", + "validmind.data_validation.HighCardinality", + "validmind.data_validation.HighPearsonCorrelation", + "validmind.data_validation.IQROutliersBarPlot", + "validmind.data_validation.IQROutliersTable", + "validmind.data_validation.IsolationForestOutliers", + "validmind.data_validation.JarqueBera", + "validmind.data_validation.KPSS", + "validmind.data_validation.LJungBox", + "validmind.data_validation.LaggedCorrelationHeatmap", + "validmind.data_validation.MissingValues", + "validmind.data_validation.MissingValuesBarPlot", + "validmind.data_validation.MutualInformation", + "validmind.data_validation.PearsonCorrelationMatrix", + "validmind.data_validation.PhillipsPerronArch", + "validmind.data_validation.ProtectedClassesCombination", + "validmind.data_validation.ProtectedClassesDescription", + "validmind.data_validation.ProtectedClassesDisparity", + "validmind.data_validation.ProtectedClassesThresholdOptimizer", + "validmind.data_validation.RollingStatsPlot", + "validmind.data_validation.RunsTest", + "validmind.data_validation.ScatterPlot", + "validmind.data_validation.ScoreBandDefaultRates", + "validmind.data_validation.SeasonalDecompose", + "validmind.data_validation.ShapiroWilk", + "validmind.data_validation.Skewness", + "validmind.data_validation.SpreadPlot", + "validmind.data_validation.TabularCategoricalBarPlots", + "validmind.data_validation.TabularDateTimeHistograms", + "validmind.data_validation.TabularDescriptionTables", + "validmind.data_validation.TabularNumericalHistograms", + "validmind.data_validation.TargetRateBarPlots", + "validmind.data_validation.TimeSeriesDescription", + "validmind.data_validation.TimeSeriesDescriptiveStatistics", + "validmind.data_validation.TimeSeriesFrequency", + "validmind.data_validation.TimeSeriesHistogram", + "validmind.data_validation.TimeSeriesLinePlot", + "validmind.data_validation.TimeSeriesMissingValues", + "validmind.data_validation.TimeSeriesOutliers", + "validmind.data_validation.TooManyZeroValues", + "validmind.data_validation.UniqueRows", + "validmind.data_validation.WOEBinPlots", + "validmind.data_validation.WOEBinTable", + "validmind.data_validation.ZivotAndrewsArch", + "validmind.data_validation.nlp.CommonWords", + "validmind.data_validation.nlp.Hashtags", + "validmind.data_validation.nlp.LanguageDetection", + "validmind.data_validation.nlp.Mentions", + "validmind.data_validation.nlp.PolarityAndSubjectivity", + "validmind.data_validation.nlp.Punctuations", + "validmind.data_validation.nlp.Sentiment", + "validmind.data_validation.nlp.StopWords", + "validmind.data_validation.nlp.TextDescription", + "validmind.data_validation.nlp.Toxicity", + "validmind.model_validation.BertScore", + "validmind.model_validation.BleuScore", + "validmind.model_validation.ClusterSizeDistribution", + "validmind.model_validation.ContextualRecall", + "validmind.model_validation.FeaturesAUC", + "validmind.model_validation.MeteorScore", + "validmind.model_validation.ModelMetadata", + "validmind.model_validation.ModelPredictionResiduals", + "validmind.model_validation.RegardScore", + "validmind.model_validation.RegressionResidualsPlot", + "validmind.model_validation.RougeScore", + "validmind.model_validation.TimeSeriesPredictionWithCI", + "validmind.model_validation.TimeSeriesPredictionsPlot", + "validmind.model_validation.TimeSeriesR2SquareBySegments", + "validmind.model_validation.TokenDisparity", + "validmind.model_validation.ToxicityScore", + "validmind.model_validation.embeddings.ClusterDistribution", + "validmind.model_validation.embeddings.CosineSimilarityComparison", + "validmind.model_validation.embeddings.CosineSimilarityDistribution", + "validmind.model_validation.embeddings.CosineSimilarityHeatmap", + "validmind.model_validation.embeddings.DescriptiveAnalytics", + "validmind.model_validation.embeddings.EmbeddingsVisualization2D", + "validmind.model_validation.embeddings.EuclideanDistanceComparison", + "validmind.model_validation.embeddings.EuclideanDistanceHeatmap", + "validmind.model_validation.embeddings.PCAComponentsPairwisePlots", + "validmind.model_validation.embeddings.StabilityAnalysisKeyword", + "validmind.model_validation.embeddings.StabilityAnalysisRandomNoise", + "validmind.model_validation.embeddings.StabilityAnalysisSynonyms", + "validmind.model_validation.embeddings.StabilityAnalysisTranslation", + "validmind.model_validation.embeddings.TSNEComponentsPairwisePlots", + "validmind.model_validation.ragas.AnswerCorrectness", + "validmind.model_validation.ragas.AspectCritic", + "validmind.model_validation.ragas.ContextEntityRecall", + "validmind.model_validation.ragas.ContextPrecision", + "validmind.model_validation.ragas.ContextPrecisionWithoutReference", + "validmind.model_validation.ragas.ContextRecall", + "validmind.model_validation.ragas.Faithfulness", + "validmind.model_validation.ragas.NoiseSensitivity", + "validmind.model_validation.ragas.ResponseRelevancy", + "validmind.model_validation.ragas.SemanticSimilarity", + "validmind.model_validation.sklearn.AdjustedMutualInformation", + "validmind.model_validation.sklearn.AdjustedRandIndex", + "validmind.model_validation.sklearn.CalibrationCurve", + "validmind.model_validation.sklearn.ClassifierPerformance", + "validmind.model_validation.sklearn.ClassifierThresholdOptimization", + "validmind.model_validation.sklearn.ClusterCosineSimilarity", + "validmind.model_validation.sklearn.ClusterPerformanceMetrics", + "validmind.model_validation.sklearn.CompletenessScore", + "validmind.model_validation.sklearn.ConfusionMatrix", + "validmind.model_validation.sklearn.FeatureImportance", + "validmind.model_validation.sklearn.FowlkesMallowsScore", + "validmind.model_validation.sklearn.HomogeneityScore", + "validmind.model_validation.sklearn.HyperParametersTuning", + "validmind.model_validation.sklearn.KMeansClustersOptimization", + "validmind.model_validation.sklearn.MinimumAccuracy", + "validmind.model_validation.sklearn.MinimumF1Score", + "validmind.model_validation.sklearn.MinimumROCAUCScore", + "validmind.model_validation.sklearn.ModelParameters", + "validmind.model_validation.sklearn.ModelsPerformanceComparison", + "validmind.model_validation.sklearn.OverfitDiagnosis", + "validmind.model_validation.sklearn.PermutationFeatureImportance", + "validmind.model_validation.sklearn.PopulationStabilityIndex", + "validmind.model_validation.sklearn.PrecisionRecallCurve", + "validmind.model_validation.sklearn.ROCCurve", + "validmind.model_validation.sklearn.RegressionErrors", + "validmind.model_validation.sklearn.RegressionErrorsComparison", + "validmind.model_validation.sklearn.RegressionPerformance", + "validmind.model_validation.sklearn.RegressionR2Square", + "validmind.model_validation.sklearn.RegressionR2SquareComparison", + "validmind.model_validation.sklearn.RobustnessDiagnosis", + "validmind.model_validation.sklearn.SHAPGlobalImportance", + "validmind.model_validation.sklearn.ScoreProbabilityAlignment", + "validmind.model_validation.sklearn.SilhouettePlot", + "validmind.model_validation.sklearn.TrainingTestDegradation", + "validmind.model_validation.sklearn.VMeasure", + "validmind.model_validation.sklearn.WeakspotsDiagnosis", + "validmind.model_validation.statsmodels.AutoARIMA", + "validmind.model_validation.statsmodels.CumulativePredictionProbabilities", + "validmind.model_validation.statsmodels.DurbinWatsonTest", + "validmind.model_validation.statsmodels.GINITable", + "validmind.model_validation.statsmodels.KolmogorovSmirnov", + "validmind.model_validation.statsmodels.Lilliefors", + "validmind.model_validation.statsmodels.PredictionProbabilitiesHistogram", + "validmind.model_validation.statsmodels.RegressionCoeffs", + "validmind.model_validation.statsmodels.RegressionFeatureSignificance", + "validmind.model_validation.statsmodels.RegressionModelForecastPlot", + "validmind.model_validation.statsmodels.RegressionModelForecastPlotLevels", + "validmind.model_validation.statsmodels.RegressionModelSensitivityPlot", + "validmind.model_validation.statsmodels.RegressionModelSummary", + "validmind.model_validation.statsmodels.RegressionPermutationFeatureImportance", + "validmind.model_validation.statsmodels.ScorecardHistogram", + "validmind.ongoing_monitoring.CalibrationCurveDrift", + "validmind.ongoing_monitoring.ClassDiscriminationDrift", + "validmind.ongoing_monitoring.ClassImbalanceDrift", + "validmind.ongoing_monitoring.ClassificationAccuracyDrift", + "validmind.ongoing_monitoring.ConfusionMatrixDrift", + "validmind.ongoing_monitoring.CumulativePredictionProbabilitiesDrift", + "validmind.ongoing_monitoring.FeatureDrift", + "validmind.ongoing_monitoring.PredictionAcrossEachFeature", + "validmind.ongoing_monitoring.PredictionCorrelation", + "validmind.ongoing_monitoring.PredictionProbabilitiesHistogramDrift", + "validmind.ongoing_monitoring.PredictionQuantilesAcrossFeatures", + "validmind.ongoing_monitoring.ROCCurveDrift", + "validmind.ongoing_monitoring.ScoreBandsDrift", + "validmind.ongoing_monitoring.ScorecardHistogramDrift", + "validmind.ongoing_monitoring.TargetPredictionDistributionPlot", + "validmind.prompt_validation.Bias", + "validmind.prompt_validation.Clarity", + "validmind.prompt_validation.Conciseness", + "validmind.prompt_validation.Delimitation", + "validmind.prompt_validation.NegativeInstruction", + "validmind.prompt_validation.Robustness", + "validmind.prompt_validation.Specificity", + "validmind.unit_metrics.classification.Accuracy", + "validmind.unit_metrics.classification.F1", + "validmind.unit_metrics.classification.Precision", + "validmind.unit_metrics.classification.ROC_AUC", + "validmind.unit_metrics.classification.Recall", + "validmind.unit_metrics.regression.AdjustedRSquaredScore", + "validmind.unit_metrics.regression.GiniCoefficient", + "validmind.unit_metrics.regression.HuberLoss", + "validmind.unit_metrics.regression.KolmogorovSmirnovStatistic", + "validmind.unit_metrics.regression.MeanAbsoluteError", + "validmind.unit_metrics.regression.MeanAbsolutePercentageError", + "validmind.unit_metrics.regression.MeanBiasDeviation", + "validmind.unit_metrics.regression.MeanSquaredError", + "validmind.unit_metrics.regression.QuantileLoss", + "validmind.unit_metrics.regression.RSquaredScore", + "validmind.unit_metrics.regression.RootMeanSquaredError", + ], + str, +] diff --git a/validmind/tests/model_validation/sklearn/FeatureImportance.py b/validmind/tests/model_validation/sklearn/FeatureImportance.py index 3e8aafcc5..4dff302f4 100644 --- a/validmind/tests/model_validation/sklearn/FeatureImportance.py +++ b/validmind/tests/model_validation/sklearn/FeatureImportance.py @@ -76,9 +76,9 @@ def FeatureImportance( for i in range(num_features): if i < len(top_features): - result[f"Feature {i + 1}"] = ( - f"[{top_features[i][0]}; {top_features[i][1]:.4f}]" - ) + result[ + f"Feature {i + 1}" + ] = f"[{top_features[i][0]}; {top_features[i][1]:.4f}]" else: result[f"Feature {i + 1}"] = None diff --git a/validmind/tests/model_validation/sklearn/HyperParametersTuning.py b/validmind/tests/model_validation/sklearn/HyperParametersTuning.py index 201b6003a..0b93decd9 100644 --- a/validmind/tests/model_validation/sklearn/HyperParametersTuning.py +++ b/validmind/tests/model_validation/sklearn/HyperParametersTuning.py @@ -23,7 +23,9 @@ def _get_metrics(scoring): return ( scoring if isinstance(scoring, list) - else list(scoring.keys()) if isinstance(scoring, dict) else [scoring] + else list(scoring.keys()) + if isinstance(scoring, dict) + else [scoring] ) diff --git a/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py b/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py index 8bfc1019d..b688d68a1 100644 --- a/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +++ b/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py @@ -323,9 +323,9 @@ def RobustnessDiagnosis( model=model.input_id, ) # rename perturbation size for baseline - results_df.loc[results_df["Perturbation Size"] == 0.0, "Perturbation Size"] = ( - "Baseline (0.0)" - ) + results_df.loc[ + results_df["Perturbation Size"] == 0.0, "Perturbation Size" + ] = "Baseline (0.0)" return ( results_df, From 8a5f46f87c700355600cb3348b14e1c40cdecc7d Mon Sep 17 00:00:00 2001 From: John Walz Date: Tue, 5 Aug 2025 13:02:04 -0400 Subject: [PATCH 3/3] refactor: remove unused import from utils.py --- validmind/ai/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/validmind/ai/utils.py b/validmind/ai/utils.py index 478176c39..0e40d2ef5 100644 --- a/validmind/ai/utils.py +++ b/validmind/ai/utils.py @@ -3,9 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial import os -from urllib.parse import urljoin -from openai import AzureOpenAI, Client, OpenAI +from openai import AzureOpenAI, OpenAI from ..logging import get_logger from ..utils import md_to_html