From 36f5d6ded2526973c8216b829c3e7daa55f11425 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Thu, 22 Oct 2026 11:13:58 -0700 Subject: [PATCH 1/3] Guard optional imports so list_tests works on minimal installs list_tests() loads every test before filtering and only skips tests that raise MissingDependencyError. StabilityAnalysisTranslation (transformers), the ragas tests (datasets) and ragas/utils.py (langchain_core) imported optional packages outside a guard, so a plain ModuleNotFoundError aborted the whole listing on installs without the nlp/llm extras. Co-Authored-By: Claude Opus 5.5 --- .../embeddings/StabilityAnalysisTranslation.py | 15 ++++++++++++++- .../model_validation/ragas/AnswerCorrectness.py | 8 ++++---- .../tests/model_validation/ragas/AspectCritic.py | 8 ++++---- .../ragas/ContextEntityRecall.py | 8 ++++---- .../model_validation/ragas/ContextPrecision.py | 8 ++++---- .../ragas/ContextPrecisionWithoutReference.py | 8 ++++---- .../model_validation/ragas/ContextRecall.py | 8 ++++---- .../tests/model_validation/ragas/Faithfulness.py | 8 ++++---- .../model_validation/ragas/NoiseSensitivity.py | 8 ++++---- .../model_validation/ragas/ResponseRelevancy.py | 8 ++++---- .../model_validation/ragas/SemanticSimilarity.py | 8 ++++---- validmind/tests/model_validation/ragas/utils.py | 16 ++++++++++++++-- 12 files changed, 68 insertions(+), 43 deletions(-) diff --git a/validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py b/validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py index c9d4cf799..b1d9474a7 100644 --- a/validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +++ b/validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py @@ -6,14 +6,27 @@ import pandas as pd import plotly.graph_objects as go -from transformers import MarianMTModel, MarianTokenizer from validmind import RawData, tags, tasks +from validmind.errors import MissingDependencyError from validmind.logging import get_logger from validmind.vm_models import VMDataset, VMModel from .utils import create_stability_analysis_result +try: + from transformers import MarianMTModel, MarianTokenizer +except ImportError as e: + if "transformers" in str(e): + raise MissingDependencyError( + "Missing required package `transformers` for StabilityAnalysisTranslation. " + "Please run `pip install validmind[nlp]` to use NLP tests", + required_dependencies=["transformers"], + extra="nlp", + ) from e + + raise e + logger = get_logger(__name__) diff --git a/validmind/tests/model_validation/ragas/AnswerCorrectness.py b/validmind/tests/model_validation/ragas/AnswerCorrectness.py index 2081bce50..bc19e3721 100644 --- a/validmind/tests/model_validation/ragas/AnswerCorrectness.py +++ b/validmind/tests/model_validation/ragas/AnswerCorrectness.py @@ -7,7 +7,6 @@ import plotly.express as px import plotly.graph_objects as go -from datasets import Dataset from validmind import RawData, tags, tasks from validmind.errors import MissingDependencyError @@ -16,14 +15,15 @@ from .utils import get_ragas_config, get_renamed_columns try: + from datasets import Dataset from ragas import evaluate from ragas.metrics import AnswerCorrectness as answer_correctness except ImportError as e: - if "ragas" in str(e): + if "ragas" in str(e) or "datasets" in str(e): raise MissingDependencyError( - "Missing required package `ragas` for AnswerCorrectness. " + "Missing required packages `ragas` and `datasets` for AnswerCorrectness. " "Please run `pip install validmind[llm]` to use LLM tests", - required_dependencies=["ragas"], + required_dependencies=["ragas", "datasets"], extra="llm", ) from e diff --git a/validmind/tests/model_validation/ragas/AspectCritic.py b/validmind/tests/model_validation/ragas/AspectCritic.py index f52446101..4e23c035d 100644 --- a/validmind/tests/model_validation/ragas/AspectCritic.py +++ b/validmind/tests/model_validation/ragas/AspectCritic.py @@ -7,7 +7,6 @@ import plotly.express as px import plotly.graph_objects as go -from datasets import Dataset from validmind import RawData, tags, tasks from validmind.errors import MissingDependencyError @@ -16,6 +15,7 @@ from .utils import get_ragas_config, get_renamed_columns try: + from datasets import Dataset from ragas import evaluate from ragas.metrics import AspectCritic as aspect_critic from ragas.metrics._aspect_critic import ( @@ -26,11 +26,11 @@ maliciousness, ) except ImportError as e: - if "ragas" in str(e): + if "ragas" in str(e) or "datasets" in str(e): raise MissingDependencyError( - "Missing required package `ragas` for AspectCritic. " + "Missing required packages `ragas` and `datasets` for AspectCritic. " "Please run `pip install validmind[llm]` to use LLM tests", - required_dependencies=["ragas"], + required_dependencies=["ragas", "datasets"], extra="llm", ) from e diff --git a/validmind/tests/model_validation/ragas/ContextEntityRecall.py b/validmind/tests/model_validation/ragas/ContextEntityRecall.py index a95bed63f..6e9512d79 100644 --- a/validmind/tests/model_validation/ragas/ContextEntityRecall.py +++ b/validmind/tests/model_validation/ragas/ContextEntityRecall.py @@ -7,7 +7,6 @@ import plotly.express as px import plotly.graph_objects as go -from datasets import Dataset from validmind import RawData, tags, tasks from validmind.errors import MissingDependencyError @@ -16,14 +15,15 @@ from .utils import get_ragas_config, get_renamed_columns try: + from datasets import Dataset from ragas import evaluate from ragas.metrics import ContextEntityRecall as context_entity_recall except ImportError as e: - if "ragas" in str(e): + if "ragas" in str(e) or "datasets" in str(e): raise MissingDependencyError( - "Missing required package `ragas` for ContextEntityRecall. " + "Missing required packages `ragas` and `datasets` for ContextEntityRecall. " "Please run `pip install validmind[llm]` to use LLM tests", - required_dependencies=["ragas"], + required_dependencies=["ragas", "datasets"], extra="llm", ) from e diff --git a/validmind/tests/model_validation/ragas/ContextPrecision.py b/validmind/tests/model_validation/ragas/ContextPrecision.py index e6890916b..4a3c96506 100644 --- a/validmind/tests/model_validation/ragas/ContextPrecision.py +++ b/validmind/tests/model_validation/ragas/ContextPrecision.py @@ -7,7 +7,6 @@ import plotly.express as px import plotly.graph_objects as go -from datasets import Dataset from validmind import RawData, tags, tasks from validmind.errors import MissingDependencyError @@ -16,14 +15,15 @@ from .utils import get_ragas_config, get_renamed_columns try: + from datasets import Dataset from ragas import evaluate from ragas.metrics import LLMContextPrecisionWithReference as context_precision except ImportError as e: - if "ragas" in str(e): + if "ragas" in str(e) or "datasets" in str(e): raise MissingDependencyError( - "Missing required package `ragas` for ContextPrecision. " + "Missing required packages `ragas` and `datasets` for ContextPrecision. " "Please run `pip install validmind[llm]` to use LLM tests", - required_dependencies=["ragas"], + required_dependencies=["ragas", "datasets"], extra="llm", ) from e diff --git a/validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py b/validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py index 3b9d91715..b69fc46ac 100644 --- a/validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py +++ b/validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py @@ -7,7 +7,6 @@ import plotly.express as px import plotly.graph_objects as go -from datasets import Dataset from validmind import RawData, tags, tasks from validmind.errors import MissingDependencyError @@ -16,14 +15,15 @@ from .utils import get_ragas_config, get_renamed_columns try: + from datasets import Dataset from ragas import evaluate from ragas.metrics import LLMContextPrecisionWithoutReference as context_precision except ImportError as e: - if "ragas" in str(e): + if "ragas" in str(e) or "datasets" in str(e): raise MissingDependencyError( - "Missing required package `ragas` for ContextPrecision. " + "Missing required packages `ragas` and `datasets` for ContextPrecision. " "Please run `pip install validmind[llm]` to use LLM tests", - required_dependencies=["ragas"], + required_dependencies=["ragas", "datasets"], extra="llm", ) from e diff --git a/validmind/tests/model_validation/ragas/ContextRecall.py b/validmind/tests/model_validation/ragas/ContextRecall.py index 511fd7b89..90a9581aa 100644 --- a/validmind/tests/model_validation/ragas/ContextRecall.py +++ b/validmind/tests/model_validation/ragas/ContextRecall.py @@ -7,7 +7,6 @@ import plotly.express as px import plotly.graph_objects as go -from datasets import Dataset from validmind import RawData, tags, tasks from validmind.errors import MissingDependencyError @@ -16,14 +15,15 @@ from .utils import get_ragas_config, get_renamed_columns try: + from datasets import Dataset from ragas import evaluate from ragas.metrics import LLMContextRecall as context_recall except ImportError as e: - if "ragas" in str(e): + if "ragas" in str(e) or "datasets" in str(e): raise MissingDependencyError( - "Missing required package `ragas` for ContextRecall. " + "Missing required packages `ragas` and `datasets` for ContextRecall. " "Please run `pip install validmind[llm]` to use LLM tests", - required_dependencies=["ragas"], + required_dependencies=["ragas", "datasets"], extra="llm", ) from e diff --git a/validmind/tests/model_validation/ragas/Faithfulness.py b/validmind/tests/model_validation/ragas/Faithfulness.py index 6c25068b9..3649f2a13 100644 --- a/validmind/tests/model_validation/ragas/Faithfulness.py +++ b/validmind/tests/model_validation/ragas/Faithfulness.py @@ -7,7 +7,6 @@ import plotly.express as px import plotly.graph_objects as go -from datasets import Dataset from validmind import RawData, tags, tasks from validmind.errors import MissingDependencyError @@ -16,14 +15,15 @@ from .utils import get_ragas_config, get_renamed_columns try: + from datasets import Dataset from ragas import evaluate from ragas.metrics import Faithfulness as faithfulness except ImportError as e: - if "ragas" in str(e): + if "ragas" in str(e) or "datasets" in str(e): raise MissingDependencyError( - "Missing required package `ragas` for Faithfulness. " + "Missing required packages `ragas` and `datasets` for Faithfulness. " "Please run `pip install validmind[llm]` to use LLM tests", - required_dependencies=["ragas"], + required_dependencies=["ragas", "datasets"], extra="llm", ) from e diff --git a/validmind/tests/model_validation/ragas/NoiseSensitivity.py b/validmind/tests/model_validation/ragas/NoiseSensitivity.py index 56d9a4bb8..a6c0f943c 100644 --- a/validmind/tests/model_validation/ragas/NoiseSensitivity.py +++ b/validmind/tests/model_validation/ragas/NoiseSensitivity.py @@ -7,7 +7,6 @@ import plotly.express as px import plotly.graph_objects as go -from datasets import Dataset from validmind import RawData, tags, tasks from validmind.errors import MissingDependencyError @@ -16,14 +15,15 @@ from .utils import get_ragas_config, get_renamed_columns try: + from datasets import Dataset from ragas import evaluate from ragas.metrics import NoiseSensitivity as noise_sensitivity except ImportError as e: - if "ragas" in str(e): + if "ragas" in str(e) or "datasets" in str(e): raise MissingDependencyError( - "Missing required package `ragas` for NoiseSensitivity. " + "Missing required packages `ragas` and `datasets` for NoiseSensitivity. " "Please run `pip install validmind[llm]` to use LLM tests", - required_dependencies=["ragas"], + required_dependencies=["ragas", "datasets"], extra="llm", ) from e diff --git a/validmind/tests/model_validation/ragas/ResponseRelevancy.py b/validmind/tests/model_validation/ragas/ResponseRelevancy.py index a60a7b8e5..42d7bf04b 100644 --- a/validmind/tests/model_validation/ragas/ResponseRelevancy.py +++ b/validmind/tests/model_validation/ragas/ResponseRelevancy.py @@ -7,7 +7,6 @@ import plotly.express as px import plotly.graph_objects as go -from datasets import Dataset from validmind import RawData, tags, tasks from validmind.errors import MissingDependencyError @@ -16,14 +15,15 @@ from .utils import get_ragas_config, get_renamed_columns try: + from datasets import Dataset from ragas import evaluate from ragas.metrics import ResponseRelevancy as response_relevancy except ImportError as e: - if "ragas" in str(e): + if "ragas" in str(e) or "datasets" in str(e): raise MissingDependencyError( - "Missing required package `ragas` for AnswerRelevance. " + "Missing required packages `ragas` and `datasets` for AnswerRelevance. " "Please run `pip install validmind[llm]` to use LLM tests", - required_dependencies=["ragas"], + required_dependencies=["ragas", "datasets"], extra="llm", ) from e diff --git a/validmind/tests/model_validation/ragas/SemanticSimilarity.py b/validmind/tests/model_validation/ragas/SemanticSimilarity.py index 309331228..ed862406f 100644 --- a/validmind/tests/model_validation/ragas/SemanticSimilarity.py +++ b/validmind/tests/model_validation/ragas/SemanticSimilarity.py @@ -7,7 +7,6 @@ import plotly.express as px import plotly.graph_objects as go -from datasets import Dataset from validmind import RawData, tags, tasks from validmind.errors import MissingDependencyError @@ -16,14 +15,15 @@ from .utils import get_ragas_config, get_renamed_columns try: + from datasets import Dataset from ragas import evaluate from ragas.metrics import SemanticSimilarity as semantic_similarity except ImportError as e: - if "ragas" in str(e): + if "ragas" in str(e) or "datasets" in str(e): raise MissingDependencyError( - "Missing required package `ragas` for AnswerSimilarity. " + "Missing required packages `ragas` and `datasets` for AnswerSimilarity. " "Please run `pip install validmind[llm]` to use LLM tests", - required_dependencies=["ragas"], + required_dependencies=["ragas", "datasets"], extra="llm", ) from e diff --git a/validmind/tests/model_validation/ragas/utils.py b/validmind/tests/model_validation/ragas/utils.py index 182b5006a..52033dce5 100644 --- a/validmind/tests/model_validation/ragas/utils.py +++ b/validmind/tests/model_validation/ragas/utils.py @@ -2,9 +2,21 @@ # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial -from langchain_core.outputs import ChatGeneration - from validmind.ai.utils import get_judge_config +from validmind.errors import MissingDependencyError + +try: + from langchain_core.outputs import ChatGeneration +except ImportError as e: + if "langchain_core" in str(e): + raise MissingDependencyError( + "Missing required package `langchain-core` for ragas tests. " + "Please run `pip install validmind[llm]` to use LLM tests", + required_dependencies=["langchain-core"], + extra="llm", + ) from e + + raise e EMBEDDINGS_MODEL = "text-embedding-3-small" From 7028bf8c26ac208e372dd28335910425e13460b0 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Thu, 22 Oct 2026 13:17:03 -0700 Subject: [PATCH 2/3] Skip tests with unguarded missing imports in list_tests A fresh `validmind[xgboost]` install still crashed in list_tests() on 18 tests that import statsmodels or nltk without a guard. _load_tests now skips any test whose import raises ModuleNotFoundError and logs one warning naming the missing packages, so a broken import stays visible. Add a list_tests smoke check to the core CI leg, using a filter so every test is actually imported, plus a unit test for the fallback. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/dependency-testing.yaml | 2 ++ tests/test_validmind_tests_module.py | 26 +++++++++++++++++++++++ validmind/tests/load.py | 17 +++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/.github/workflows/dependency-testing.yaml b/.github/workflows/dependency-testing.yaml index b07ca4c9f..c4c8ceaf4 100644 --- a/.github/workflows/dependency-testing.yaml +++ b/.github/workflows/dependency-testing.yaml @@ -89,6 +89,8 @@ jobs: python -m pip install "${WHEEL}" jinja2 pip check python -c 'import sklearn, validmind; print(f"ValidMind {validmind.__version__} with scikit-learn {sklearn.__version__}")' + # a filter forces list_tests to import every test, so an unguarded optional import fails here + python -c 'import validmind as vm; print(len(vm.tests.list_tests(filter="classification", pretty=False)), "classification tests")' python -m unittest \ tests.unit_tests.model_validation.sklearn.test_ROCCurve \ tests.unit_tests.model_validation.sklearn.test_PrecisionRecallCurve \ diff --git a/tests/test_validmind_tests_module.py b/tests/test_validmind_tests_module.py index 4118f6449..89de9ec82 100644 --- a/tests/test_validmind_tests_module.py +++ b/tests/test_validmind_tests_module.py @@ -4,6 +4,7 @@ import unittest from unittest import TestCase +from unittest.mock import patch from typing import Callable, List import pandas as pd @@ -85,6 +86,31 @@ def test_list_tests_filter_2(self): self.assertTrue(len(tests) == 1) self.assertTrue(tests[0].startswith("validmind.model_validation.ModelMetadata")) + def test_list_tests_skips_unguarded_missing_import(self): + from validmind.errors import LoadTestError + from validmind.tests import load as load_module + + broken_id = "validmind.model_validation.ModelMetadata" + real_load_test = load_module.load_test + + def fake_load_test(test_id, *args, **kwargs): + if test_id == broken_id: + raise LoadTestError( + "Unable to load test", + original_error=ModuleNotFoundError( + "No module named 'fakepkg'", name="fakepkg" + ), + ) + return real_load_test(test_id, *args, **kwargs) + + with patch.object(load_module, "load_test", fake_load_test): + with self.assertLogs(load_module.logger, level="WARNING") as logs: + tests = list_tests(filter="validmind.model_validation", pretty=False) + + self.assertNotIn(broken_id, tests) + self.assertTrue(len(tests) > 0) + self.assertIn("`fakepkg` (1 tests)", "\n".join(logs.output)) + def test_list_tests_tasks(self): task = list_tasks()[0] tests = list_tests(task=task, pretty=False) diff --git a/validmind/tests/load.py b/validmind/tests/load.py index f316ab99e..aeb8a9b85 100644 --- a/validmind/tests/load.py +++ b/validmind/tests/load.py @@ -248,11 +248,18 @@ def _list_test_ids() -> List[str]: def _load_tests(test_ids: List[str]) -> Dict[str, Callable[..., Any]]: """Load a set of tests, handling missing dependencies.""" tests = {} + unguarded_missing = {} for test_id in test_ids: try: tests[test_id] = load_test(test_id) except LoadTestError as e: + # an optional import without a MissingDependencyError guard: skip the test + # instead of failing the whole listing, but warn so a broken import is visible + if isinstance(e.original_error, ModuleNotFoundError): + unguarded_missing.setdefault(e.original_error.name, []).append(test_id) + continue + if not e.original_error or not isinstance( e.original_error, MissingDependencyError ): @@ -273,6 +280,16 @@ def _load_tests(test_ids: List[str]) -> Dict[str, Callable[..., Any]]: " Please install the missing dependencies to view and run this test." ) + if unguarded_missing: + logger.warning( + "Skipping tests that import packages that aren't installed: " + + "; ".join( + f"`{module}` ({len(ids)} tests)" + for module, ids in sorted(unguarded_missing.items()) + ) + ) + logger.debug(f"Skipped tests by missing package: {unguarded_missing}") + return tests From 9958621fa4ed23e53adbdc203584746526153f49 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Thu, 22 Oct 2026 13:27:03 -0700 Subject: [PATCH 3/3] Declare jinja2 as a core dependency `import validmind` needs jinja2: vm_models/result/utils.py imports it directly and pandas' Styler (format_dataframe, Styler test outputs) requires it. It was only installed transitively through optional extras, so a plain `pip install validmind` failed to import outside Jupyter. Drop the core CI leg's manual jinja2 install so that leg exercises the bare wheel. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/dependency-testing.yaml | 5 +---- pyproject.toml | 1 + uv.lock | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dependency-testing.yaml b/.github/workflows/dependency-testing.yaml index c4c8ceaf4..aa10746bd 100644 --- a/.github/workflows/dependency-testing.yaml +++ b/.github/workflows/dependency-testing.yaml @@ -83,10 +83,7 @@ jobs: # Exercise the unbounded core dependency set separately from the # constrained optional extras (for example, latest scikit-learn). WHEEL=$(ls dist/*.whl | head -n 1) - # Jinja2 is currently imported by the library but is only present - # transitively through optional extras. Install it as test tooling - # so this leg can exercise the otherwise unbounded core set. - python -m pip install "${WHEEL}" jinja2 + python -m pip install "${WHEEL}" pip check python -c 'import sklearn, validmind; print(f"ValidMind {validmind.__version__} with scikit-learn {sklearn.__version__}")' # a filter forces list_tests to import every test, so an unguarded optional import fails here diff --git a/pyproject.toml b/pyproject.toml index 9911ee78f..e784ac494 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "aiohttp[speedups] (<3.13.1)", "requests (>=2.28.0,<3.0.0)", "ipywidgets", + "jinja2 (>=3.1.2)", "kaleido (>=1.2.0,<2.0.0)", "matplotlib", "mistune (>=3.0.2,<4.0.0)", diff --git a/uv.lock b/uv.lock index 747083579..ee84879b0 100644 --- a/uv.lock +++ b/uv.lock @@ -11244,6 +11244,7 @@ dependencies = [ { name = "anywidget", version = "0.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "beautifulsoup4" }, { name = "ipywidgets" }, + { name = "jinja2" }, { name = "kaleido" }, { name = "matplotlib", version = "3.9.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "matplotlib", version = "3.10.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -11438,6 +11439,7 @@ requires-dist = [ { name = "google-genai", marker = "extra == 'all'" }, { name = "google-genai", marker = "extra == 'llm'" }, { name = "ipywidgets" }, + { name = "jinja2", specifier = ">=3.1.2" }, { name = "kaleido", specifier = ">=1.2.0,<2.0.0" }, { name = "langchain", marker = "extra == 'all'", specifier = ">=0.2.0,<0.4.0" }, { name = "langchain", marker = "extra == 'llm'", specifier = ">=0.2.0,<0.4.0" },