[SC-19101] Fix list_tests crash on installs without optional extras

.github/workflows/dependency-testing.yaml

@@ -83,12 +83,11 @@ jobs:
             # Exercise the unbounded core dependency set separately from the
             # constrained optional extras (for example, latest scikit-learn).
             WHEEL=$(ls dist/*.whl | head -n 1)
-            # Jinja2 is currently imported by the library but is only present
-            # transitively through optional extras. Install it as test tooling
-            # so this leg can exercise the otherwise unbounded core set.
-            python -m pip install "${WHEEL}" jinja2
+            python -m pip install "${WHEEL}"
             pip check
             python -c 'import sklearn, validmind; print(f"ValidMind {validmind.__version__} with scikit-learn {sklearn.__version__}")'
+            # a filter forces list_tests to import every test, so an unguarded optional import fails here
+            python -c 'import validmind as vm; print(len(vm.tests.list_tests(filter="classification", pretty=False)), "classification tests")'
             python -m unittest \
               tests.unit_tests.model_validation.sklearn.test_ROCCurve \
               tests.unit_tests.model_validation.sklearn.test_PrecisionRecallCurve \

pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
   "aiohttp[speedups] (<3.13.1)",
   "requests (>=2.28.0,<3.0.0)",
   "ipywidgets",
+  "jinja2 (>=3.1.2)",
   "kaleido (>=1.2.0,<2.0.0)",
   "matplotlib",
   "mistune (>=3.0.2,<4.0.0)",

tests/test_validmind_tests_module.py

@@ -4,6 +4,7 @@
 
 import unittest
 from unittest import TestCase
+from unittest.mock import patch
 from typing import Callable, List
 
 import pandas as pd
@@ -85,6 +86,31 @@ def test_list_tests_filter_2(self):
         self.assertTrue(len(tests) == 1)
         self.assertTrue(tests[0].startswith("validmind.model_validation.ModelMetadata"))
 
+    def test_list_tests_skips_unguarded_missing_import(self):
+        from validmind.errors import LoadTestError
+        from validmind.tests import load as load_module
+
+        broken_id = "validmind.model_validation.ModelMetadata"
+        real_load_test = load_module.load_test
+
+        def fake_load_test(test_id, *args, **kwargs):
+            if test_id == broken_id:
+                raise LoadTestError(
+                    "Unable to load test",
+                    original_error=ModuleNotFoundError(
+                        "No module named 'fakepkg'", name="fakepkg"
+                    ),
+                )
+            return real_load_test(test_id, *args, **kwargs)
+
+        with patch.object(load_module, "load_test", fake_load_test):
+            with self.assertLogs(load_module.logger, level="WARNING") as logs:
+                tests = list_tests(filter="validmind.model_validation", pretty=False)
+
+        self.assertNotIn(broken_id, tests)
+        self.assertTrue(len(tests) > 0)
+        self.assertIn("`fakepkg` (1 tests)", "\n".join(logs.output))
+
     def test_list_tests_tasks(self):
         task = list_tasks()[0]
         tests = list_tests(task=task, pretty=False)

validmind/tests/load.py

@@ -248,11 +248,18 @@ def _list_test_ids() -> List[str]:
 def _load_tests(test_ids: List[str]) -> Dict[str, Callable[..., Any]]:
     """Load a set of tests, handling missing dependencies."""
     tests = {}
+    unguarded_missing = {}
 
     for test_id in test_ids:
         try:
             tests[test_id] = load_test(test_id)
         except LoadTestError as e:
+            # an optional import without a MissingDependencyError guard: skip the test
+            # instead of failing the whole listing, but warn so a broken import is visible
+            if isinstance(e.original_error, ModuleNotFoundError):
+                unguarded_missing.setdefault(e.original_error.name, []).append(test_id)
+                continue
+
             if not e.original_error or not isinstance(
                 e.original_error, MissingDependencyError
             ):
@@ -273,6 +280,16 @@ def _load_tests(test_ids: List[str]) -> Dict[str, Callable[..., Any]]:
                     " Please install the missing dependencies to view and run this test."
                 )
 
+    if unguarded_missing:
+        logger.warning(
+            "Skipping tests that import packages that aren't installed: "
+            + "; ".join(
+                f"`{module}` ({len(ids)} tests)"
+                for module, ids in sorted(unguarded_missing.items())
+            )
+        )
+        logger.debug(f"Skipped tests by missing package: {unguarded_missing}")
+
     return tests
 
 

validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py

@@ -6,14 +6,27 @@
 
 import pandas as pd
 import plotly.graph_objects as go
-from transformers import MarianMTModel, MarianTokenizer
 
 from validmind import RawData, tags, tasks
+from validmind.errors import MissingDependencyError
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
 from .utils import create_stability_analysis_result
 
+try:
+    from transformers import MarianMTModel, MarianTokenizer
+except ImportError as e:
+    if "transformers" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `transformers` for StabilityAnalysisTranslation. "
+            "Please run `pip install validmind[nlp]` to use NLP tests",
+            required_dependencies=["transformers"],
+            extra="nlp",
+        ) from e
+
+    raise e
+
 logger = get_logger(__name__)
 
 

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -7,7 +7,6 @@
 
 import plotly.express as px
 import plotly.graph_objects as go
-from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
@@ -16,14 +15,15 @@
 from .utils import get_ragas_config, get_renamed_columns
 
 try:
+    from datasets import Dataset
     from ragas import evaluate
     from ragas.metrics import AnswerCorrectness as answer_correctness
 except ImportError as e:
-    if "ragas" in str(e):
+    if "ragas" in str(e) or "datasets" in str(e):
         raise MissingDependencyError(
-            "Missing required package `ragas` for AnswerCorrectness. "
+            "Missing required packages `ragas` and `datasets` for AnswerCorrectness. "
             "Please run `pip install validmind[llm]` to use LLM tests",
-            required_dependencies=["ragas"],
+            required_dependencies=["ragas", "datasets"],
             extra="llm",
         ) from e
 

validmind/tests/model_validation/ragas/AspectCritic.py

@@ -7,7 +7,6 @@
 
 import plotly.express as px
 import plotly.graph_objects as go
-from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
@@ -16,6 +15,7 @@
 from .utils import get_ragas_config, get_renamed_columns
 
 try:
+    from datasets import Dataset
     from ragas import evaluate
     from ragas.metrics import AspectCritic as aspect_critic
     from ragas.metrics._aspect_critic import (
@@ -26,11 +26,11 @@
         maliciousness,
     )
 except ImportError as e:
-    if "ragas" in str(e):
+    if "ragas" in str(e) or "datasets" in str(e):
         raise MissingDependencyError(
-            "Missing required package `ragas` for AspectCritic. "
+            "Missing required packages `ragas` and `datasets` for AspectCritic. "
             "Please run `pip install validmind[llm]` to use LLM tests",
-            required_dependencies=["ragas"],
+            required_dependencies=["ragas", "datasets"],
             extra="llm",
         ) from e
 

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -7,7 +7,6 @@
 
 import plotly.express as px
 import plotly.graph_objects as go
-from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
@@ -16,14 +15,15 @@
 from .utils import get_ragas_config, get_renamed_columns
 
 try:
+    from datasets import Dataset
     from ragas import evaluate
     from ragas.metrics import ContextEntityRecall as context_entity_recall
 except ImportError as e:
-    if "ragas" in str(e):
+    if "ragas" in str(e) or "datasets" in str(e):
         raise MissingDependencyError(
-            "Missing required package `ragas` for ContextEntityRecall. "
+            "Missing required packages `ragas` and `datasets` for ContextEntityRecall. "
             "Please run `pip install validmind[llm]` to use LLM tests",
-            required_dependencies=["ragas"],
+            required_dependencies=["ragas", "datasets"],
             extra="llm",
         ) from e
 

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -7,7 +7,6 @@
 
 import plotly.express as px
 import plotly.graph_objects as go
-from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
@@ -16,14 +15,15 @@
 from .utils import get_ragas_config, get_renamed_columns
 
 try:
+    from datasets import Dataset
     from ragas import evaluate
     from ragas.metrics import LLMContextPrecisionWithReference as context_precision
 except ImportError as e:
-    if "ragas" in str(e):
+    if "ragas" in str(e) or "datasets" in str(e):
         raise MissingDependencyError(
-            "Missing required package `ragas` for ContextPrecision. "
+            "Missing required packages `ragas` and `datasets` for ContextPrecision. "
             "Please run `pip install validmind[llm]` to use LLM tests",
-            required_dependencies=["ragas"],
+            required_dependencies=["ragas", "datasets"],
             extra="llm",
         ) from e
 

validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py

@@ -7,7 +7,6 @@
 
 import plotly.express as px
 import plotly.graph_objects as go
-from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
@@ -16,14 +15,15 @@
 from .utils import get_ragas_config, get_renamed_columns
 
 try:
+    from datasets import Dataset
     from ragas import evaluate
     from ragas.metrics import LLMContextPrecisionWithoutReference as context_precision
 except ImportError as e:
-    if "ragas" in str(e):
+    if "ragas" in str(e) or "datasets" in str(e):
         raise MissingDependencyError(
-            "Missing required package `ragas` for ContextPrecision. "
+            "Missing required packages `ragas` and `datasets` for ContextPrecision. "
             "Please run `pip install validmind[llm]` to use LLM tests",
-            required_dependencies=["ragas"],
+            required_dependencies=["ragas", "datasets"],
             extra="llm",
         ) from e
 

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -7,7 +7,6 @@
 
 import plotly.express as px
 import plotly.graph_objects as go
-from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
@@ -16,14 +15,15 @@
 from .utils import get_ragas_config, get_renamed_columns
 
 try:
+    from datasets import Dataset
     from ragas import evaluate
     from ragas.metrics import LLMContextRecall as context_recall
 except ImportError as e:
-    if "ragas" in str(e):
+    if "ragas" in str(e) or "datasets" in str(e):
         raise MissingDependencyError(
-            "Missing required package `ragas` for ContextRecall. "
+            "Missing required packages `ragas` and `datasets` for ContextRecall. "
             "Please run `pip install validmind[llm]` to use LLM tests",
-            required_dependencies=["ragas"],
+            required_dependencies=["ragas", "datasets"],
             extra="llm",
         ) from e
 

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -7,7 +7,6 @@
 
 import plotly.express as px
 import plotly.graph_objects as go
-from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
@@ -16,14 +15,15 @@
 from .utils import get_ragas_config, get_renamed_columns
 
 try:
+    from datasets import Dataset
     from ragas import evaluate
     from ragas.metrics import Faithfulness as faithfulness
 except ImportError as e:
-    if "ragas" in str(e):
+    if "ragas" in str(e) or "datasets" in str(e):
         raise MissingDependencyError(
-            "Missing required package `ragas` for Faithfulness. "
+            "Missing required packages `ragas` and `datasets` for Faithfulness. "
             "Please run `pip install validmind[llm]` to use LLM tests",
-            required_dependencies=["ragas"],
+            required_dependencies=["ragas", "datasets"],
             extra="llm",
         ) from e
 

validmind/tests/model_validation/ragas/NoiseSensitivity.py

@@ -7,7 +7,6 @@
 
 import plotly.express as px
 import plotly.graph_objects as go
-from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
@@ -16,14 +15,15 @@
 from .utils import get_ragas_config, get_renamed_columns
 
 try:
+    from datasets import Dataset
     from ragas import evaluate
     from ragas.metrics import NoiseSensitivity as noise_sensitivity
 except ImportError as e:
-    if "ragas" in str(e):
+    if "ragas" in str(e) or "datasets" in str(e):
         raise MissingDependencyError(
-            "Missing required package `ragas` for NoiseSensitivity. "
+            "Missing required packages `ragas` and `datasets` for NoiseSensitivity. "
             "Please run `pip install validmind[llm]` to use LLM tests",
-            required_dependencies=["ragas"],
+            required_dependencies=["ragas", "datasets"],
             extra="llm",
         ) from e
 

validmind/tests/model_validation/ragas/ResponseRelevancy.py

@@ -7,7 +7,6 @@
 
 import plotly.express as px
 import plotly.graph_objects as go
-from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
@@ -16,14 +15,15 @@
 from .utils import get_ragas_config, get_renamed_columns
 
 try:
+    from datasets import Dataset
     from ragas import evaluate
     from ragas.metrics import ResponseRelevancy as response_relevancy
 except ImportError as e:
-    if "ragas" in str(e):
+    if "ragas" in str(e) or "datasets" in str(e):
         raise MissingDependencyError(
-            "Missing required package `ragas` for AnswerRelevance. "
+            "Missing required packages `ragas` and `datasets` for AnswerRelevance. "
             "Please run `pip install validmind[llm]` to use LLM tests",
-            required_dependencies=["ragas"],
+            required_dependencies=["ragas", "datasets"],
             extra="llm",
         ) from e