[SC-19086] Send each figure's position in its test result

tests/test_comparison_figures.py

@@ -0,0 +1,45 @@
+import json
+import unittest
+from types import SimpleNamespace
+
+import matplotlib.pyplot as plt
+
+from validmind.tests.comparison import _combine_figures
+from validmind.vm_models.figure import Figure
+from validmind.vm_models.result import TestResult
+
+
+class TestComparisonFigureVariants(unittest.TestCase):
+    def test_each_figure_sends_the_combination_that_made_it(self):
+        """Reordered combinations must not reuse another combination's figure"""
+        results = []
+        for dataset in ("train_ds", "test_ds"):
+            fig, _ = plt.subplots()
+            results.append(
+                TestResult(
+                    result_id="roc",
+                    inputs={"dataset": SimpleNamespace(input_id=dataset)},
+                    params={"threshold": 0.5},
+                    figures=[Figure(key=f"roc:{dataset}", figure=fig, ref_id="r")],
+                )
+            )
+
+        figures = _combine_figures(results, show_params=False)
+        variants = [
+            json.loads(json.loads(f.serialize()["metadata"])["_variant"])
+            for f in figures
+        ]
+
+        self.assertEqual(
+            variants,
+            [
+                {"inputs": {"dataset": "train_ds"}, "params": {"threshold": 0.5}},
+                {"inputs": {"dataset": "test_ds"}, "params": {"threshold": 0.5}},
+            ],
+        )
+        for figure in figures:
+            plt.close(figure.figure)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_results.py

@@ -1,4 +1,5 @@
 import asyncio
+import json
 import os
 import unittest
 from unittest.mock import patch
@@ -314,6 +315,57 @@ async def record(figure):
         self.assertEqual(cached_at_upload, [True])
         plt.close(fig)
 
+    def test_figure_serialize_omits_index_until_logged(self):
+        """A figure that was never logged has no position to send"""
+        fig, _ = plt.subplots()
+        figure = Figure(key="test_1:b2c4", figure=fig, ref_id="ref_1")
+
+        self.assertNotIn("_index", json.loads(figure.serialize()["metadata"]))
+        figure.index = 0
+        self.assertEqual(json.loads(figure.serialize()["metadata"])["_index"], 0)
+        plt.close(fig)
+
+    @patch("validmind.vm_models.result.result.update_metadata")
+    @patch("validmind.api_client.alog_test_result")
+    @patch("validmind.api_client.alog_figure")
+    def test_test_result_log_async_sends_each_figures_position(
+        self, mock_figure, mock_test_result, mock_update_metadata
+    ):
+        """Each figure sends its position, so a re-run can be matched to it"""
+        mock_test_result.return_value = {"cuid": "123"}
+        mock_update_metadata.return_value = None
+        sent = []
+
+        async def record(figure):
+            sent.append((figure.key, json.loads(figure.serialize()["metadata"])))
+            return {"cuid": "456"}
+
+        mock_figure.side_effect = record
+
+        figs = [plt.subplots()[0] for _ in range(2)]
+        test_result = TestResult(
+            result_id="test_1",
+            inputs={},
+            figures=[
+                Figure(key=f"test_1:{tag}", figure=fig, ref_id="ref_1")
+                for tag, fig in zip(("b2c4", "0a1f"), figs)
+            ],
+        )
+
+        # Sync test + asyncio.run: an async test method on a plain TestCase
+        # is never awaited, so it would pass without running.
+        asyncio.run(test_result.log_async(section_id="section_1", position=0))
+
+        self.assertEqual(
+            sorted(sent),
+            [
+                ("test_1:0a1f", {"_ref_id": "ref_1", "_index": 1}),
+                ("test_1:b2c4", {"_ref_id": "ref_1", "_index": 0}),
+            ],
+        )
+        for fig in figs:
+            plt.close(fig)
+
     def test_text_generation_result(self):
         """Test TextGenerationResult initialization and methods"""
         text_result = TextGenerationResult(

validmind/tests/comparison.py

@@ -2,6 +2,7 @@
 # Refer to the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import json
 from itertools import product
 from typing import Any, Dict, List, Tuple, Union
 
@@ -219,12 +220,27 @@ def _update_figure_title(figure: Any, input_param_str: str) -> None:
         raise ValueError(f"Unsupported figure type: {type(figure)}")
 
 
+def _variant_key(result: TestResult) -> str:
+    """The input and param combination a sub-result ran with, as a stable string."""
+    inputs = {
+        name: _get_input_key(input_obj)
+        for name, input_obj in (result.inputs or {}).items()
+    }
+    return json.dumps(
+        {"inputs": inputs, "params": result.params or {}}, sort_keys=True, default=str
+    )
+
+
 def _combine_figures(results: List[TestResult], show_params: bool) -> List[Any]:
     """Combine figures from multiple test results (gets raw figure objects, not vm Figures)"""
     combined_figures = []
 
     for result in results:
+        variant = _variant_key(result)
         for figure in result.figures or []:
+            # Positions shift when the combinations come in another order; the
+            # platform checks this too before it reuses a figure's position.
+            figure.variant = variant
             # update the figure object in-place with the new title
             _update_figure_title(
                 figure=figure.figure,

validmind/vm_models/figure.py

@@ -59,12 +59,20 @@ class Figure:
         title: Optional caption/title for the figure. When set, it is sent to
             the platform as ``metadata.caption`` and rendered by the document
             media registry as ``Figure N. <title>``.
+        index: The figure's position in its test result, set when the result
+            is logged and sent as ``metadata._index``. Keys change on every
+            run, so the platform uses it to find the same figure in a re-run.
+        variant: For a comparison test, the input and param combination that
+            made the figure, sent as ``metadata._variant``. The platform
+            refuses a figure at the same position from another combination.
     """
 
     key: str
     figure: Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget, bytes]
     ref_id: str  # used to link figures to results
     title: Optional[str] = None  # caption/title used by the document media registry
+    index: Optional[int] = None  # position in the test result, set when logged
+    variant: Optional[str] = None  # comparison tests: the inputs/params behind it
 
     _type: str = "plot"  # for now this is the only figure type
     _cached_png_bytes: bytes = None  # cached PNG bytes for async-safe serialization
@@ -142,6 +150,10 @@ def serialize(self):
         metadata = {"_ref_id": self.ref_id}
         if self.title:
             metadata["caption"] = self.title
+        if self.index is not None:
+            metadata["_index"] = self.index
+        if self.variant is not None:
+            metadata["_variant"] = self.variant
 
         return {
             "type": self._type,

validmind/vm_models/result/result.py

@@ -567,7 +567,10 @@ def _pre_serialize_figures(self):
         outright (ZD-626). log() calls this outside the loop; log_async() calls
         it too because the test-suite runner awaits log_async() directly.
         """
-        for figure in self.figures or []:
+        for index, figure in enumerate(self.figures or []):
+            # The platform matches a figure across runs by its position: the
+            # key changes on every run, and uploads run concurrently.
+            figure.index = index
             figure.pre_serialize()
 
     async def log_async(