From 1d5da5127647dc62feedc80fcaf7dfee1eba7b21 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Wed, 21 Oct 2026 16:00:05 -0700 Subject: [PATCH 1/3] Send each figure's position in its test result Figure keys are ":<4 random chars>" and change on every run. They must stay unique per run, because the platform stores the image at figures//.png and a stable key would overwrite older runs' images. Uploads also run concurrently, so the platform cannot read an order from them. Each figure now sends metadata._index, its position in result.figures when the result is logged. The platform uses it to find the same figure after a re-run (live figure embeds in documents). Co-Authored-By: Claude Opus 5.5 --- tests/test_results.py | 42 ++++++++++++++++++++++++++++ validmind/vm_models/figure.py | 6 ++++ validmind/vm_models/result/result.py | 5 +++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/tests/test_results.py b/tests/test_results.py index e6716f7ce..743216f5e 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -1,4 +1,5 @@ import asyncio +import json import os import unittest from unittest.mock import patch @@ -314,6 +315,47 @@ async def record(figure): self.assertEqual(cached_at_upload, [True]) plt.close(fig) + @patch("validmind.vm_models.result.result.update_metadata") + @patch("validmind.api_client.alog_test_result") + @patch("validmind.api_client.alog_figure") + def test_test_result_log_async_sends_each_figures_position( + self, mock_figure, mock_test_result, mock_update_metadata + ): + """Each figure sends its position, so a re-run can be matched to it""" + mock_test_result.return_value = {"cuid": "123"} + mock_update_metadata.return_value = None + sent = [] + + async def record(figure): + sent.append((figure.key, json.loads(figure.serialize()["metadata"]))) + return {"cuid": "456"} + + mock_figure.side_effect = record + + figs = [plt.subplots()[0] for _ in range(2)] + test_result = TestResult( + result_id="test_1", + inputs={}, + figures=[ + Figure(key=f"test_1:{tag}", figure=fig, ref_id="ref_1") + for tag, fig in zip(("b2c4", "0a1f"), figs) + ], + ) + + # Sync test + asyncio.run: an async test method on a plain TestCase + # is never awaited, so it would pass without running. + asyncio.run(test_result.log_async(section_id="section_1", position=0)) + + self.assertEqual( + sorted(sent), + [ + ("test_1:0a1f", {"_ref_id": "ref_1", "_index": 1}), + ("test_1:b2c4", {"_ref_id": "ref_1", "_index": 0}), + ], + ) + for fig in figs: + plt.close(fig) + def test_text_generation_result(self): """Test TextGenerationResult initialization and methods""" text_result = TextGenerationResult( diff --git a/validmind/vm_models/figure.py b/validmind/vm_models/figure.py index e91886bc0..a9489c2cc 100644 --- a/validmind/vm_models/figure.py +++ b/validmind/vm_models/figure.py @@ -59,12 +59,16 @@ class Figure: title: Optional caption/title for the figure. When set, it is sent to the platform as ``metadata.caption`` and rendered by the document media registry as ``Figure N. ``. + index: The figure's position in its test result, set when the result + is logged and sent as ``metadata._index``. Keys change on every + run, so the platform uses it to find the same figure in a re-run. """ key: str figure: Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget, bytes] ref_id: str # used to link figures to results title: Optional[str] = None # caption/title used by the document media registry + index: Optional[int] = None # position in the test result, set when logged _type: str = "plot" # for now this is the only figure type _cached_png_bytes: bytes = None # cached PNG bytes for async-safe serialization @@ -142,6 +146,8 @@ def serialize(self): metadata = {"_ref_id": self.ref_id} if self.title: metadata["caption"] = self.title + if self.index is not None: + metadata["_index"] = self.index return { "type": self._type, diff --git a/validmind/vm_models/result/result.py b/validmind/vm_models/result/result.py index 6e6a0202d..317b81ddf 100644 --- a/validmind/vm_models/result/result.py +++ b/validmind/vm_models/result/result.py @@ -567,7 +567,10 @@ def _pre_serialize_figures(self): outright (ZD-626). log() calls this outside the loop; log_async() calls it too because the test-suite runner awaits log_async() directly. """ - for figure in self.figures or []: + for index, figure in enumerate(self.figures or []): + # The platform matches a figure across runs by its position: the + # key changes on every run, and uploads run concurrently. + figure.index = index figure.pre_serialize() async def log_async( From f933a2b616c93220cb42ee8c9c30298882e9f850 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez <andres@validmind.ai> Date: Thu, 22 Oct 2026 13:15:24 -0700 Subject: [PATCH 2/3] Test that a figure that was never logged sends no position Review feedback: pin the backward-compatible branch of Figure.serialize(), where index is None and metadata has no _index. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- tests/test_results.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_results.py b/tests/test_results.py index 743216f5e..876e149be 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -315,6 +315,16 @@ async def record(figure): self.assertEqual(cached_at_upload, [True]) plt.close(fig) + def test_figure_serialize_omits_index_until_logged(self): + """A figure that was never logged has no position to send""" + fig, _ = plt.subplots() + figure = Figure(key="test_1:b2c4", figure=fig, ref_id="ref_1") + + self.assertNotIn("_index", json.loads(figure.serialize()["metadata"])) + figure.index = 0 + self.assertEqual(json.loads(figure.serialize()["metadata"])["_index"], 0) + plt.close(fig) + @patch("validmind.vm_models.result.result.update_metadata") @patch("validmind.api_client.alog_test_result") @patch("validmind.api_client.alog_figure") From 2d1ef945f0f8fd930dde85d0de68df90ac642039 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez <andres@validmind.ai> Date: Thu, 22 Oct 2026 13:28:03 -0700 Subject: [PATCH 3/3] Send the input and param combination behind each comparison figure Review question: a figure's position is not a safe identity for comparison tests. They append each combination's figures in input order, so running ROC Curve on [train, test] and then on [test, train] keeps both positions but swaps the curves. Each figure of a comparison result now sends metadata._variant, its sub-result's inputs and params as sorted JSON. The platform takes a figure at the same position only from the same variant. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- tests/test_comparison_figures.py | 45 ++++++++++++++++++++++++++++++++ validmind/tests/comparison.py | 16 ++++++++++++ validmind/vm_models/figure.py | 6 +++++ 3 files changed, 67 insertions(+) create mode 100644 tests/test_comparison_figures.py diff --git a/tests/test_comparison_figures.py b/tests/test_comparison_figures.py new file mode 100644 index 000000000..2bb800a71 --- /dev/null +++ b/tests/test_comparison_figures.py @@ -0,0 +1,45 @@ +import json +import unittest +from types import SimpleNamespace + +import matplotlib.pyplot as plt + +from validmind.tests.comparison import _combine_figures +from validmind.vm_models.figure import Figure +from validmind.vm_models.result import TestResult + + +class TestComparisonFigureVariants(unittest.TestCase): + def test_each_figure_sends_the_combination_that_made_it(self): + """Reordered combinations must not reuse another combination's figure""" + results = [] + for dataset in ("train_ds", "test_ds"): + fig, _ = plt.subplots() + results.append( + TestResult( + result_id="roc", + inputs={"dataset": SimpleNamespace(input_id=dataset)}, + params={"threshold": 0.5}, + figures=[Figure(key=f"roc:{dataset}", figure=fig, ref_id="r")], + ) + ) + + figures = _combine_figures(results, show_params=False) + variants = [ + json.loads(json.loads(f.serialize()["metadata"])["_variant"]) + for f in figures + ] + + self.assertEqual( + variants, + [ + {"inputs": {"dataset": "train_ds"}, "params": {"threshold": 0.5}}, + {"inputs": {"dataset": "test_ds"}, "params": {"threshold": 0.5}}, + ], + ) + for figure in figures: + plt.close(figure.figure) + + +if __name__ == "__main__": + unittest.main() diff --git a/validmind/tests/comparison.py b/validmind/tests/comparison.py index da5090c37..98544f1fb 100644 --- a/validmind/tests/comparison.py +++ b/validmind/tests/comparison.py @@ -2,6 +2,7 @@ # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial +import json from itertools import product from typing import Any, Dict, List, Tuple, Union @@ -219,12 +220,27 @@ def _update_figure_title(figure: Any, input_param_str: str) -> None: raise ValueError(f"Unsupported figure type: {type(figure)}") +def _variant_key(result: TestResult) -> str: + """The input and param combination a sub-result ran with, as a stable string.""" + inputs = { + name: _get_input_key(input_obj) + for name, input_obj in (result.inputs or {}).items() + } + return json.dumps( + {"inputs": inputs, "params": result.params or {}}, sort_keys=True, default=str + ) + + def _combine_figures(results: List[TestResult], show_params: bool) -> List[Any]: """Combine figures from multiple test results (gets raw figure objects, not vm Figures)""" combined_figures = [] for result in results: + variant = _variant_key(result) for figure in result.figures or []: + # Positions shift when the combinations come in another order; the + # platform checks this too before it reuses a figure's position. + figure.variant = variant # update the figure object in-place with the new title _update_figure_title( figure=figure.figure, diff --git a/validmind/vm_models/figure.py b/validmind/vm_models/figure.py index a9489c2cc..97c41b032 100644 --- a/validmind/vm_models/figure.py +++ b/validmind/vm_models/figure.py @@ -62,6 +62,9 @@ class Figure: index: The figure's position in its test result, set when the result is logged and sent as ``metadata._index``. Keys change on every run, so the platform uses it to find the same figure in a re-run. + variant: For a comparison test, the input and param combination that + made the figure, sent as ``metadata._variant``. The platform + refuses a figure at the same position from another combination. """ key: str @@ -69,6 +72,7 @@ class Figure: ref_id: str # used to link figures to results title: Optional[str] = None # caption/title used by the document media registry index: Optional[int] = None # position in the test result, set when logged + variant: Optional[str] = None # comparison tests: the inputs/params behind it _type: str = "plot" # for now this is the only figure type _cached_png_bytes: bytes = None # cached PNG bytes for async-safe serialization @@ -148,6 +152,8 @@ def serialize(self): metadata["caption"] = self.title if self.index is not None: metadata["_index"] = self.index + if self.variant is not None: + metadata["_variant"] = self.variant return { "type": self._type,