From 271c4073f455ec44bffc7ae5fdb1d841c3f49b2f Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 12:13:21 +0100 Subject: [PATCH] fix(judge): keep an earlier try's judgement when its retry is judged A retry runs an attempt again under the same trial name. store.save kept one record per question version, so judging the retry replaced the earlier try's record and the paid judgement was gone. The record judged last stays under its version, where every read finds it as before; an earlier try's moves beside it, under `@`. A read that names a start finds that try's record in either place, and judging an earlier try again merges into its own record. Files written before this change read as they are. --- app/nullius/judge/store.py | 20 +++++++++----- app/tests/test_judge_store_tries.py | 41 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 app/tests/test_judge_store_tries.py diff --git a/app/nullius/judge/store.py b/app/nullius/judge/store.py index a7d47365..1e8c1ba5 100644 --- a/app/nullius/judge/store.py +++ b/app/nullius/judge/store.py @@ -5,7 +5,9 @@ both, apart, ``lane``) - keyed by question version inside. Never under the job's ``agent/`` directory: the run tree is Harbor's, and a re-parse of the job does not touch this. -Every version is kept. A read without a version asks for the current one +Every version is kept, and under each every try of a retried trial: the +try judged last under the version, each earlier one under +``@``. A read without a version asks for the current one and falls back to the earlier ones the vocabulary still names (``questions.EARLIER``), newest first, so a judgement made under v1 is still shown after the bump - under its own version, which the record @@ -85,11 +87,16 @@ def _of(rec: Optional[dict], started: Optional[str]) -> bool: return not started or not rec or rec.get("trial_started_at") in (None, started) +def _try(version: str, started: str) -> str: + """Where a version keeps an earlier try's record, beside the latest try's under `version`.""" + return f"{version}@{started}" + + def _pick(data: dict, version: Optional[str], started: Optional[str] = None) -> Optional[dict]: for v in _versions(version): - rec = data.get(v) - if rec and _of(rec, started): - return rec + for rec in (data.get(v), started and data.get(_try(v, started))): + if rec and _of(rec, started): + return rec return None @@ -119,8 +126,9 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "d data = _read(paths) version = record.get("questions", Q.VERSION) earlier = data.get(version) or {} - if not _of(earlier, started): - earlier = {} # an earlier try's: none of its answers are this one's + if not _of(earlier, started): # another try's: kept beside, and none of its answers are this one's + data[_try(version, earlier["trial_started_at"])] = earlier + earlier = data.pop(_try(version, started), None) or {} merged = dict(record, **({"trial_started_at": started} if started else {})) merged["answers"] = {**(earlier.get("answers") or {}), **(record.get("answers") or {})} if "hygiene" in earlier or "hygiene" in record: diff --git a/app/tests/test_judge_store_tries.py b/app/tests/test_judge_store_tries.py new file mode 100644 index 00000000..1dd4a526 --- /dev/null +++ b/app/tests/test_judge_store_tries.py @@ -0,0 +1,41 @@ +"""A retry's judgement is kept beside the earlier try's, not over it.""" +from __future__ import annotations + +import shutil +import tempfile +import unittest + +from nullius.judge import questions as Q, store + + +def rec(choice, q="q"): + return {"questions": Q.VERSION, "answers": {q: {"choice": choice}}, "judge": {"model": "m"}} + + +class Tries(unittest.TestCase): + S1, S2 = "2026-01-01T00:00:00", "2026-01-01T01:00:00" + + def setUp(self): + self.ws = tempfile.mkdtemp(prefix="nullius-judge-tries-") + self.addCleanup(shutil.rmtree, self.ws, True) + + def test_judging_a_retry_keeps_the_earlier_try_judgement(self): + store.save(self.ws, "job", "t1", rec("first try"), started=self.S1) + store.save(self.ws, "job", "t1", rec("retry"), started=self.S2) + self.assertEqual(store.load(self.ws, "job", "t1", started=self.S2)["answers"]["q"]["choice"], "retry") + first = store.load(self.ws, "job", "t1", started=self.S1) + self.assertIsNotNone(first, "the first try's judgement is gone") + self.assertEqual(first["answers"]["q"]["choice"], "first try") + self.assertEqual(store.for_job(self.ws, "job", {"t1": self.S1})["t1"]["answers"]["q"]["choice"], "first try") + self.assertEqual(store.for_job(self.ws, "job", {"t1": self.S2})["t1"]["answers"]["q"]["choice"], "retry") + + def test_judging_an_earlier_try_again_merges_into_its_own_record(self): + store.save(self.ws, "job", "t1", rec("first try"), started=self.S1) + store.save(self.ws, "job", "t1", rec("retry"), started=self.S2) + store.save(self.ws, "job", "t1", rec("later", q="r"), started=self.S1) + self.assertEqual(sorted(store.load(self.ws, "job", "t1", started=self.S1)["answers"]), ["q", "r"]) + self.assertEqual(store.load(self.ws, "job", "t1", started=self.S2)["answers"], {"q": {"choice": "retry"}}) + + +if __name__ == "__main__": + unittest.main()