diff --git a/app/nullius/judge/store.py b/app/nullius/judge/store.py index a7d47365..1e8c1ba5 100644 --- a/app/nullius/judge/store.py +++ b/app/nullius/judge/store.py @@ -5,7 +5,9 @@ both, apart, ``lane``) - keyed by question version inside. Never under the job's ``agent/`` directory: the run tree is Harbor's, and a re-parse of the job does not touch this. -Every version is kept. A read without a version asks for the current one +Every version is kept, and under each every try of a retried trial: the +try judged last under the version, each earlier one under +``@``. A read without a version asks for the current one and falls back to the earlier ones the vocabulary still names (``questions.EARLIER``), newest first, so a judgement made under v1 is still shown after the bump - under its own version, which the record @@ -85,11 +87,16 @@ def _of(rec: Optional[dict], started: Optional[str]) -> bool: return not started or not rec or rec.get("trial_started_at") in (None, started) +def _try(version: str, started: str) -> str: + """Where a version keeps an earlier try's record, beside the latest try's under `version`.""" + return f"{version}@{started}" + + def _pick(data: dict, version: Optional[str], started: Optional[str] = None) -> Optional[dict]: for v in _versions(version): - rec = data.get(v) - if rec and _of(rec, started): - return rec + for rec in (data.get(v), started and data.get(_try(v, started))): + if rec and _of(rec, started): + return rec return None @@ -119,8 +126,9 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "d data = _read(paths) version = record.get("questions", Q.VERSION) earlier = data.get(version) or {} - if not _of(earlier, started): - earlier = {} # an earlier try's: none of its answers are this one's + if not _of(earlier, started): # another try's: kept beside, and none of its answers are this one's + data[_try(version, earlier["trial_started_at"])] = earlier + earlier = data.pop(_try(version, started), None) or {} merged = dict(record, **({"trial_started_at": started} if started else {})) merged["answers"] = {**(earlier.get("answers") or {}), **(record.get("answers") or {})} if "hygiene" in earlier or "hygiene" in record: diff --git a/app/tests/test_judge_store_tries.py b/app/tests/test_judge_store_tries.py new file mode 100644 index 00000000..1dd4a526 --- /dev/null +++ b/app/tests/test_judge_store_tries.py @@ -0,0 +1,41 @@ +"""A retry's judgement is kept beside the earlier try's, not over it.""" +from __future__ import annotations + +import shutil +import tempfile +import unittest + +from nullius.judge import questions as Q, store + + +def rec(choice, q="q"): + return {"questions": Q.VERSION, "answers": {q: {"choice": choice}}, "judge": {"model": "m"}} + + +class Tries(unittest.TestCase): + S1, S2 = "2026-01-01T00:00:00", "2026-01-01T01:00:00" + + def setUp(self): + self.ws = tempfile.mkdtemp(prefix="nullius-judge-tries-") + self.addCleanup(shutil.rmtree, self.ws, True) + + def test_judging_a_retry_keeps_the_earlier_try_judgement(self): + store.save(self.ws, "job", "t1", rec("first try"), started=self.S1) + store.save(self.ws, "job", "t1", rec("retry"), started=self.S2) + self.assertEqual(store.load(self.ws, "job", "t1", started=self.S2)["answers"]["q"]["choice"], "retry") + first = store.load(self.ws, "job", "t1", started=self.S1) + self.assertIsNotNone(first, "the first try's judgement is gone") + self.assertEqual(first["answers"]["q"]["choice"], "first try") + self.assertEqual(store.for_job(self.ws, "job", {"t1": self.S1})["t1"]["answers"]["q"]["choice"], "first try") + self.assertEqual(store.for_job(self.ws, "job", {"t1": self.S2})["t1"]["answers"]["q"]["choice"], "retry") + + def test_judging_an_earlier_try_again_merges_into_its_own_record(self): + store.save(self.ws, "job", "t1", rec("first try"), started=self.S1) + store.save(self.ws, "job", "t1", rec("retry"), started=self.S2) + store.save(self.ws, "job", "t1", rec("later", q="r"), started=self.S1) + self.assertEqual(sorted(store.load(self.ws, "job", "t1", started=self.S1)["answers"]), ["q", "r"]) + self.assertEqual(store.load(self.ws, "job", "t1", started=self.S2)["answers"], {"q": {"choice": "retry"}}) + + +if __name__ == "__main__": + unittest.main()