Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions app/nullius/judge/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
both, apart, ``lane``) - keyed by question version inside. Never under the job's ``agent/`` directory: the
run tree is Harbor's, and a re-parse of the job does not touch this.

Every version is kept. A read without a version asks for the current one
Every version is kept, and under each every try of a retried trial: the
try judged last under the version, each earlier one under
``<version>@<its start>``. A read without a version asks for the current one
and falls back to the earlier ones the vocabulary still names
(``questions.EARLIER``), newest first, so a judgement made under v1 is
still shown after the bump - under its own version, which the record
Expand Down Expand Up @@ -85,11 +87,16 @@ def _of(rec: Optional[dict], started: Optional[str]) -> bool:
return not started or not rec or rec.get("trial_started_at") in (None, started)


def _try(version: str, started: str) -> str:
"""Where a version keeps an earlier try's record, beside the latest try's under `version`."""
return f"{version}@{started}"


def _pick(data: dict, version: Optional[str], started: Optional[str] = None) -> Optional[dict]:
for v in _versions(version):
rec = data.get(v)
if rec and _of(rec, started):
return rec
for rec in (data.get(v), started and data.get(_try(v, started))):
if rec and _of(rec, started):
return rec
return None


Expand Down Expand Up @@ -119,8 +126,9 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "d
data = _read(paths)
version = record.get("questions", Q.VERSION)
earlier = data.get(version) or {}
if not _of(earlier, started):
earlier = {} # an earlier try's: none of its answers are this one's
if not _of(earlier, started): # another try's: kept beside, and none of its answers are this one's
data[_try(version, earlier["trial_started_at"])] = earlier
earlier = data.pop(_try(version, started), None) or {}
merged = dict(record, **({"trial_started_at": started} if started else {}))
merged["answers"] = {**(earlier.get("answers") or {}), **(record.get("answers") or {})}
if "hygiene" in earlier or "hygiene" in record:
Expand Down
41 changes: 41 additions & 0 deletions app/tests/test_judge_store_tries.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
"""A retry's judgement is kept beside the earlier try's, not over it."""
from __future__ import annotations

import shutil
import tempfile
import unittest

from nullius.judge import questions as Q, store


def rec(choice, q="q"):
return {"questions": Q.VERSION, "answers": {q: {"choice": choice}}, "judge": {"model": "m"}}


class Tries(unittest.TestCase):
S1, S2 = "2026-01-01T00:00:00", "2026-01-01T01:00:00"

def setUp(self):
self.ws = tempfile.mkdtemp(prefix="nullius-judge-tries-")
self.addCleanup(shutil.rmtree, self.ws, True)

def test_judging_a_retry_keeps_the_earlier_try_judgement(self):
store.save(self.ws, "job", "t1", rec("first try"), started=self.S1)
store.save(self.ws, "job", "t1", rec("retry"), started=self.S2)
self.assertEqual(store.load(self.ws, "job", "t1", started=self.S2)["answers"]["q"]["choice"], "retry")
first = store.load(self.ws, "job", "t1", started=self.S1)
self.assertIsNotNone(first, "the first try's judgement is gone")
self.assertEqual(first["answers"]["q"]["choice"], "first try")
self.assertEqual(store.for_job(self.ws, "job", {"t1": self.S1})["t1"]["answers"]["q"]["choice"], "first try")
self.assertEqual(store.for_job(self.ws, "job", {"t1": self.S2})["t1"]["answers"]["q"]["choice"], "retry")

def test_judging_an_earlier_try_again_merges_into_its_own_record(self):
store.save(self.ws, "job", "t1", rec("first try"), started=self.S1)
store.save(self.ws, "job", "t1", rec("retry"), started=self.S2)
store.save(self.ws, "job", "t1", rec("later", q="r"), started=self.S1)
self.assertEqual(sorted(store.load(self.ws, "job", "t1", started=self.S1)["answers"]), ["q", "r"])
self.assertEqual(store.load(self.ws, "job", "t1", started=self.S2)["answers"], {"q": {"choice": "retry"}})


if __name__ == "__main__":
unittest.main()
Loading