diff --git a/app/nullius/judge/batch.py b/app/nullius/judge/batch.py index 6410e3be..09aee66c 100644 --- a/app/nullius/judge/batch.py +++ b/app/nullius/judge/batch.py @@ -115,9 +115,9 @@ def ask_and_store(ws_dir: str, job_id: str, trial_name: str, states: dict, key, calibration = store.calibration(ws_dir) rec = ask_states(states, key, post=post, calibration=calibration) if rec is not None: - # a call over some of the kinds keeps the earlier readings for the others, as the answers are kept - lane = getattr(key, "lane", "decision") # the LLM judge's records apart from Decision's - earlier = store.load(ws_dir, job_id, trial_name, version=rec["questions"], lane=lane, started=started) or {} - rec["hygiene"] = {**(earlier.get("hygiene") or {}), **rec.get("hygiene", {})} - store.save(ws_dir, job_id, trial_name, rec, lane=lane, started=started) + # the LLM judge's records apart from Decision's; a call over some of the kinds keeps the + # earlier readings for the others, as the answers are kept (`store.save`), and returns them + lane = getattr(key, "lane", "decision") + rec.setdefault("hygiene", {}) + rec["hygiene"] = store.save(ws_dir, job_id, trial_name, rec, lane=lane, started=started)["hygiene"] return rec diff --git a/app/nullius/judge/store.py b/app/nullius/judge/store.py index fe35cb87..b6f1b232 100644 --- a/app/nullius/judge/store.py +++ b/app/nullius/judge/store.py @@ -14,6 +14,8 @@ import json import os import re +import tempfile +import threading from typing import Optional from . import client, questions as Q @@ -21,6 +23,15 @@ _SAFE = re.compile(r"[^A-Za-z0-9._@+-]") +_LOCKS: dict = {} # one per judgement file: a save reads, merges and writes it whole +_LOCKS_GUARD = threading.Lock() + + +def _lock(path: str) -> threading.Lock: + with _LOCKS_GUARD: + return _LOCKS.setdefault(path, threading.Lock()) + + ROOTS = {"decision": "judgements", "llm": "judgements-llm"} # one directory per judgement @@ -70,28 +81,37 @@ def is_current(record: Optional[dict]) -> bool: def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "decision", - started: Optional[str] = None) -> None: - """The record under its question version; other versions kept. A - record's answers merge over the version's earlier answers, so judging - one question later does not drop the others.""" + started: Optional[str] = None) -> dict: + """The record under its question version, returned as stored; other versions kept. A + record's answers and hygiene readings merge over the version's earlier + ones, so judging one question later does not drop the others. Saves to + one file take turns; each writes through a temporary file of its own.""" path = _path(ws_dir, job_id, trial_name, lane) os.makedirs(os.path.dirname(path), exist_ok=True) - try: - with open(path, encoding="utf-8") as fh: - data = json.load(fh) or {} - except (OSError, ValueError): - data = {} - version = record.get("questions", Q.VERSION) - earlier = data.get(version) or {} - if not _of(earlier, started): - earlier = {} # an earlier try's: none of its answers are this one's - merged = dict(record, **({"trial_started_at": started} if started else {})) - merged["answers"] = {**(earlier.get("answers") or {}), **(record.get("answers") or {})} - data[version] = merged - tmp = path + ".tmp" - with open(tmp, "w", encoding="utf-8") as fh: - json.dump(data, fh, indent=1) - os.replace(tmp, path) + with _lock(path): + try: + with open(path, encoding="utf-8") as fh: + data = json.load(fh) or {} + except (OSError, ValueError): + data = {} + version = record.get("questions", Q.VERSION) + earlier = data.get(version) or {} + if not _of(earlier, started): + earlier = {} # an earlier try's: none of its answers are this one's + merged = dict(record, **({"trial_started_at": started} if started else {})) + merged["answers"] = {**(earlier.get("answers") or {}), **(record.get("answers") or {})} + if "hygiene" in earlier or "hygiene" in record: + merged["hygiene"] = {**(earlier.get("hygiene") or {}), **(record.get("hygiene") or {})} + data[version] = merged + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix=os.path.basename(path) + ".", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + json.dump(data, fh, indent=1) + os.replace(tmp, path) + except BaseException: + os.unlink(tmp) + raise + return merged def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "decision", diff --git a/app/tests/test_judge_store_race.py b/app/tests/test_judge_store_race.py new file mode 100644 index 00000000..66cfda08 --- /dev/null +++ b/app/tests/test_judge_store_race.py @@ -0,0 +1,78 @@ +"""Two saves to one trial's judgement that overlap both land in the file.""" +from __future__ import annotations + +import json +import shutil +import tempfile +import threading +import unittest +from unittest import mock + +from nullius.judge import store + + +class OverlappingSaves(unittest.TestCase): + def test_two_overlapping_saves_keep_both_answers(self): + ws = tempfile.mkdtemp(prefix="nullius-judge-race-") + self.addCleanup(shutil.rmtree, ws, True) + both_read = threading.Barrier(2, timeout=2) + real_load = json.load + + def load_then_wait(fh, *a, **k): + data = real_load(fh, *a, **k) + try: + both_read.wait() # each save has read the file before either writes, unless a lock keeps them apart + except threading.BrokenBarrierError: + pass + return data + + store.save(ws, "job", "t1", {"questions": "v1", "answers": {}}) + errors = [] + + def save(question): + try: + store.save(ws, "job", "t1", {"questions": "v1", "answers": {question: "yes"}}) + except Exception as e: + errors.append(repr(e)) + + with mock.patch.object(store.json, "load", load_then_wait): + threads = [threading.Thread(target=save, args=(q,)) for q in ("q1", "q2")] + for t in threads: + t.start() + for t in threads: + t.join() + answers = store.load(ws, "job", "t1", "v1")["answers"] + self.assertEqual((errors, sorted(answers)), ([], ["q1", "q2"])) + + def test_saves_to_two_trials_do_not_wait_on_each_other(self): + ws = tempfile.mkdtemp(prefix="nullius-judge-race-") + self.addCleanup(shutil.rmtree, ws, True) + for t in ("t1", "t2"): + store.save(ws, "job", t, {"questions": "v1", "answers": {}}) + both_read = threading.Barrier(2, timeout=5) + real_load = json.load + + def load_then_wait(fh, *a, **k): + data = real_load(fh, *a, **k) + both_read.wait() # raises if one save holds the other back + return data + + errors = [] + + def save(trial): + try: + store.save(ws, "job", trial, {"questions": "v1", "answers": {"q": "yes"}}) + except Exception as e: + errors.append(repr(e)) + + with mock.patch.object(store.json, "load", load_then_wait): + threads = [threading.Thread(target=save, args=(t,)) for t in ("t1", "t2")] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual(errors, []) + + +if __name__ == "__main__": + unittest.main()