diff --git a/app/ui_tests/test_every_route.py b/app/ui_tests/test_every_route.py index 905adab8..b6aec388 100644 --- a/app/ui_tests/test_every_route.py +++ b/app/ui_tests/test_every_route.py @@ -244,7 +244,7 @@ def test_the_batch_judge_routes_judge_on_a_post(self): with patch.object(judge, "judge_trial", stub), patch.object(judge, "judge_side", stub), \ patch.object(judge, "judge_task", stub), patch.object(judge, "judge_control", stub), \ patch.object(judge, "tasks_todo", lambda j: list(j.trials)), \ - patch.object(judge, "_sides_todo", lambda view: [(t.task_base, job, t) for t in job.trials]), \ + patch.object(judge, "_sides_todo", lambda view: [(t.task_base, job, t, None) for t in job.trials]), \ patch.dict(os.environ, TYPESAFE_API_KEY="ts-fake"): with self.client.session_transaction() as sess: sess["csrf"] = "route-token" diff --git a/app/ui_tests/test_judge_other_workspace.py b/app/ui_tests/test_judge_other_workspace.py new file mode 100644 index 00000000..558349fc --- /dev/null +++ b/app/ui_tests/test_judge_other_workspace.py @@ -0,0 +1,72 @@ +"""A job of another workspace, opened with ``?workspace=``, shows and keeps its +judgements in that workspace's store, not the open one's.""" +from __future__ import annotations + +import os +import shutil +import tempfile +import unittest +from contextlib import ExitStack +from pathlib import Path +from unittest.mock import patch + +from flask import url_for + +from nullius import workspace +from nullius.judge import questions as Q, store + +from .test_routes import web + +JOB = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "harbor-jobs" / "liveboard-oracle-x4" +TRIAL = "hello-world__mAXMKXR" + + +class JudgementsOfAnotherWorkspace(unittest.TestCase): + def setUp(self): + stack = ExitStack() + self.addCleanup(stack.close) + tmp = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-judge-ws-"))) + shutil.copytree(JOB, tmp / "a" / "job-x") + shutil.copytree(JOB, tmp / "b" / "job-y") + stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces"))) + stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True)) + self.a = workspace.create(web.shared.WS_HOME, [str(tmp / "a")], "A") + self.b = workspace.create(web.shared.WS_HOME, [str(tmp / "b")], "B") + web.shared.open_workspace(self.a) + self.url = f"/trial/job-y/{TRIAL}?workspace={self.b.name}" + + def test_the_trial_page_reads_the_judgement_kept_in_its_own_workspace(self): + store.save(self.b.dir, "job-y", TRIAL, {"questions": Q.VERSION, "answers": {"q": {"choice": "x"}}, "judge": {"model": "m"}, + "asked_at": "2026-11-01T00:00:00Z"}) + seen = [] + real = web.traces._judgement_panel + with patch.object(web.traces, "_judgement_panel", side_effect=lambda *a: seen.append(real(*a)) or seen[-1]): + self.assertEqual(web.app.test_client().get(self.url).status_code, 200) + self.assertTrue(seen[0]["judged"], "B's judgement of B's trial is not shown") + + def test_judging_that_trial_writes_into_its_own_workspace(self): + rec = {"questions": Q.VERSION, "answers": {"q": {"choice": "x"}}, "judge": {"model": "m"}} + with web.app.test_request_context(self.url): + job, trial = web.traces._trial_evidence("job-y", TRIAL) + with patch.object(web.judge.batch, "ask_states", return_value=dict(rec)): + web.judge.batch.ask_and_store(web.judge.ws_dir(), job.job_id, trial.trial_name, {}, None) + self.assertNotIn("judgements", os.listdir(self.a.dir), "a judgement of B's trial was written into A") + self.assertIsNotNone(store.load(self.b.dir, "job-y", TRIAL)) + + def test_the_judge_forms_post_back_to_that_workspace(self): + with web.app.test_request_context(self.url): + for endpoint, args in (("judge_trial_route", {"trial_name": TRIAL}), ("judge_job_route", {}), ("judge_tasks_route", {})): + self.assertIn(f"workspace={self.b.name}", url_for(endpoint, job_id="job-y", **args)) + + def test_each_compare_column_reads_its_own_workspace(self): + with web.app.test_request_context(f"/compare?pick={self.a.name}:job-x&pick={self.b.name}:job-y"): + view = web.compare._comparison_requested() + self.assertEqual([web.judge.ws_dir(s.workspace) for s in view.selected], [self.a.dir, self.b.dir]) + + def test_a_job_of_the_open_workspace_keeps_its_store(self): + with web.app.test_request_context(f"/trial/job-x/{TRIAL}"): + self.assertEqual(web.judge.ws_dir(), self.a.dir) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/judge.py b/app/web/judge.py index 7edcae59..9c05b2dc 100644 --- a/app/web/judge.py +++ b/app/web/judge.py @@ -59,8 +59,11 @@ def one(item): return done, failed -def ws_dir(): - ws = shared.STATE.get("workspace") +def ws_dir(name=None): + """The judgement store of the workspace that holds the job on screen + (`shared._record_workspace`): a job opened from another workspace, or a + Compare column (`name`), keeps its judgements in its own.""" + ws = shared._record_workspace(name) return ws.dir if ws else None @@ -387,11 +390,10 @@ def side_ids(trial) -> list: return own + [q for q in Q.for_trial(trial.outcome == "resolved", trial.outcome) if q not in own] -def side_of(job, trial) -> dict | None: +def side_of(job, trial, d) -> dict | None: """One side's score as the compare page shows it, from the trial's own - current record: the expectation, the level it rounds to, the judge's + current record in its column's store `d`: the expectation, the level it rounds to, the judge's confidence, the provenance. None until scored under the current vocabulary.""" - d = ws_dir() rec = store.load(d, job.job_id, trial.trial_name, started=started_of(trial)) if d else None if not rec or not store.is_current(rec): return None @@ -406,8 +408,8 @@ def side_of(job, trial) -> dict | None: def pair_records(view) -> dict: """{task: record} for the pairs judged under v1 - the pair state, the - model's own pick - under job A in the open workspace. Read, never written.""" - d = ws_dir() + model's own pick - under job A in its column's workspace. Read, never written.""" + d = ws_dir(view.selected[0].workspace) if view.selected else None if d is None or len(view.jobs) != 2: return {} out = {} @@ -429,8 +431,9 @@ def compare_summary(view) -> dict | None: pairs = pairs_of(view) legacy = pair_records(view) rows, judged, earlier = {}, 0, 0 + da, db = (ws_dir(s.workspace) for s in view.selected) for task, a, b in pairs: - sa, sb = side_of(view.jobs[0], a), side_of(view.jobs[1], b) + sa, sb = side_of(view.jobs[0], a, da), side_of(view.jobs[1], b, db) if sa or sb: verdict = comparison.better_of(sa and sa["score"], sb and sb["score"]) rows[task] = {"a": sa, "b": sb, "verdict": verdict, "earlier": False} @@ -444,13 +447,13 @@ def compare_summary(view) -> dict | None: "margin": comparison.TIE_MARGIN, "question": SIDE_QUESTION} -def judge_side(job, trial, api_key: str, post=None, calibration=None) -> dict | None: +def judge_side(job, trial, d, api_key: str, post=None, calibration=None) -> dict | None: """Score one side: the questions it still lacks under the current - vocabulary, one call per state, merged over the trial's own record - - the same record the trial page shows. Nothing when it has them all.""" + vocabulary, one call per state, merged over the trial's own record in + its column's store `d` - the same record the trial page shows. Nothing + when it has them all.""" if not trial.has_trajectory: raise client.JudgeError(0, "this trial has no trajectory to judge") - d = ws_dir() rec = store.load(d, job.job_id, trial.trial_name, started=started_of(trial)) have = (rec.get("answers") or {}) if store.is_current(rec) else {} ids = [q for q in side_ids(trial) if q not in have] @@ -464,12 +467,13 @@ def judge_side(job, trial, api_key: str, post=None, calibration=None) -> dict | def _sides_todo(view) -> list: - """(job, trial) for every side of an unjudged pair that lacks its score.""" + """(task, job, trial, its column's store) for every side of an unjudged pair that lacks its score.""" out = [] + da, db = (ws_dir(s.workspace) for s in view.selected) for task, a, b in pairs_of(view): - for job, t in ((view.jobs[0], a), (view.jobs[1], b)): - if side_of(job, t) is None: - out.append((task, job, t)) + for job, t, d in ((view.jobs[0], a, da), (view.jobs[1], b, db)): + if side_of(job, t, d) is None: + out.append((task, job, t, d)) return out @@ -482,10 +486,10 @@ def judge_compare_route(): if len(view.jobs) != 2: abort(400, description="A judged comparison needs exactly two runs.") sides = _sides_todo(view) - todo = list(dict.fromkeys(task for task, _, _ in sides)) + todo = list(dict.fromkeys(s[0] for s in sides)) _guard() k, cal = key(), batch_calibration() - done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], k, calibration=cal), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})") + done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], s[3], k, calibration=cal), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})") from flask import flash flash(f"Judged {done} of {len(sides)} side(s) over {len(todo)} pair(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "") return redirect(url_for("compare", pick=view.references, cohort=view.cohort, view=compare_web._view()) + "#judgement") diff --git a/app/web/reconcile.py b/app/web/reconcile.py index e288d161..6d0d0f4b 100644 --- a/app/web/reconcile.py +++ b/app/web/reconcile.py @@ -14,11 +14,6 @@ from . import judge, shared -def ws_dir(): - ws = shared.STATE.get("workspace") - return ws.dir if ws else None - - def threshold_requested() -> float: """`?threshold=` as a number in 0..1, else the module's default.""" raw = request.args.get("threshold") @@ -30,7 +25,7 @@ def threshold_requested() -> float: def view_of(job, threshold: float) -> dict: - d = ws_dir() + d = judge.ws_dir() records = store.for_job(d, job.job_id, judge.started_by_name(job)) if d else {} return reconcile.reconcile(job, records, threshold) diff --git a/app/web/shared.py b/app/web/shared.py index a0575d98..337f3007 100644 --- a/app/web/shared.py +++ b/app/web/shared.py @@ -419,8 +419,11 @@ def _jobs_for_workspace(name=None): return jobs -def _record_workspace(): - name = request.args.get("workspace") if has_request_context() else None +def _record_workspace(name=None): + """The workspace that holds the record on screen: `name` ("" the open + one), else this request's `?workspace=`, else the open one.""" + if name is None: + name = request.args.get("workspace") if has_request_context() else None if not name: return STATE["workspace"] ws = workspace.load(WS_HOME, name) @@ -437,7 +440,8 @@ def _jobs_for_request(): def _preserve_record_workspace(endpoint, values): """Carry evidence provenance through links, forms, and lazy detail requests.""" record_endpoints = {"job", "trial", "save_trace", "artifact_page", "conversation_data", - "trace_span_data"} + "trace_span_data", "reconcile_page", "feasibility_page", "judge_trial_route", + "judge_job_route", "judge_tasks_route", "judge_feasibility_route"} if has_request_context() and endpoint in record_endpoints and "workspace" not in values: name = request.args.get("workspace") if name: