diff --git a/app/nullius/judge/questions.py b/app/nullius/judge/questions.py index 98a3c5ae..e4be0db1 100644 --- a/app/nullius/judge/questions.py +++ b/app/nullius/judge/questions.py @@ -251,7 +251,7 @@ def hygiene_ids() -> list: # --- task-instruction quality: the dataset's own text, no run involved ----------------- # A Score over the `task` state - the instruction alone. Stored once per -# dataset and task (`web.judge.task_key`), not per trial: an ambiguous +# dataset, task and instruction text (`web.judge.task_key`), not per trial: an ambiguous # instruction caps every agent on that task, which is a harness finding. register("task", { "instruction_quality": { diff --git a/app/ui_tests/test_task_judgement_slot.py b/app/ui_tests/test_task_judgement_slot.py new file mode 100644 index 00000000..7232058c --- /dev/null +++ b/app/ui_tests/test_task_judgement_slot.py @@ -0,0 +1,67 @@ +"""A task instruction's score belongs to the instruction it scored: a +changed instruction, or another dataset of the same folder name, is +judged on its own.""" +from __future__ import annotations + +import tempfile +import unittest +from contextlib import ExitStack +from pathlib import Path +from types import SimpleNamespace as NS +from unittest.mock import patch + +from nullius import workspace +from nullius.judge import questions as Q, store + +from .test_routes import web + +SCORED = {"questions": Q.VERSION, "judge": {"model": "m"}, + "answers": {"instruction_quality": {"score": 2, "legend": ["bad", "ok", "good"]}}} + + +class TaskSlot(unittest.TestCase): + def setUp(self): + stack = ExitStack() + self.addCleanup(stack.close) + tmp = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-task-slot-"))) + stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces"))) + stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True)) + web.shared.open_workspace(workspace.create(web.shared.WS_HOME, [], "w")) + self.instruction = "Fix the bug in parse.py." + stack.enter_context(patch.object(web.judge, "_task_instruction", lambda t: self.instruction)) + stack.enter_context(patch.object(web.judge.batch, "ask_states", return_value=dict(SCORED))) + + def job(self, job_id, dataset, path=""): + trial = NS(task_base="fix-bug", has_trajectory=True) + return NS(job_id=job_id, dataset=dataset, dataset_path=path, trials=[trial]), trial + + def test_a_changed_instruction_is_judged_again(self): + job, trial = self.job("run-1", "tasks") + web.judge.judge_task(job, trial, None) + self.instruction = "Fix the bug in render.py, and add a test." # the task was edited, same name + self.assertEqual(web.judge.tasks_todo(job), [trial], + "still scored: " + str(web.judge.task_quality_of(job, trial))) + + def test_two_datasets_with_one_folder_name_are_judged_apart(self): + a, ta = self.job("bench-a", "tasks", "/x/bench-a/tasks") # parse.py keeps the basename as the dataset + b, tb = self.job("bench-b", "tasks", "/y/bench-b/tasks") + web.judge.judge_task(a, ta, None) + self.assertIsNone(web.judge.task_quality_of(b, tb), "bench-b shows bench-a's score, slot " + web.judge.task_key(b)) + again, _ = self.job("run-2", "tasks", "/x/bench-a/tasks") + self.assertEqual(web.judge.task_quality_of(again, ta)["score"], 2) # another run of bench-a: judged once for both + + def test_two_jobs_of_no_dataset_are_judged_apart(self): + a, ta = self.job("run-1", "") + b, tb = self.job("run-2", "") + web.judge.judge_task(a, ta, None) + self.assertIsNone(web.judge.task_quality_of(b, tb)) + + def test_a_score_kept_without_its_instruction_is_not_shown(self): + job, trial = self.job("run-1", "tasks", "/x/bench-a/tasks") + store.save(web.judge.ws_dir(), web.judge.task_key(job), "fix-bug", dict(SCORED)) # as kept before the digest + self.assertIsNone(web.judge.task_quality_of(job, trial)) + self.assertEqual(web.judge.tasks_todo(job), [trial]) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/judge.py b/app/web/judge.py index 58f0c7fb..b7b06196 100644 --- a/app/web/judge.py +++ b/app/web/judge.py @@ -3,6 +3,8 @@ the recorded evidence, never in its place.""" from __future__ import annotations +import functools +import hashlib import os import threading from concurrent.futures import ThreadPoolExecutor @@ -558,17 +560,25 @@ def triage_summary(report, jobs) -> dict: "question": Q.QUESTIONS["control_failure_cause"]["instructions"] if "control_failure_cause" in Q.QUESTIONS else ""} -# -- task-instruction quality: once per dataset and task --------------------------------- +# -- task-instruction quality: once per dataset, task and instruction text --------------- def task_key(job) -> str: """The store's job slot for a dataset's task judgements: `task:`, - so an instruction is judged once for every run of that dataset.""" - return "task:" + (job.dataset or "unknown") + so an instruction is judged once for every run of that dataset. The + dataset is named by its recorded path, or its registry name, so two + datasets of one folder name keep apart; a job that names none has its own.""" + return "task:" + (job.dataset_path or job.dataset or "job:" + job.job_id) -def task_record(job, task_base: str) -> dict | None: +def task_slot(trial) -> str: + """A task's name in its dataset's slot, with a digest of the instruction + the judge reads: an edited instruction is a task not yet judged.""" + return trial.task_base + "@" + hashlib.sha256(_task_instruction(trial).encode("utf-8")).hexdigest()[:16] + + +def task_record(job, trial) -> dict | None: d = ws_dir() - rec = store.load(d, task_key(job), task_base) if d else None + rec = store.load(d, task_key(job), task_slot(trial)) if d else None return rec if store.is_current(rec) else None @@ -576,7 +586,7 @@ def task_record(job, task_base: str) -> dict | None: def task_quality_of(job, trial) -> dict | None: """The instruction's score for a trial's task: the expectation, the level it rounds to (display only), the judge's confidence; None until judged.""" - rec = task_record(job, trial.task_base) + rec = task_record(job, trial) a = (rec.get("answers") or {}).get("instruction_quality") if rec else None if not a or not isinstance(a.get("score"), (int, float)): return None @@ -602,7 +612,17 @@ def _task_instruction(trial) -> str: """The task's text as the agent received it, from the trial's trajectory.""" if not trial.has_trajectory: return "" - traj = trajectory.load(trial.trajectory_path) + try: + stamp = os.stat(trial.trajectory_path).st_mtime_ns + except OSError: + return "" + return _instruction_at(trial.trajectory_path, stamp) + + +@functools.lru_cache(maxsize=1024) +def _instruction_at(path: str, stamp: int) -> str: + """`_task_instruction` once per trajectory file and change: a job page asks it of every row.""" + traj = trajectory.load(path) return "" if traj.error else judge_state.instruction_of(traj) @@ -613,23 +633,23 @@ def tasks_todo(job) -> list: if t.task_base in seen: continue seen.add(t.task_base) - if task_record(job, t.task_base) is None and t.has_trajectory: + if t.has_trajectory and task_record(job, t) is None: out.append(t) return out def task_summary(job) -> dict: """For the job page: how many of the job's tasks have their instruction judged.""" - tasks = {t.task_base for t in job.trials} - judged = sum(1 for tb in tasks if task_record(job, tb) is not None) + tasks = {t.task_base: t for t in reversed(job.trials)} # each task by its first trial, as tasks_todo reads it + judged = sum(1 for t in tasks.values() if task_record(job, t) is not None) return {"tasks": len(tasks), "judged": judged, "unjudged": len(tasks_todo(job)), "keyed": bool(key())} def judge_task(job, trial, api_key: str, post=None) -> dict | None: """Score one task's instruction over the `task` state alone, stored under - the dataset's task slot - never under the trial.""" + the dataset's task slot and the instruction's digest - never under the trial.""" instruction = _task_instruction(trial) if not instruction: raise client.JudgeError(0, f"no instruction can be read for {trial.task_base}") states = batch.states_for(Q.group_ids("task"), lambda kind: judge_state.build(kind, None, instruction=instruction)) - return batch.ask_and_store(ws_dir(), task_key(job), trial.task_base, states, api_key, post=post) + return batch.ask_and_store(ws_dir(), task_key(job), task_slot(trial), states, api_key, post=post)