From 6855ee199e883207e4a2bfdf985d36daa37e981a Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Wed, 18 Nov 2026 14:49:28 +0100 Subject: [PATCH 1/8] perf(runs): check an ended job by its own records, not trial by trial A job whose result records a finish with nothing running or pending is read from the cache while its result.json, config.json and folder mtime are unchanged, without stating every trial's files. A resume rewrites result.json and a trial folder added or removed moves the folder, so both are still parsed again; a running job keeps the full signature. Refresh reads every job again. --- app/nullius/workspace.py | 99 ++++++++++++++++++++++++-------- app/tests/test_standard_shard.py | 3 +- app/tests/test_workspace.py | 66 +++++++++++++++++++++ 3 files changed, 143 insertions(+), 25 deletions(-) diff --git a/app/nullius/workspace.py b/app/nullius/workspace.py index 2020aa5f..53137e23 100644 --- a/app/nullius/workspace.py +++ b/app/nullius/workspace.py @@ -19,6 +19,13 @@ still going changes its files, the signature moves, the job is parsed again. A cached job that was parsed by an older version of ``parse`` is refreshed too — the signature carries ``parse.SCHEMA`` so a change there invalidates. + +A job that has ended (``_ended``: its result records a finish and nothing +running or pending) is checked by its *head* alone: its own result.json and +config.json, and its folder, whose mtime moves when a trial folder comes or +goes. A Harbor resume rewrites result.json, so it is caught; a file changed +inside a trial of an ended job and nothing else is not. The workspace's +Refresh (``clear_cache``) reads every job again. """ from __future__ import annotations @@ -311,6 +318,25 @@ def stat(path): return hashlib.sha256("\n".join(parts).encode()).hexdigest(), trials +def _head(job_dir: str) -> str: + """What an ended job's parse can change by without a trial changing in place: + its own records, and its folder's mtime (a trial folder added or removed).""" + parts = [f"schema={SCHEMA}/{parse.SCHEMA}"] + for path in (job_dir, os.path.join(job_dir, "result.json"), os.path.join(job_dir, "config.json")): + try: + st = os.stat(path) + except OSError: + parts.append("-") + continue + parts.append(f"{st.st_size}:{st.st_mtime_ns}") + return "\n".join(parts) + + +def _ended(job: parse.Job) -> bool: + """Its result records a finish and nothing running or pending: its trials are final.""" + return job.finished_at is not None and job.state == "finished" + + def _encode(obj): if isinstance(obj, datetime): return {"$dt": obj.isoformat()} @@ -381,37 +407,66 @@ def _write_index(ws: Workspace, index: dict) -> None: _write_atomically(_index_path(ws), lambda fh: json.dump(index, fh, ensure_ascii=False, indent=1)) -# Jobs already decoded in this process, by cache file: (signature, Job). A page -# that loads the workspace then skips the JSON decode and the rebuild of every -# unchanged job. A Job is never changed once loaded (parse.split_agents and the -# compare page make new ones with dataclasses.replace). +# Jobs already decoded in this process, by cache file: (signature, Job, trial +# signatures, head). A page that loads the workspace then skips the JSON decode +# and the rebuild of every unchanged job. A Job is never changed once loaded +# (parse.split_agents and the compare page make new ones with dataclasses.replace). _DECODED: dict = {} _DECODED_LOCK = threading.Lock() +def _read_cache(path: str, job_dir: str) -> tuple | None: + """(signature, head, Job) of the cache file *path* when it is *job_dir*'s, else None.""" + try: + with open(path, encoding="utf-8") as fh: + cached = json.load(fh, object_hook=_decode) + # "agents": a cache from before split_agents knows nothing of a job's several agents + if cached.get("dir") == job_dir and "agents" in cached.get("job", {}): + return cached.get("signature"), cached.get("head"), job_from_dict(cached["job"]) + except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError, AttributeError): + pass + return None + + +def _write_cache(ws: Workspace, path: str, sig: str, head: str, job: parse.Job) -> None: + os.makedirs(ws.jobs_dir, exist_ok=True) + _write_atomically(path, lambda fh: json.dump({"signature": sig, "head": head, "dir": job.dir, "cached": _now(), + "job": job_to_dict(job)}, fh, default=_encode, ensure_ascii=False)) + + def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple: - """(Job, from_cache). Parses only when the signature changed. ``index``: - the rail's index as load_jobs holds it, noted in place.""" - sig, trial_sigs = _signatures(job_dir) + """(Job, from_cache). Parses only when the signature changed; an ended job + whose head is unchanged is not stated trial by trial. ``index``: the + rail's index as load_jobs holds it, noted in place.""" + head = _head(job_dir) # before anything is read: a change after it is caught next time path = _cache_file(ws, job_dir) with _DECODED_LOCK: known = _DECODED.get(path) + if known is not None and known[1].dir == job_dir and known[3] == head and _ended(known[1]): + _note_in_index(ws, job_dir, known[1], index) + return known[1], True + cached = _read_cache(path, job_dir) if known is None else None + if cached is not None and cached[1] == head and _ended(cached[2]): + with _DECODED_LOCK: + _DECODED[path] = (cached[0], cached[2], {}, head) + _note_in_index(ws, job_dir, cached[2], index) + return cached[2], True + sig, trial_sigs = _signatures(job_dir) if sig: if known is not None and known[0] == sig and known[1].dir == job_dir: + with _DECODED_LOCK: + _DECODED[path] = (sig, known[1], known[2], head) _note_in_index(ws, job_dir, known[1], index) return known[1], True - try: - with open(path, encoding="utf-8") as fh: - cached = json.load(fh, object_hook=_decode) - # "agents": a cache from before split_agents knows nothing of a job's several agents - if cached.get("signature") == sig and cached.get("dir") == job_dir and "agents" in cached.get("job", {}): - job = job_from_dict(cached["job"]) - with _DECODED_LOCK: - _DECODED[path] = (sig, job, trial_sigs) - _note_in_index(ws, job_dir, job, index) - return job, True - except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError): - pass + if known is not None: + cached = _read_cache(path, job_dir) + if cached is not None and cached[0] == sig: + if cached[1] != head and _ended(cached[2]): + _write_cache(ws, path, sig, head, cached[2]) # read by its head from the next process on + with _DECODED_LOCK: + _DECODED[path] = (sig, cached[2], trial_sigs, head) + _note_in_index(ws, job_dir, cached[2], index) + return cached[2], True reuse = None if known is not None and known[1].dir == job_dir: before = {t.dir: t for t in known[1].trials} @@ -420,11 +475,9 @@ def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple: reuse = lambda sub: before.get(sub) if old_sigs.get(sub) and old_sigs.get(sub) == trial_sigs.get(sub) else None job = parse.parse_job(job_dir, reuse=reuse) if sig: - os.makedirs(ws.jobs_dir, exist_ok=True) - _write_atomically(path, lambda fh: json.dump({"signature": sig, "dir": job_dir, "cached": _now(), - "job": job_to_dict(job)}, fh, default=_encode, ensure_ascii=False)) + _write_cache(ws, path, sig, head, job) with _DECODED_LOCK: - _DECODED[path] = (sig, job, trial_sigs) + _DECODED[path] = (sig, job, trial_sigs, head) _note_in_index(ws, job_dir, job, index) return job, False diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index ebe97fc6..1762cfad 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -315,8 +315,7 @@ def test_a_fold_is_read_again_only_when_the_workspace_parses_a_shard_again(self) first = launches.fold([workspace.load_job(ws, d)[0] for d in dirs]) self.assertEqual(launches.fold([workspace.load_job(ws, d)[0] for d in dirs]), first) self.assertEqual(check.call_count, 1) - trial = os.path.join(dirs[1], sorted(n for n in os.listdir(dirs[1]) if n.startswith("t-"))[0], "result.json") - with open(trial, "a") as f: + with open(os.path.join(dirs[1], "result.json"), "a") as f: # an ended shard is read again by its own record f.write("\n") launches.fold([workspace.load_job(ws, d)[0] for d in dirs]) self.assertEqual(check.call_count, 2) diff --git a/app/tests/test_workspace.py b/app/tests/test_workspace.py index a40589a9..3ee6b352 100644 --- a/app/tests/test_workspace.py +++ b/app/tests/test_workspace.py @@ -296,6 +296,72 @@ def test_a_running_job_reads_only_the_trials_that_moved(self): self.assertEqual(len(again.trials), 3) self.assertIs(again.trials[0], first.trials[0]); self.assertIs(again.trials[1], first.trials[1]) + def _end(self, **stats): + _write_json(os.path.join(self.job_dir, "result.json"), { + "id": "j1", "started_at": "2026-09-04T01:00:00Z", "finished_at": "2026-09-04T01:05:00Z", + "stats": {"n_running_trials": 0, "n_pending_trials": 0, **stats}}) + + def _rewrite_trial(self, reward): + target = os.path.join(self.job_dir, "tau3-bench__t1__abc", "result.json") + with open(target, encoding="utf-8") as fh: + data = json.load(fh) + data["verifier_result"] = {"rewards": {"reward": reward}} + _write_json(target, data) + + def test_an_ended_job_is_not_stated_trial_by_trial(self): + self._end() + first, _ = workspace.load_job(self.ws, self.job_dir) + with patch.object(workspace, "_signatures", side_effect=AssertionError("trials stated")): + again, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertIs(again, first); self.assertTrue(from_cache) + workspace._DECODED.clear() # a new process: the file on disk, by its head + again, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertTrue(from_cache); self.assertEqual(again, first) + + def test_a_cache_from_before_the_head_learns_it_once(self): + self._end() + workspace.load_job(self.ws, self.job_dir) + path = workspace._cache_file(self.ws, self.job_dir) + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + del data["head"] + _write_json(path, data) + workspace._DECODED.clear() + _, from_cache = workspace.load_job(self.ws, self.job_dir) # by its signature, the head written down + self.assertTrue(from_cache) + workspace._DECODED.clear() + with patch.object(workspace, "_signatures", side_effect=AssertionError("trials stated")): + _, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertTrue(from_cache) + + def test_an_ended_job_a_resume_rewrites_is_parsed_again(self): + self._end() + workspace.load_job(self.ws, self.job_dir) + self._rewrite_trial(1.0) + _write_json(os.path.join(self.job_dir, "result.json"), + {"id": "j1", "started_at": "2026-09-04T01:00:00Z", "stats": {"n_running_trials": 1}}) + job, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertFalse(from_cache) + self.assertEqual([t.reward for t in job.trials], [1.0, 1.0]) + self.assertTrue(job.is_running) + + def test_an_ended_job_that_gains_a_trial_is_parsed_again(self): + self._end() + workspace.load_job(self.ws, self.job_dir) + time.sleep(0.01) + shutil.copytree(os.path.join(self.job_dir, "tau3-bench__t1__abc"), os.path.join(self.job_dir, "tau3-bench__t9__abc")) + job, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertFalse(from_cache); self.assertEqual(len(job.trials), 3) + + def test_a_running_job_updates_with_each_trial(self): + self._end(n_running_trials=1) + workspace.load_job(self.ws, self.job_dir) + for reward in (1.0, 0.5): + time.sleep(0.01) + self._rewrite_trial(reward) # its result.json, config.json and folder as they were + job, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertFalse(from_cache); self.assertEqual(job.trials[1].reward, reward) + def test_index_lists_runs_without_a_parse(self): self.assertEqual(workspace.job_index(self.ws), {}) workspace.load_job(self.ws, self.job_dir) From 90a478249ef9774c3a9aa5fe3b4ccaa39f2bf5f8 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Wed, 18 Nov 2026 14:50:46 +0100 Subject: [PATCH 2/8] perf(web): read a single job's trial and judge pages from its own folder The trial, judge, reconcile, live-board and builder-like lookups found one job among every job of the workspace. When the open workspace has exactly one folder of that job's name, it alone is read; another workspace asked for, no such folder or several read the whole workspace as before. The job page keeps the whole walk: its Delete groups every launch of the workspace anyway. --- app/ui_tests/test_job_lookup.py | 60 +++++++++++++++++++++++++++++++++ app/web/builder.py | 2 +- app/web/judge.py | 2 +- app/web/judge_tasks.py | 2 +- app/web/reconcile.py | 2 +- app/web/runs.py | 3 +- app/web/shared.py | 17 ++++++++++ app/web/traces.py | 2 +- 8 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 app/ui_tests/test_job_lookup.py diff --git a/app/ui_tests/test_job_lookup.py b/app/ui_tests/test_job_lookup.py new file mode 100644 index 00000000..6d5d3fdd --- /dev/null +++ b/app/ui_tests/test_job_lookup.py @@ -0,0 +1,60 @@ +"""A single job's page reads that job's folder alone, and the whole workspace +on any doubt (``shared._job_for_request``), finding what ``_find_job`` over +every job finds.""" +from __future__ import annotations + +import os +import shutil +import tempfile +import unittest +from contextlib import ExitStack +from pathlib import Path +from unittest.mock import patch + +from nullius import workspace + +from .test_routes import web + +JOB = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "harbor-jobs" / "liveboard-oracle-x4" + + +class JobLookup(unittest.TestCase): + def setUp(self): + self.stack = ExitStack() + self.addCleanup(self.stack.close) + tmp = Path(self.stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-job-lookup-"))) + self.roots = [str(tmp / "a"), str(tmp / "b")] + shutil.copytree(JOB, tmp / "a" / "job-x") + shutil.copytree(JOB, tmp / "b" / "job-y") + self.stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces"))) + self.stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True)) + self.ws = workspace.create(web.shared.WS_HOME, self.roots, "lookup") + web.shared.open_workspace(self.ws) + self.whole = patch.object(workspace, "load_jobs", wraps=workspace.load_jobs) + + def lookup(self, job_id, query=""): + with web.app.test_request_context(f"/job/{job_id}{query}"), self.whole as whole: + return web.shared._job_for_request(job_id), whole.called + + def test_one_folder_of_that_name_is_read_alone(self): + job, whole = self.lookup("job-x") + self.assertFalse(whole) + self.assertEqual(job.dir, os.path.join(self.roots[0], "job-x")) + self.assertEqual(web.app.test_client().get("/job/job-x").status_code, 200) + + def test_another_workspace_asked_for_reads_the_whole_of_it(self): + job, whole = self.lookup("job-x", f"?workspace={self.ws.name}") + self.assertTrue(whole); self.assertEqual(job.dir, os.path.join(self.roots[0], "job-x")) + + def test_two_folders_of_that_name_read_the_whole_workspace_and_find_the_first(self): + shutil.copytree(JOB, os.path.join(self.roots[1], "job-x")) + job, whole = self.lookup("job-x") + self.assertTrue(whole); self.assertEqual(job.dir, os.path.join(self.roots[0], "job-x")) + + def test_no_such_folder_reads_the_whole_workspace(self): + job, whole = self.lookup("job-z") + self.assertTrue(whole); self.assertIsNone(job) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/builder.py b/app/web/builder.py index e190398f..001c5e6a 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -46,7 +46,7 @@ def new_run(): data["loadRecipe"] = ex.plan.recipe data["loadRun"] = run elif like: - job = shared._find_job(shared._jobs_for_request(), like) + job = shared._job_for_request(like) if job is None: abort(404) data["loadLike"] = {"job": job.job_id, "agent": job.agent_name, "launch": job.launch, diff --git a/app/web/judge.py b/app/web/judge.py index b83542d9..19e4601d 100644 --- a/app/web/judge.py +++ b/app/web/judge.py @@ -290,7 +290,7 @@ def judge_trial_route(job_id, trial_name): def judge_job_route(job_id): """Judge the job's unjudged trials with one judge, the form's ``lane`` (Decision unless it names the LLM judge): the click is the request.""" - job = shared._find_job(shared._jobs_for_request(), job_id) + job = shared._job_for_request(job_id) if job is None: abort(404) lane = request.form.get("lane") if request.form.get("lane") in client.LANES else "decision" diff --git a/app/web/judge_tasks.py b/app/web/judge_tasks.py index 786c76ee..960bf985 100644 --- a/app/web/judge_tasks.py +++ b/app/web/judge_tasks.py @@ -16,7 +16,7 @@ def judge_tasks_route(job_id): """Ask the instruction score of every task of the job that has none, on the click of the button.""" from . import judge - job = shared._find_job(shared._jobs_for_request(), job_id) + job = shared._job_for_request(job_id) if job is None: abort(404) todo = judge.tasks_todo(job) diff --git a/app/web/reconcile.py b/app/web/reconcile.py index 4bb71e78..a91ab65e 100644 --- a/app/web/reconcile.py +++ b/app/web/reconcile.py @@ -37,7 +37,7 @@ def view_of(job, threshold: float) -> dict: @app.route("/job//reconcile") def reconcile_page(job_id): - j = shared._find_job(shared._jobs_for_request(), job_id) + j = shared._job_for_request(job_id) if not j: abort(404) return render_template("runs/reconcile.html", job=j, view=view_of(j, threshold_requested()), diff --git a/app/web/runs.py b/app/web/runs.py index a5693077..bf596836 100644 --- a/app/web/runs.py +++ b/app/web/runs.py @@ -544,6 +544,7 @@ def upload(): @app.route("/job/", methods=["GET", "HEAD"]) def job(job_id): + # every job, not the one alone: its Delete (_deletable) walks the workspace anyway, and one folder first costs more j = shared._find_job(shared._jobs_for_request(), job_id) ex = execution.for_job(job_id) # this job's own state beside the run's: ended, or a part still to come while another runs @@ -820,7 +821,7 @@ def job_live(job_id): the cache on every request or, worse, hit it and show a board that stopped. Folding a few thousand lines is cheap enough to do every second. """ - j = shared._find_job(shared._jobs_for_request(), job_id) + j = shared._job_for_request(job_id) if not j: abort(404) if not live.has_feed(j.dir): diff --git a/app/web/shared.py b/app/web/shared.py index 65521036..598b30df 100644 --- a/app/web/shared.py +++ b/app/web/shared.py @@ -12,6 +12,7 @@ from markupsafe import Markup, escape from werkzeug.exceptions import Forbidden from nullius import importers as harnesses +from nullius.runs import parse from nullius import execution from nullius.catalogue import catalog from nullius import workspace @@ -414,6 +415,22 @@ def _find_job(jobs, job_id): return next((j for j in jobs if j.job_id == job_id), None) +def _job_for_request(job_id): + """``_find_job(_jobs_for_request(), job_id)``, reading only the job's own + folder when the open workspace has exactly one of that name (a view of one + of a job's agents is ``~``). The whole workspace on any + doubt: another workspace asked for, no such folder, or several.""" + ws = STATE["workspace"] + if ws is not None and not request.args.get("workspace") and "" not in g.get("_jobs", {}): + names = {job_id, job_id.rsplit("~", 1)[0]} + dirs = [d for d in parse.find_job_dirs(STATE["job_roots"]) if os.path.basename(d) in names] + if len(dirs) == 1: + job = _find_job(parse.split_agents([workspace.load_job(ws, dirs[0])[0]]), job_id) + if job is not None: + return job + return _find_job(_jobs_for_request(), job_id) + + def _safe_dest(base: str, rel: str) -> str | None: """Join *rel* onto *base*, refusing anything that escapes *base*.""" dest = os.path.normpath(os.path.join(base, rel)) diff --git a/app/web/traces.py b/app/web/traces.py index e41a4fa9..e490c409 100644 --- a/app/web/traces.py +++ b/app/web/traces.py @@ -111,7 +111,7 @@ def _trial_evidence(job_id, trial_name, tries=False): """Resolve URL identifiers through parsed runs, never as filesystem paths. ``tries``: an attempt's earlier try too, ``.try-N``: evidence to read, never a trial to judge.""" - job = shared._find_job(shared._jobs_for_request(), job_id) + job = shared._job_for_request(job_id) if job is None: abort(404) trial = _find_trial(job, trial_name) or (_find_try(job, trial_name) if tries else None) From faa02884fea42a3d3a080d89068f67f941d80400 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Wed, 18 Nov 2026 14:51:57 +0100 Subject: [PATCH 3/8] perf(web): resolve each run record's jobs folder once per request The Workspaces page matched every run record against every workspace, resolving the record's folder and each root again for each pair. Each workspace's roots are now resolved once, and each record's folder once per request. --- app/web/runs.py | 31 +++++++++++++++++++++---------- app/web/shared.py | 9 +++++++-- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/app/web/runs.py b/app/web/runs.py index bf596836..22d7f564 100644 --- a/app/web/runs.py +++ b/app/web/runs.py @@ -116,8 +116,8 @@ def _grouped(ws=None): have = {j.job_id for j in jobs} # The run records are the app's, not the workspace's: only those launched # into this workspace's roots, or with a job already written here, belong. - roots = None if ws is None else list(ws.roots) + list(ws.job_roots) - mine = [e for e in execution.EXECUTIONS.values() if _launched_here(e, have, roots)] + real = _real_roots(None if ws is None else list(ws.roots) + list(ws.job_roots)) + mine = [e for e in execution.EXECUTIONS.values() if _launched_here(e, have, real)] merged = launches.merged_sources(jobs) jobs = [j for j in jobs if j.job_id not in merged] return jobs, have, mine, launches.group(jobs, {e.id: _record(e, merged) for e in mine}) @@ -151,16 +151,27 @@ def _tally(g, by_id) -> dict: return {"n_runs": len(written) + len(pending) - checks, "n_checks": checks} -def _launched_here(ex, have, roots=None) -> bool: - """A run record belongs to the open workspace (or one of ``roots``) when its - job root is one of the workspace's roots (a Run adopts it there) or one of - its jobs is here.""" +def _real_roots(roots=None) -> set: + """``roots`` (the open workspace's when None), links resolved: what ``_launched_here`` matches against.""" + if roots is None: + roots = list(shared.STATE.get("roots") or []) + list(shared.STATE.get("job_roots") or []) + return {os.path.realpath(r) for r in roots} + + +def _launched_here(ex, have, real) -> bool: + """A run record belongs to a workspace when its job root is one of the + workspace's roots, ``real`` (``_real_roots``; a Run adopts it there), or + one of its jobs is here, ``have``. Each record's root is resolved once per + request, however many workspaces ask.""" if any(str(x.get("name")) in have for x in ex.plan.jobs): return True jobs_dir = getattr(ex, "jobs", None) - if roots is None: - roots = list(shared.STATE.get("roots") or []) + list(shared.STATE.get("job_roots") or []) - return bool(jobs_dir) and os.path.realpath(jobs_dir) in {os.path.realpath(r) for r in roots} + if not jobs_dir: + return False + memo = shared.request_memo("real_jobs") + if jobs_dir not in memo: + memo[jobs_dir] = os.path.realpath(jobs_dir) + return memo[jobs_dir] in real # A launch's runs, in the order the block lists them: finished first, best @@ -372,7 +383,7 @@ def _roots_pulse() -> str: Nothing is parsed; this is a handful of stats per job.""" dirs = parse.find_job_dirs(shared.STATE["job_roots"] or []) have = {os.path.basename(d) for d in dirs} - marks = [(f"run:{e.id}", e.state, 0) for e in execution.EXECUTIONS.values() if _launched_here(e, have)] + marks = [(f"run:{e.id}", e.state, 0) for e in execution.EXECUTIONS.values() if _launched_here(e, have, _real_roots())] for d in dirs: feed = os.path.join(d, "events.jsonl") marks.append((os.path.basename(d), os.path.isfile(os.path.join(d, "result.json")), diff --git a/app/web/shared.py b/app/web/shared.py index 598b30df..dab36e83 100644 --- a/app/web/shared.py +++ b/app/web/shared.py @@ -367,11 +367,16 @@ def _load_jobs(): return workspace.load_jobs(ws, STATE["job_roots"]) +def request_memo(name: str) -> dict: + """A dict kept for the rest of this request under *name*; a new one outside a request.""" + return g.setdefault(f"_memo_{name}", {}) if has_request_context() else {} + + def _jobs_for_workspace(name=None): """Read an indexed workspace without making it active or converting sources. Once per request and workspace: a page that asks for the jobs several times (the job page's controls, the rail, a summary) walks them once.""" - memo = g.setdefault("_jobs", {}) if has_request_context() else {} + memo = request_memo("jobs") key = name if name and not (STATE["workspace"] and name == STATE["workspace"].name) else "" if key in memo: return memo[key] @@ -421,7 +426,7 @@ def _job_for_request(job_id): of a job's agents is ``~``). The whole workspace on any doubt: another workspace asked for, no such folder, or several.""" ws = STATE["workspace"] - if ws is not None and not request.args.get("workspace") and "" not in g.get("_jobs", {}): + if ws is not None and not request.args.get("workspace") and "" not in request_memo("jobs"): names = {job_id, job_id.rsplit("~", 1)[0]} dirs = [d for d in parse.find_job_dirs(STATE["job_roots"]) if os.path.basename(d) in names] if len(dirs) == 1: From c8fc5319cd2923f9fe3f0201ec59dfb33edf2020 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Wed, 18 Nov 2026 14:52:01 +0100 Subject: [PATCH 4/8] perf(execution): walk an ended run's folder again only when it moves The Launches page walked every run folder on each load. A run still alive is walked at each ask; an ended one again only when its state, its record, its folder or its jobs folder moved. Delete's question walks the folder afresh. --- app/nullius/execution.py | 31 +++++++++++++++++++++++++++++++ app/tests/test_run_delete.py | 24 ++++++++++++++++++++++++ app/web/runs.py | 2 +- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 6de6d3d9..71587762 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -1762,6 +1762,37 @@ def size_of(path: str) -> int: return total +# Sizes of ended runs' folders, by folder: (what they were read at, bytes). +_SIZES: Dict[str, tuple] = {} +_SIZES_LOCK = threading.Lock() + + +def run_size(ex: "Execution") -> int: + """``size_of`` the run's folder, as the Launches page lists it: walked at + each ask while the run is alive; once it has ended, again only when its + state, its record, its folder or its jobs folder moved. What Delete asks + (``deletion``) walks the folder afresh. + shortcut: a file changed deep in an ended run alone (a trace view saved on + its trial page) is counted from the next move of those; walk the job folders too if that matters.""" + if ex.state in ALIVE: + return size_of(ex.dir) + + def mtime(path): + try: + return os.stat(path).st_mtime_ns + except OSError: + return None + seen = (ex.state, ex.finished, *map(mtime, (ex.dir, os.path.join(ex.dir, RECORD), ex.jobs))) + with _SIZES_LOCK: + known = _SIZES.get(ex.dir) + if known is not None and known[0] == seen: + return known[1] + size = size_of(ex.dir) + with _SIZES_LOCK: + _SIZES[ex.dir] = (seen, size) + return size + + def _deletable(name: str, root: Optional[str]) -> tuple: """A run's folder and record, when it is a run that has ended; ValueError (alive, or not a run) or OSError (its record unreadable) otherwise.""" diff --git a/app/tests/test_run_delete.py b/app/tests/test_run_delete.py index dda38f8b..19a35f19 100644 --- a/app/tests/test_run_delete.py +++ b/app/tests/test_run_delete.py @@ -171,6 +171,30 @@ def rmtree(path, *a, **kw): self.assertFalse(os.path.exists(os.path.join(self.root, "s1"))); self.assertTrue(os.path.isdir(s2)) +class RunSize(unittest.TestCase): + setUp, run_ = RunDelete.setUp, RunDelete.run_ + + def _grow(self): + with open(os.path.join(self.jobs, "job-a", "tau3-bench__t0__abc", "result.json"), "a") as fh: + fh.write(" " * 1000) + + def test_an_ended_run_is_walked_again_once_it_moves_and_delete_always_asks_afresh(self): + ex = execution.Execution(execution.Plan(name="r1", script="true"), root=self.root, record={"state": "done"}) + first = execution.run_size(ex) + self.assertEqual(first, execution.size_of(ex.dir)) + self._grow() # deep in a trial: the listing keeps what it read + self.assertEqual(execution.run_size(ex), first) + self.assertEqual(execution.deletion(["r1"], self.home, self.root)["bytes"], first + 1000) + _write_json(os.path.join(ex.dir, "nullius.log"), {}) # its folder moves: walked again + self.assertEqual(execution.run_size(ex), execution.size_of(ex.dir)) + + def test_a_run_still_going_is_walked_at_each_ask(self): + ex = execution.Execution(execution.Plan(name="r1", script="true"), root=self.root, record={"state": "running"}) + first = execution.run_size(ex) + self._grow() + self.assertEqual(execution.run_size(ex), first + 1000) + + class SizeOf(unittest.TestCase): def test_a_file_gone_since_the_walk_counts_nothing(self): d = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, d) diff --git a/app/web/runs.py b/app/web/runs.py index 22d7f564..0accd154 100644 --- a/app/web/runs.py +++ b/app/web/runs.py @@ -667,7 +667,7 @@ def _inventory(g, runs, wss, by_id) -> dict: its Delete would take is asked only when it opens (``launch_ask``).""" return {"launch": g, "runs": runs, "why": _undeletable(runs), "shards": len(runs), "state": _most_telling(map(_run_state, runs)), **_tally(g, by_id), "machines": list(dict.fromkeys(e.plan.host.label if e.plan.host else "this machine" for e in runs)), - "bytes": sum(execution.size_of(e.dir) for e in runs), "workspaces": wss, + "bytes": sum(map(execution.run_size, runs)), "workspaces": wss, # the agents its jobs recorded, and those its records name for jobs not written yet "agents": sorted({by_id[i].agent_label for i in g.job_ids if by_id[i].agent_label} | {str(x["agent"]) for e in runs for x in e.plan.jobs if x.get("agent")})} From 1fb20a48f6c268a303bde3042417419fb7973ec3 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Wed, 18 Nov 2026 14:57:40 +0100 Subject: [PATCH 5/8] perf(runs): index the run records by job once per request The Runs page asked for_job of every job and launch, each a scan of every run record. execution.owners indexes them once, with for_job's tie-break: the newest start, the first listed of a tie. The Runs page reads it once per request, and the Launches page's own index is the same one, so the two cannot name different runs for a job. --- app/nullius/execution.py | 25 ++++++++++++++++------- app/tests/test_run_delete.py | 16 +++++++++++++++ app/ui_tests/test_runs_by_launch.py | 2 +- app/web/runs.py | 31 +++++++++++++++-------------- app/web/shared.py | 18 +++++++++++++++++ 5 files changed, 69 insertions(+), 23 deletions(-) diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 71587762..3dae04c9 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -1674,25 +1674,36 @@ def active() -> List[Execution]: return [e for e in EXECUTIONS.values() if e.state in ALIVE] -def for_job(job_id: str) -> Optional[Execution]: +def owners() -> Dict[str, Execution]: + """Each job a plan names -> the newest execution whose script writes it; of + several started at once, the first listed.""" + out: Dict[str, Execution] = {} + for e in EXECUTIONS.values(): + for j in e.plan.jobs: + name = str(j.get("name")) + if name not in out or e.started > out[name].started: + out[name] = e + return out + + +def for_job(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Optional[Execution]: """The newest execution whose script writes this job, if any - the one the job page shows beside the job, or instead of it while Harbor has - not written it yet.""" - found = [e for e in EXECUTIONS.values() if any(str(j.get("name")) == job_id for j in e.plan.jobs)] - return max(found, key=lambda e: e.started) if found else None + not written it yet. ``owner``: ``owners()``, when the caller holds it.""" + return (owners() if owner is None else owner).get(job_id) ENDED = ("stopped", "failed", "done") -def ended_for(job_id: str) -> Optional[tuple]: +def ended_for(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Optional[tuple]: """``(state, finished_at)`` when the run that writes this job has ended - stopped, failed or done - by the runner's own record; None while it is alive or when no run of this application wrote the job. A Harbor killed by a Stop leaves its feed without ``job.end`` and its result record with trials "running", so a page reading either alone would say - the job goes on; this is the fact that says otherwise.""" - ex = for_job(job_id) + the job goes on; this is the fact that says otherwise. ``owner``: as ``for_job``'s.""" + ex = for_job(job_id, owner) return ex.job_state(job_id) if ex is not None else None diff --git a/app/tests/test_run_delete.py b/app/tests/test_run_delete.py index 19a35f19..c710539e 100644 --- a/app/tests/test_run_delete.py +++ b/app/tests/test_run_delete.py @@ -195,6 +195,22 @@ def test_a_run_still_going_is_walked_at_each_ask(self): self.assertEqual(execution.run_size(ex), first + 1000) +class Owners(unittest.TestCase): + def test_several_records_naming_a_job_give_it_the_newest_and_the_first_listed_of_a_tie(self): + def rec(name, started, *jobs): + plan = execution.Plan(name=name, script="true", jobs=[{"name": j} for j in jobs]) + return execution.Execution(plan, root=tempfile.gettempdir(), record={"id": name, "state": "done", "started": started}) + recs = [rec("old", 1, "a", "b"), rec("tie-1", 5, "b", "c"), rec("tie-2", 5, "b", "c"), rec("new", 9, "a"), rec("older", 3, "a", "c")] + with patch.dict(execution.EXECUTIONS, {e.id: e for e in recs}, clear=True): + owner = execution.owners() + for job in "abcd": + found = [e for e in recs if any(j["name"] == job for j in e.plan.jobs)] + before = max(found, key=lambda e: e.started) if found else None # for_job as it was: one scan per job + self.assertIs(owner.get(job), before) + self.assertIs(execution.for_job(job), before); self.assertIs(execution.for_job(job, owner), before) + self.assertEqual({j: e.id for j, e in owner.items()}, {"a": "new", "b": "tie-1", "c": "tie-1"}) + + class SizeOf(unittest.TestCase): def test_a_file_gone_since_the_walk_counts_nothing(self): d = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, d) diff --git a/app/ui_tests/test_runs_by_launch.py b/app/ui_tests/test_runs_by_launch.py index 10ffd688..bc91efc0 100644 --- a/app/ui_tests/test_runs_by_launch.py +++ b/app/ui_tests/test_runs_by_launch.py @@ -174,7 +174,7 @@ def test_a_later_paused_shard_has_a_resume_link(self): group = SimpleNamespace(job_ids=[], pending=["first", "second"], source="record", dataset="") first = SimpleNamespace(state="done", plan=SimpleNamespace(recipe={}, jobs=[]), refused=lambda _: None, never_ran=lambda _: False) second = SimpleNamespace(state="paused", plan=SimpleNamespace(recipe={}, jobs=[]), live_state=lambda _: "paused") - with patch.object(execution, "for_job", side_effect=lambda key: {"first": first, "second": second}[key]), \ + with patch("web.shared.run_for", side_effect=lambda key: {"first": first, "second": second}[key]), \ patch("web.runs._control_planned", return_value=""): self.assertEqual(_launch_view(group, {}, {}, {})["paused"], "second") diff --git a/app/web/runs.py b/app/web/runs.py index 0accd154..429c1ab6 100644 --- a/app/web/runs.py +++ b/app/web/runs.py @@ -51,7 +51,7 @@ def index(): workspaces=workspace.list_all(shared.WS_HOME)) # A Harbor killed by a Stop (or dead with the run) leaves its result # record saying trials are running; the runner's record says otherwise. - ended = {j.job_id: execution.ended_for(j.job_id) for j in jobs} + ended = {j.job_id: shared.ended_for(j.job_id) for j in jobs} # The header, the chips and the mix bar describe the agents' trials. A # control (nop, oracle) is a check on the tasks, not a competitor: it is # listed on its own line as behaved or not (`feasibility.summary_of`), @@ -191,7 +191,7 @@ def _state_of(job_id, j, ended): "running".""" if j is not None and ended.get(job_id): return _said(ended[job_id][0]) - ex = execution.for_job(job_id) + ex = shared.run_for(job_id) if ex is not None and ex.state in execution.ALIVE: return ex.live_state(job_id) if j is None: @@ -211,7 +211,7 @@ def _control_planned(job_id) -> str: """``nop`` or ``oracle`` when a job not written yet is one of the run's controls by its plan, else ''. A standard plan names a declared control by that word; its setups are ``original`` or ``model``, never one.""" - ex = execution.for_job(job_id) + ex = shared.run_for(job_id) return "" if ex is None else next((_control_of(j) for j in ex.plan.jobs if str(j.get("name")) == job_id), "") @@ -249,7 +249,7 @@ def _most_telling(states) -> str: def _host_of(job_id) -> str: - ex = execution.for_job(job_id) + ex = shared.run_for(job_id) return "" if ex is None else ex.plan.host.label if ex.plan.host else "this machine" @@ -322,7 +322,7 @@ def _launch_view(g, by_id, verdicts, ended) -> dict: rows += [{"job": None, "job_id": i, "state": _state_of(i, None, ended)} for i in g.pending if i not in dict(waiting)] for r in rows: if r["state"] == "refused": - r["why"] = execution.for_job(r["job_id"]).refused(r["job_id"]) + r["why"] = shared.run_for(r["job_id"]).refused(r["job_id"]) rows = _fold(rows, ended) rate = lambda r: r["job"].adjusted_solve_rate if r["job"] is not None and r["job"].adjusted_solve_rate is not None else -1.0 rows.sort(key=lambda r: (_STATE_ORDER.get(r["state"], 9), -rate(r), r["job_id"])) @@ -360,14 +360,14 @@ def _launch_view(g, by_id, verdicts, ended) -> dict: counts = {} for r in leaves(controls + rows): counts[r["state"]] = counts.get(r["state"], 0) + 1 - ex = execution.for_job((g.job_ids + g.pending)[0]) + ex = shared.run_for((g.job_ids + g.pending)[0]) return {"gone": _deletion(_runs_of(g)), "launch": g, "checks": checks, "verdict": verdict, "controls": controls, "checked": checked, "agents": rows, "n_tasks": n_tasks, "compare": finished, **_tally(g, by_id), "paused": next((i for i in g.job_ids + g.pending - if (execution.for_job(i) is not None and execution.for_job(i).state == "paused")), None), + if (shared.run_for(i) is not None and shared.run_for(i).state == "paused")), None), "summary": " · ".join(f"{n} {s}" for s, n in sorted(counts.items(), key=lambda kv: _STATE_ORDER.get(kv[0], 9))), "recipe": (g.job_ids + g.pending)[0] if g.source == "record" and ex is not None and ex.plan.recipe else None, "feasibility": checks[0][0].job_id if checks else None, @@ -557,10 +557,10 @@ def upload(): def job(job_id): # every job, not the one alone: its Delete (_deletable) walks the workspace anyway, and one folder first costs more j = shared._find_job(shared._jobs_for_request(), job_id) - ex = execution.for_job(job_id) + ex = shared.run_for(job_id) # this job's own state beside the run's: ended, or a part still to come while another runs own = ((execution.NOT_RUN, None) if ex is not None and ex.state not in execution.ALIVE and ex.never_ran(job_id) - else execution.ended_for(job_id) or ((ex.live_state(job_id), None) + else shared.ended_for(job_id) or ((ex.live_state(job_id), None) if ex is not None and ex.live_state(job_id) in ("queued", execution.NOT_RUN) else None)) if not j: # Started from the builder and not written by Harbor yet: the run's @@ -573,13 +573,14 @@ def job(job_id): return render_template("runs/job.html", job=j, live_board=board, view=_view_of(board, j), execution=ex, csrf=shared._csrf_token(), judgement=judge.job_summary(j) if j.trials else None, - control=feasibility.summary_of(j), ended=execution.ended_for(job_id), own=own, deletable=_deletable(job_id, ex), + control=feasibility.summary_of(j), ended=shared.ended_for(job_id), own=own, deletable=_deletable(job_id, ex), trial_rows=attempts.grouped(j), pass_k=attempts.for_job(j)) -def _runs_of(g, owner=execution.for_job) -> list: +def _runs_of(g, owner=None) -> list: """The runs Nullius made that hold every row of the launch *g*, and no job outside it; [] when a row is not one of theirs. ``owner``: a job's run.""" + owner = owner or shared.run_for ids = g.job_ids + g.pending runs = list({id(e): e for e in map(owner, ids)}.values()) if None in runs or any(not {str(x.get("name")) for x in e.plan.jobs} <= set(ids) for e in runs): @@ -653,7 +654,7 @@ def _held_launches() -> tuple: launches from it.""" recs = sorted(execution.EXECUTIONS.values(), key=lambda e: e.started) held = workspace.holding_all(shared.WS_HOME, {os.path.realpath(e.dir) for e in recs}) - owner = {str(x.get("name")): e for e in recs for x in e.plan.jobs} # the newest run naming a job, as for_job + owner = execution.owners() by_id, every = _every() for j in by_id.values(): # a merge beside its shards: their run's owner.setdefault(j.job_id, next((owner[m] for m in getattr(j, "merged_from", []) if m in owner), None)) @@ -811,7 +812,7 @@ def _live_board_state(j): if not live.has_feed(j.dir): return None try: - snap = live.snapshot(j.dir, ended=execution.ended_for(j.job_id), with_timeline=True) + snap = live.snapshot(j.dir, ended=shared.ended_for(j.job_id), with_timeline=True) except live.FeedError as e: # A malformed line in the middle of the feed is a broken writer, not a # race. The page must still render — the settled sections read @@ -836,7 +837,7 @@ def job_live(job_id): if not j: abort(404) if not live.has_feed(j.dir): - ended = execution.ended_for(job_id) + ended = shared.ended_for(job_id) if ended: # The preparing page must settle even if the runner never wrote a feed. response = jsonify({"state": "finished", "running": False, "ended_by": ended[0]}) @@ -844,7 +845,7 @@ def job_live(job_id): return response abort(404) try: - snap = live.snapshot(j.dir, ended=execution.ended_for(j.job_id), with_timeline=True) + snap = live.snapshot(j.dir, ended=shared.ended_for(j.job_id), with_timeline=True) except live.FeedError as e: response = jsonify({"error": f"events.jsonl is not a feed: {e}"}) response.status_code = 422 diff --git a/app/web/shared.py b/app/web/shared.py index dab36e83..732aa500 100644 --- a/app/web/shared.py +++ b/app/web/shared.py @@ -372,6 +372,24 @@ def request_memo(name: str) -> dict: return g.setdefault(f"_memo_{name}", {}) if has_request_context() else {} +def _owners() -> dict: + """``execution.owners``, once per request: a page asks it of every job it lists.""" + memo = request_memo("owners") + if "owners" not in memo: + memo["owners"] = execution.owners() + return memo["owners"] + + +def run_for(job_id): + """``execution.for_job`` over this request's ``_owners``.""" + return execution.for_job(job_id, _owners()) + + +def ended_for(job_id): + """``execution.ended_for`` over this request's ``_owners``.""" + return execution.ended_for(job_id, _owners()) + + def _jobs_for_workspace(name=None): """Read an indexed workspace without making it active or converting sources. Once per request and workspace: a page that asks for the jobs several From 7e88e200f428caad8668c576c632b0e7161a457a Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Wed, 18 Nov 2026 16:22:50 +0100 Subject: [PATCH 6/8] perf(web): read each job's outcome counts once per row Job.counts walks every trial on each read, and the Runs list read it up to a dozen times per row. The templates that read a job's counts, or a feasibility report's, more than once now read them once into oc. The counts stay a property: a job's trials can change after it is made. --- app/templates/compare/compare.html | 6 +++--- app/templates/runs/feasibility.html | 2 +- app/templates/runs/index.html | 16 ++++++++-------- app/templates/runs/job.html | 14 +++++++------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/app/templates/compare/compare.html b/app/templates/compare/compare.html index 0dffc29c..0ececc2a 100644 --- a/app/templates/compare/compare.html +++ b/app/templates/compare/compare.html @@ -146,10 +146,10 @@ {% endif %} Outcome mix - {% call(i) each() %}{% set j = jobs[i] %} + {% call(i) each() %}{% set j = jobs[i] %}{% set oc = j.counts %} - {%- for k in ORDER %}{% if j.counts[k] %}{% endif %}{% endfor -%} + {%- for k in ORDER %}{% if oc[k] %}{% endif %}{% endfor -%} {% endcall %} diff --git a/app/templates/runs/feasibility.html b/app/templates/runs/feasibility.html index bf2cc289..c31b463f 100644 --- a/app/templates/runs/feasibility.html +++ b/app/templates/runs/feasibility.html @@ -74,7 +74,7 @@
{{ report.n_feasible }}/{{ report.rows|length }} feasible - {% for k in F_ORDER %}{% if report.counts[k] %}{{ F_LABELS[k] }} {{ report.counts[k] }}{% endif %}{% endfor %} + {% set oc = report.counts %}{% for k in F_ORDER %}{% if oc[k] %}{{ F_LABELS[k] }} {{ oc[k] }}{% endif %}{% endfor %}
{% if report.rows %} diff --git a/app/templates/runs/index.html b/app/templates/runs/index.html index 49316d72..57c9bdf7 100644 --- a/app/templates/runs/index.html +++ b/app/templates/runs/index.html @@ -14,10 +14,10 @@ {#- One row of a launch's table: a task check (`kind` 'control') or an agent; `part` 'fold' for a participant's shards as one row, 'shard' for one of them, '' for a run of one job. #} -{% macro launch_row(r, g, kind, part='', first=False) %}{% set L = g.launch %}{% set j = r.job %}{% set v = r.verdict %} +{% macro launch_row(r, g, kind, part='', first=False) %}{% set L = g.launch %}{% set j = r.job %}{% set v = r.verdict %}{% set oc = j.counts if j else none %} + {%- else %} data-outcomes="{% if j %}{% for k in ORDER %}{% if oc[k] %}{{ k }} {% endif %}{% endfor %}{% if j.n_regressions %}regression{% endif %}{% endif %}"{% endif %}> {%- if part == 'shard' %}
{{ shard_link(r) }}
{%- else %} @@ -45,9 +45,9 @@ {%- elif j and j.adjusted_solve_rate is not none %}
{{ j.adjusted_solve_rate|pct }}
{%- else %}
—
{% endif %} - {%- if j and j.trials and not j.scored_only and j.counts.unknown and j.counts.unknown * 4 >= j.trials|length %}⚠ {{ j.counts.unknown }} of {{ j.trials|length }} have no verdict{% endif %} + {%- if j and j.trials and not j.scored_only and oc.unknown and oc.unknown * 4 >= j.trials|length %}⚠ {{ oc.unknown }} of {{ j.trials|length }} have no verdict{% endif %} {%- endif %} - {% if j and j.trials %}{% for k in ORDER %}{% if j.counts[k] %}{% endif %}{% endfor %}{% else %}{% endif %} + {% if j and j.trials %}{% for k in ORDER %}{% if oc[k] %}{% endif %}{% endfor %}{% else %}{% endif %} {%- if kind == 'control' %} — {%- else %} @@ -178,8 +178,8 @@

{{ g.dataset_title if L.dataset else (L.name or g.dataset_title) }}{% if g.n {% for j in jobs %} {# A control row carries no outcomes to filter by: the chips count the agents' trials, and a nop's rejections are not failed verifications. #} - {% set v = verdicts[j.job_id] %} - {{ check_cell('Compare ' ~ j.job_id, j.job_id) }} @@ -213,8 +213,8 @@

{{ g.dataset_title if L.dataset else (L.name or g.dataset_title) }}{% if g.n {% else %} - - {%- for k in ORDER %}{% if j.counts[k] %}{% endif %}{% endfor -%} + + {%- for k in ORDER %}{% if oc[k] %}{% endif %}{% endfor -%} {% endif %} diff --git a/app/templates/runs/job.html b/app/templates/runs/job.html index 8c9d1570..ae65b060 100644 --- a/app/templates/runs/job.html +++ b/app/templates/runs/job.html @@ -6,7 +6,7 @@ {% block crumb %}/ {{ job.job_id }}{% endblock %} {% block head %} {% endblock %} -{% block body %} +{% block body %}{% set oc = job.counts %} {% set meta %}
@@ -227,14 +227,14 @@
Outcomes
- {% for k in ORDER %}{% if job.counts[k] %}{% endif %}{% endfor %} + {% for k in ORDER %}{% if oc[k] %}{% endif %}{% endfor %}
- {% for k in ORDER %}{% if job.counts[k] %}{% set h = 45 + 30 * ((nlines.get(k) or 1) - 1) %} + {% for k in ORDER %}{% if oc[k] %}{% set h = 45 + 30 * ((nlines.get(k) or 1) - 1) %} {{ job.counts[k] }}{{ LABELS[k] }}{% if k in INFRA %}infra{% endif %} + data-tip="{{ DESCRIPTIONS[k] }}">{{ oc[k] }}{{ LABELS[k] }}{% if k in INFRA %}infra{% endif %} {% endif %}{% endfor %}
{{ job.trials|length }}Trials{% if job.n_planned and job.n_planned != job.trials|length %} of {{ job.n_planned }} planned{% endif %}
@@ -341,7 +341,7 @@

Trials

{{ job.trials|length }}
OutcomeAll▾
- {% for k in ORDER %}{% if job.counts[k] %}{{ LABELS[k] }}{{ job.counts[k] }}{% endif %}{% endfor %} + {% for k in ORDER %}{% if oc[k] %}{{ LABELS[k] }}{{ oc[k] }}{% endif %}{% endfor %}
{% if judgement and judgement.causes %}
Judge causeAll▾ @@ -478,7 +478,7 @@

Trials

Definitionsoutcomes · rates · judge
- {% for k in ORDER %}{% if job.counts[k] %}
{{ LABELS[k] }}{% if k in INFRA %} infra{% endif %}
{{ DESCRIPTIONS[k] }}
{% endif %}{% endfor %} + {% for k in ORDER %}{% if oc[k] %}
{{ LABELS[k] }}{% if k in INFRA %} infra{% endif %}
{{ DESCRIPTIONS[k] }}
{% endif %}{% endfor %}
Adjusted rate
Solves divided by the trials that actually ran: it drops the outcomes marked infra, which failed before the agent could be judged. The raw rate is Harbor's mean over every trial, shown beside it only when the two differ. A run in which no trial carries a verdict shows —, never 0%.
{% if not control %}
Solved meter
One unit per trial that ran, solves filled; an infra trial is a hollow unit after a gap. Past 12 trials the units become one bar. A hovered trial row lights its unit.
{% endif %}
Tokens · Total
Under each trial's value a thin bar from zero, on one scale for the column: the largest trial fills its track. Total is the trial start to finish and includes overhead that Env, Agent and Verify do not.
From f71dc1e500268b821a39f96dcd48aaa5627d0c02 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Wed, 18 Nov 2026 16:25:02 +0100 Subject: [PATCH 7/8] fix(runs): check a job trial by trial until the run writing it ends A host run's copy can bring a job's final result.json while a trial's file is still mid-write; the next copy corrects the trial alone, which the head of an ended job does not see. While a run of this application writes a job, the job is checked trial by trial, and its head carries the run's end, so the last copy is checked once more. The web layer passes each run's job_state; a job no run writes is unchanged. --- app/nullius/workspace.py | 26 +++++++++++++++++--------- app/tests/test_workspace.py | 21 +++++++++++++++++++++ app/ui_tests/test_compare_kept.py | 2 +- app/ui_tests/test_comparison_routes.py | 2 +- app/ui_tests/test_controls_apart.py | 2 +- app/ui_tests/test_launch_shards.py | 2 +- app/ui_tests/test_live_board.py | 14 +++++++------- app/ui_tests/test_routes.py | 2 +- app/ui_tests/test_runs_by_launch.py | 2 +- app/web/runs.py | 2 +- app/web/shared.py | 15 ++++++++++++--- 11 files changed, 64 insertions(+), 26 deletions(-) diff --git a/app/nullius/workspace.py b/app/nullius/workspace.py index 53137e23..a8f9a9e2 100644 --- a/app/nullius/workspace.py +++ b/app/nullius/workspace.py @@ -25,7 +25,10 @@ config.json, and its folder, whose mtime moves when a trial folder comes or goes. A Harbor resume rewrites result.json, so it is caught; a file changed inside a trial of an ended job and nothing else is not. The workspace's -Refresh (``clear_cache``) reads every job again. +Refresh (``clear_cache``) reads every job again. A job a run of this +application writes is checked trial by trial while that run goes on (a host +run's copy can bring a trial's files after the job's own), and its head +carries the run's end, so the last copy is checked trial by trial once more. """ from __future__ import annotations @@ -434,19 +437,23 @@ def _write_cache(ws: Workspace, path: str, sig: str, head: str, job: parse.Job) "job": job_to_dict(job)}, fh, default=_encode, ensure_ascii=False)) -def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple: +def load_job(ws: Workspace, job_dir: str, index: dict | None = None, runs: dict | None = None) -> tuple: """(Job, from_cache). Parses only when the signature changed; an ended job whose head is unchanged is not stated trial by trial. ``index``: the - rail's index as load_jobs holds it, noted in place.""" - head = _head(job_dir) # before anything is read: a change after it is caught next time + rail's index as load_jobs holds it, noted in place. ``runs``: job folder + name -> the end of the run of this application that writes it, None while + that run goes on; a job no run writes has no entry.""" + run = (runs or {}).get(os.path.basename(job_dir), "") + short = run is not None # the head alone may answer + head = _head(job_dir) + (f"\nrun={run}" if run else "") # before anything is read: a change after it is caught next time path = _cache_file(ws, job_dir) with _DECODED_LOCK: known = _DECODED.get(path) - if known is not None and known[1].dir == job_dir and known[3] == head and _ended(known[1]): + if short and known is not None and known[1].dir == job_dir and known[3] == head and _ended(known[1]): _note_in_index(ws, job_dir, known[1], index) return known[1], True cached = _read_cache(path, job_dir) if known is None else None - if cached is not None and cached[1] == head and _ended(cached[2]): + if short and cached is not None and cached[1] == head and _ended(cached[2]): with _DECODED_LOCK: _DECODED[path] = (cached[0], cached[2], {}, head) _note_in_index(ws, job_dir, cached[2], index) @@ -482,14 +489,15 @@ def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple: return job, False -def load_jobs(ws: Workspace, roots) -> list: +def load_jobs(ws: Workspace, roots, runs: dict | None = None) -> list: """Every job under *roots*, each from the cache when its files are - unchanged. Same order and de-duplication as ``parse.load_all``.""" + unchanged. Same order and de-duplication as ``parse.load_all``. + ``runs``: as ``load_job``'s.""" # the rail's index read once and written once, not once per job: re-reading it for # each made a workspace of n jobs cost n parses of an index of n entries index = job_index(ws) before = dict(index) - jobs = [load_job(ws, d, index)[0] for d in parse.find_job_dirs(roots)] + jobs = [load_job(ws, d, index, runs)[0] for d in parse.find_job_dirs(roots)] if index != before: _write_index(ws, index) return parse.split_agents(jobs) diff --git a/app/tests/test_workspace.py b/app/tests/test_workspace.py index 3ee6b352..35f792db 100644 --- a/app/tests/test_workspace.py +++ b/app/tests/test_workspace.py @@ -362,6 +362,27 @@ def test_a_running_job_updates_with_each_trial(self): job, from_cache = workspace.load_job(self.ws, self.job_dir) self.assertFalse(from_cache); self.assertEqual(job.trials[1].reward, reward) + def test_a_host_run_s_trial_copied_after_its_job_is_read_once_the_run_ends(self): + """A host run's copy can bring the job's final result.json with a trial's + file still mid-write, and the next copy correct the trial alone.""" + self._end() + workspace.load_job(self.ws, self.job_dir, runs={"job-a": None}) # the stale trial parsed, its run going on + time.sleep(0.01) + self._rewrite_trial(1.0) + done = {"job-a": ("done", "2026-09-04T01:06:00Z")} # marked done after its last copy + job, from_cache = workspace.load_job(self.ws, self.job_dir, runs=done) + self.assertFalse(from_cache); self.assertEqual(job.trials[1].reward, 1.0) + with patch.object(workspace, "_signatures", side_effect=AssertionError("trials stated")): + again, from_cache = workspace.load_job(self.ws, self.job_dir, runs=done) + self.assertIs(again, job); self.assertTrue(from_cache) + + def test_a_job_no_run_writes_is_read_by_its_head(self): + self._end() + first, _ = workspace.load_job(self.ws, self.job_dir, runs={"job-b": None}) + with patch.object(workspace, "_signatures", side_effect=AssertionError("trials stated")): + again, from_cache = workspace.load_job(self.ws, self.job_dir, runs={"job-b": None}) + self.assertIs(again, first); self.assertTrue(from_cache) + def test_index_lists_runs_without_a_parse(self): self.assertEqual(workspace.job_index(self.ws), {}) workspace.load_job(self.ws, self.job_dir) diff --git a/app/ui_tests/test_compare_kept.py b/app/ui_tests/test_compare_kept.py index 9d2f06fe..bb3a5134 100644 --- a/app/ui_tests/test_compare_kept.py +++ b/app/ui_tests/test_compare_kept.py @@ -47,7 +47,7 @@ def scored(task, groups, **kw): self.stack.enter_context(patch.object(workspace, "list_all", return_value=list(ws.values()))) self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: ws.get(name))) - self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda w, roots: jobs[w.name])) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda w, roots, runs=None: jobs[w.name])) self.client = web.app.test_client() def get(self, path="/compare", **query): diff --git a/app/ui_tests/test_comparison_routes.py b/app/ui_tests/test_comparison_routes.py index 88a1539c..75d9893b 100644 --- a/app/ui_tests/test_comparison_routes.py +++ b/app/ui_tests/test_comparison_routes.py @@ -32,7 +32,7 @@ def setUp(self): self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: {"active": self.active, "foreign": self.foreign}.get(name))) - self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: self.jobs[ws.name])) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: self.jobs[ws.name])) for obj, name in ((web.shared, "open_workspace"), (web.shared, "_import_roots"), (workspace, "touch"), (workspace, "save")): self.stack.enter_context(patch.object(obj, name, side_effect=AssertionError(f"Unexpected workspace mutation: {name}"))) diff --git a/app/ui_tests/test_controls_apart.py b/app/ui_tests/test_controls_apart.py index 35120baf..cb1c33d5 100644 --- a/app/ui_tests/test_controls_apart.py +++ b/app/ui_tests/test_controls_apart.py @@ -76,7 +76,7 @@ def setUp(self): self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: self.ws)) self.jobs = [self.agent, self.nop, self.oracle] - self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: self.jobs)) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: self.jobs)) self.stack.enter_context(patch.object(web.shared, "_jobs_for_request", side_effect=lambda: self.jobs)) self.client = web.app.test_client() diff --git a/app/ui_tests/test_launch_shards.py b/app/ui_tests/test_launch_shards.py index a17e82a1..298f9cf9 100644 --- a/app/ui_tests/test_launch_shards.py +++ b/app/ui_tests/test_launch_shards.py @@ -82,7 +82,7 @@ def setUp(self): for name, value in (("list_all", [ws]), ("job_index", {})): self.stack.enter_context(patch.object(workspace, name, return_value=value)) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: ws)) - self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda w, roots: self.jobs)) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda w, roots, runs=None: self.jobs)) self.stack.enter_context(patch.object(web.shared, "_jobs_for_request", side_effect=lambda: self.jobs)) self.stack.enter_context(patch.dict(execution.EXECUTIONS, {}, clear=True)) self.client = web.app.test_client() diff --git a/app/ui_tests/test_live_board.py b/app/ui_tests/test_live_board.py index 6415561b..cafcbf46 100644 --- a/app/ui_tests/test_live_board.py +++ b/app/ui_tests/test_live_board.py @@ -98,7 +98,7 @@ def setUp(self): self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: self.ws)) self.stack.enter_context(patch.object(workspace, "load_jobs", - side_effect=lambda ws, roots: [self.running, self.ended, self.no_feed])) + side_effect=lambda ws, roots, runs=None: [self.running, self.ended, self.no_feed])) for obj, name in [(web.shared, "open_workspace"), (web.shared, "_import_roots"), (workspace, "save"), (workspace, "touch")]: self.stack.enter_context(patch.object(obj, name, side_effect=AssertionError(f"read-only route called {name}"))) self.client = web.app.test_client() @@ -156,7 +156,7 @@ def test_a_broken_feed_is_reported_not_fatal(self): complete = text[: text.rfind("\n") + 1] (broken / "events.jsonl").write_text(complete + "this is not json\n" + complete.splitlines()[0] + "\n", encoding="utf-8") job = parse.parse_job(str(broken)) - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: [job]): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: [job]): page = self.client.get("/job/broken-run"); self.addCleanup(page.close) self.assertEqual(page.status_code, 200) html = page.get_data(as_text=True) @@ -172,7 +172,7 @@ def test_live_route_reads_harbors_recorded_run(self): job_with_feed(self.root, "oracle-x4") shutil.copy(FIXTURE / "harbor-oracle-x4" / "events.jsonl", recorded / "events.jsonl") job = parse.parse_job(str(recorded)) - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: [job]): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: [job]): r = self.client.get("/job/oracle-x4/live"); self.addCleanup(r.close) self.assertEqual(r.status_code, 200) s = r.get_json() @@ -196,7 +196,7 @@ def test_a_launched_job_is_listed_as_preparing_and_its_page_waits_for_the_feed(s "datasets": [{"name": "tau3-demo", "n_tasks": 3}], "n_concurrent_trials": 3}) job = parse.parse_job(str(launched)) self.assertTrue(job.is_preparing) - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: [job]): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: [job]): runs = self.client.get("/runs"); self.addCleanup(runs.close) html = runs.get_data(as_text=True) self.assertIn(">preparing", html) @@ -257,7 +257,7 @@ def test_a_feed_with_only_job_prepare_puts_the_preparation_window_on_the_board(s (launched / "events.jsonl").write_text("\n".join(text[:2]) + "\n", encoding="utf-8") # as a poller saw it at 08:00:20 job = parse.parse_job(str(launched)) self.assertTrue(job.is_preparing) # no result record yet - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: [job]): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: [job]): r = self.client.get("/job/prepare-run/live"); self.addCleanup(r.close) snap = r.get_json() self.assertEqual((snap["state"], snap["running"], snap["preparing"]["stage"], snap["trials"]), ("preparing", False, "image", [])) @@ -291,7 +291,7 @@ def test_a_runner_s_arm_is_watched_on_its_board_then_read_as_results(self): (120, 60, 0.0004, 1, True, True)) self.assertIsNotNone(cases["lme-arm"].duration_min) self.assertEqual((cases["lme-arm"].runner_judge, cases["lme-arm"].questions_from), ("official gpt-4o · evaluate_qa.py", "longmemeval_s")) - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: list(cases.values())): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: list(cases.values())): for job_id, opens, base in (("lme-running", "live", ""), ("lme-arm", "results", "/job/lme-arm")): with self.subTest(job_id): page = self.client.get("/job/" + job_id); self.addCleanup(page.close) @@ -332,7 +332,7 @@ def test_a_task_scored_on_its_parts_shows_its_score_and_no_solve_rate(self): shutil.copytree(NATIVE / "parts-arm", arm) job = parse.parse_job(str(arm)) self.assertEqual(sorted((t.trial_name, t.outcome_label, t.score, t.n_parts) for t in job.trials), [("t1", "Scored", 0.5, 2), ("t2", "Scored", 1.0, 1)]) - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: [job]): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: [job]): page = self.client.get("/job/parts-arm"); self.addCleanup(page.close) html = page.get_data(as_text=True) self.assertEqual(page.status_code, 200) diff --git a/app/ui_tests/test_routes.py b/app/ui_tests/test_routes.py index 529eb82d..c29124bd 100644 --- a/app/ui_tests/test_routes.py +++ b/app/ui_tests/test_routes.py @@ -137,7 +137,7 @@ def setUp(self): self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: {"active": self.active, "foreign": self.foreign}.get(name))) self.load_jobs = self.stack.enter_context(patch.object(workspace, "load_jobs", - side_effect=lambda ws, roots: [self.foreign_job if ws.name == "foreign" else self.active_job])) + side_effect=lambda ws, roots, runs=None: [self.foreign_job if ws.name == "foreign" else self.active_job])) for obj, name in [(web.shared, "open_workspace"), (web.shared, "_import_roots"), (workspace, "save"), (workspace, "touch")]: self.stack.enter_context(patch.object(obj, name, side_effect=AssertionError( diff --git a/app/ui_tests/test_runs_by_launch.py b/app/ui_tests/test_runs_by_launch.py index bc91efc0..570c7e53 100644 --- a/app/ui_tests/test_runs_by_launch.py +++ b/app/ui_tests/test_runs_by_launch.py @@ -60,7 +60,7 @@ def setUp(self): self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: self.ws)) self.jobs = [self.later, self.strong, self.nop, self.weak, self.oracle] - self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: self.jobs)) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: self.jobs)) self.stack.enter_context(patch.object(web.shared, "_jobs_for_request", side_effect=lambda: self.jobs)) self.stack.enter_context(patch.dict(execution.EXECUTIONS, {}, clear=True)) self.client = web.app.test_client() diff --git a/app/web/runs.py b/app/web/runs.py index 429c1ab6..63188775 100644 --- a/app/web/runs.py +++ b/app/web/runs.py @@ -112,7 +112,7 @@ def _grouped(ws=None): shows: the runs grouped by the launch that made them (nullius.runs.launches) - the builder's run record, else the same dataset within half an hour. ``ws``: another workspace's, read from its cache without opening it.""" - jobs = shared._load_jobs() if ws is None else workspace.load_jobs(ws, ws.job_roots or ws.roots) + jobs = shared._load_jobs() if ws is None else workspace.load_jobs(ws, ws.job_roots or ws.roots, shared.run_ends()) have = {j.job_id for j in jobs} # The run records are the app's, not the workspace's: only those launched # into this workspace's roots, or with a job already written here, belong. diff --git a/app/web/shared.py b/app/web/shared.py index 732aa500..943c3054 100644 --- a/app/web/shared.py +++ b/app/web/shared.py @@ -364,7 +364,7 @@ def _load_jobs(): ws = STATE["workspace"] if ws is None: return [] - return workspace.load_jobs(ws, STATE["job_roots"]) + return workspace.load_jobs(ws, STATE["job_roots"], run_ends()) def request_memo(name: str) -> dict: @@ -390,6 +390,15 @@ def ended_for(job_id): return execution.ended_for(job_id, _owners()) +def run_ends(): + """Each job a run of this application writes -> that run's ``job_state``, the ``runs`` of + ``workspace.load_jobs``. Once per request: the Workspaces page reads every workspace.""" + memo = request_memo("run_ends") + if "run_ends" not in memo: + memo["run_ends"] = {name: e.job_state(name) for name, e in _owners().items()} + return memo["run_ends"] + + def _jobs_for_workspace(name=None): """Read an indexed workspace without making it active or converting sources. Once per request and workspace: a page that asks for the jobs several @@ -404,7 +413,7 @@ def _jobs_for_workspace(name=None): ws = workspace.load(WS_HOME, name) if ws is None: abort(404) - jobs = workspace.load_jobs(ws, ws.job_roots or ws.roots) + jobs = workspace.load_jobs(ws, ws.job_roots or ws.roots, run_ends()) memo[key] = jobs return jobs @@ -448,7 +457,7 @@ def _job_for_request(job_id): names = {job_id, job_id.rsplit("~", 1)[0]} dirs = [d for d in parse.find_job_dirs(STATE["job_roots"]) if os.path.basename(d) in names] if len(dirs) == 1: - job = _find_job(parse.split_agents([workspace.load_job(ws, dirs[0])[0]]), job_id) + job = _find_job(parse.split_agents([workspace.load_job(ws, dirs[0], runs=run_ends())[0]]), job_id) if job is not None: return job return _find_job(_jobs_for_request(), job_id) From 9de40ff294db9824cb958bb16e64516ca71132b4 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 10:52:26 +0100 Subject: [PATCH 8/8] fix(runs): count a part's end only once its whole run has ended A part's end is written down before the run's last copy from its host, for example under Stop. Counted alone, it let a page cache a job whose trial that copy then corrected. The Delete prompt reads jobs with the same run states. --- app/nullius/execution.py | 16 +++++++++++++--- app/tests/test_execution_stop.py | 15 +++++++++++++++ app/web/shared.py | 6 +++--- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 3dae04c9..4998f745 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -1686,6 +1686,15 @@ def owners() -> Dict[str, Execution]: return out +def run_ends(owner: Optional[Dict[str, Execution]] = None) -> Dict[str, Optional[tuple]]: + """Each job a run writes -> its ``job_state`` once the whole run has ended, + None while the run is alive: the ``runs`` of ``workspace.load_jobs``. A + part's end is written down before the run's last copy from its host. + ``owner``: ``owners()``, when the caller holds it.""" + return {name: None if e.state in ALIVE else e.job_state(name) + for name, e in (owners() if owner is None else owner).items()} + + def for_job(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Optional[Execution]: """The newest execution whose script writes this job, if any - the one the job page shows beside the job, or instead of it while Harbor has @@ -1823,7 +1832,7 @@ def deletable(names: List[str], root: Optional[str] = None) -> None: _deletable(n, root) -def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable) -> dict: +def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable, ends: dict) -> dict: path, rec = _deletable(name, root) plan = rec.get("plan") or {} # a run refused at the name check never made the host folder: one of that name there is not its own @@ -1831,7 +1840,7 @@ def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable) held = [] for ws, roots in holding(path): dirs = parse.find_job_dirs(roots) - ids = [j.job_id for j in workspace.load_jobs(ws, roots)] + ids = [j.job_id for j in workspace.load_jobs(ws, roots, ends)] # a job of the same id from another root shares these judgements: they stay others = {e.get("job_id") for d, e in workspace.job_index(ws).items() if d not in dirs} judged = judgements.job_dirs(ws.dir, [i for i in ids if i.split("~")[0] not in others]) @@ -1849,7 +1858,8 @@ def deletion(names: List[str], home: str, root: Optional[str] = None, holding: O a run: nothing goes then. ``holding``: a run folder's ``workspace.holding``, when the caller has it already.""" holding = holding or (lambda path: workspace.holding(home, path)) - runs = [_run_deletion(n, home, root, holding) for n in dict.fromkeys(names)] + ends = run_ends() + runs = [_run_deletion(n, home, root, holding, ends) for n in dict.fromkeys(names)] copies: Dict[str, int] = {} for r in runs: if r["host"]: diff --git a/app/tests/test_execution_stop.py b/app/tests/test_execution_stop.py index a0f8e0e3..8115c052 100644 --- a/app/tests/test_execution_stop.py +++ b/app/tests/test_execution_stop.py @@ -51,6 +51,21 @@ def test_a_stop_as_a_part_ends_is_never_paused_over(self): self.assertEqual(ex.ends["oracle"]["rc"], 3) # it did end, with its own result self.assertFalse((self.tmp / "r" / "run-next.log").exists()) + def test_a_stopped_part_s_job_is_written_until_the_run_ends(self): + """Its part's end is written down before the run's last copy from its + host: until the run itself ends, ``run_ends`` says the job is still written.""" + plan = execution.Plan(name="r", script="", parts=self.PARTS, jobs=[{"name": "r-oracle", "agent": "oracle"}]) + ex = execution.Execution(plan, root=str(self.tmp)) + seen = [] + class Seen(StopAsItEnds): + def __setitem__(self, key, value): + super().__setitem__(key, value) + seen.append((ex.state, execution.run_ends({"r-oracle": ex})["r-oracle"])) + ex.ends = Seen(ex) + self.wait(ex.start(), 5) + self.assertEqual(seen, [("stopping", None)]) + self.assertEqual(execution.run_ends({"r-oracle": ex})["r-oracle"][0], "failed") # the part's own result, once the run ended + def test_a_run_stopped_before_any_part_says_it_was_stopped(self): """Stopped as the target was made ready: no part ran, the run's exit is the one it had (0), and its last line says it was stopped and diff --git a/app/web/shared.py b/app/web/shared.py index 943c3054..fb15f90d 100644 --- a/app/web/shared.py +++ b/app/web/shared.py @@ -391,11 +391,11 @@ def ended_for(job_id): def run_ends(): - """Each job a run of this application writes -> that run's ``job_state``, the ``runs`` of - ``workspace.load_jobs``. Once per request: the Workspaces page reads every workspace.""" + """``execution.run_ends`` over this request's ``_owners``, once per request: + the Workspaces page reads every workspace.""" memo = request_memo("run_ends") if "run_ends" not in memo: - memo["run_ends"] = {name: e.job_state(name) for name, e in _owners().items()} + memo["run_ends"] = execution.run_ends(_owners()) return memo["run_ends"]